From 522ffe7da1a7eeb2fdc31f37f70ef5fe2bbb3ddf Mon Sep 17 00:00:00 2001 From: Marat K Date: Tue, 27 Oct 2026 16:09:02 +0100 Subject: [PATCH 1/3] bench: measure PHP source mutations Compare native and pure mutations in GitHub CI with exact-source and independent semantic oracles. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/benchmark.yml | 57 +++++++++++++++++++++++ bench/mutations.php | 81 +++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 bench/mutations.php diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..7d10402 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,57 @@ +name: Mutation benchmark (manual) + +on: + pull_request: + paths: + - 'bench/mutations.php' + - '.github/workflows/benchmark.yml' + - 'src/**' + workflow_dispatch: + inputs: + warmups: + description: Warmup mutation iterations per fixture/backend + required: false + default: '5' + repetitions: + description: Timed mutation iterations per fixture/backend + required: false + default: '30' + +permissions: + contents: read + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: dtolnay/rust-toolchain@stable + - name: Build native release library + run: cargo build --release -p ktav-cabi + - name: Setup PHP 7.4 with FFI + uses: shivammathur/setup-php@v2 + with: + php-version: '7.4' + extensions: ffi, json, mbstring + ini-values: ffi.enable=1 + - name: Install Composer dependencies and generate autoloader + run: composer install --no-interaction --no-progress --prefer-dist + - name: Run mutation benchmark + env: + KTAV_LIB_PATH: ${{ github.workspace }}/target/release/libktav_cabi.so + BENCH_WARMUPS: ${{ inputs.warmups }} + BENCH_REPETITIONS: ${{ inputs.repetitions }} + BENCH_SERIES: '7' + shell: bash + run: | + set -euo pipefail + php -d ffi.enable=1 bench/mutations.php | tee mutation-benchmark.json + - name: Publish benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: mutation-benchmark-results + path: mutation-benchmark.json + if-no-files-found: warn diff --git a/bench/mutations.php b/bench/mutations.php new file mode 100644 index 0000000..d2ea251 --- /dev/null +++ b/bench/mutations.php @@ -0,0 +1,81 @@ + 10000) { + fwrite(STDERR, "$label must be an integer between $minimum and 10000\n"); exit(2); + } + return $number; +} + +function checkResult(string $backend, int $fixture, string $actual, string $literal, array $semantic): void +{ + if ($actual !== $literal) { throw new RuntimeException("$backend fixture $fixture literal bytes differ"); } + if ($backend === 'native') { $parsed = Ktav::loads($actual); } + else { $parsed = PureBackend::loads($actual); } + if ($parsed !== $semantic) { throw new RuntimeException("$backend fixture $fixture semantic oracle differs"); } +} + +$warmups = boundedBenchmarkInt(getenv('BENCH_WARMUPS'), 'warmups', 5, 0); +$repetitions = boundedBenchmarkInt(getenv('BENCH_REPETITIONS'), 'repetitions', 30, 1); +$series = boundedBenchmarkInt(getenv('BENCH_SERIES'), 'series', 7, 1); +if (!extension_loaded('ffi')) { fwrite(STDERR, "FFI extension required\n"); exit(2); } + +$cases = [ + ['insert', "cfg: {old: +01}\n", ['cfg', 'new'], 2, "cfg: {old: +01, new: 2}\n", ['cfg' => ['old' => 1, 'new' => 2]]], + ['remove', "obj: {a: 1}\r\n", ['obj', 'a'], null, "obj: {}\r\n", ['obj' => []]], + ['rename', "cfg: {\"a.b\": 1, c: 2}\n", ['cfg', 'a.b'], 'd', "cfg: {d: 1, c: 2}\n", ['cfg' => ['d' => 1, 'c' => 2]]], + ['insert', "\xEF\xBB\xBFfirst: 1\r\ncfg: [{a: 1,}]\nlast: 2\r", ['cfg', '0', 'b'], 2, "\xEF\xBB\xBFfirst: 1\r\ncfg: [{a: 1, b: 2,}]\nlast: 2\r", ['first' => 1, 'cfg' => [['a' => 1, 'b' => 2]], 'last' => 2]], +]; +$operations = [ + 'insert' => static function ($backend, $src, $path, $value) { return $backend::insertValueSegments($src, $path, $value); }, + 'remove' => static function ($backend, $src, $path) { return $backend::removeValueSegments($src, $path); }, + 'rename' => static function ($backend, $src, $path, $value) { return $backend::renameKeySegments($src, $path, $value); }, +]; +$backends = ['native' => Ktav::class, 'pure' => PureBackend::class]; +$order = []; +for ($s = 0; $s < $series; ++$s) { $order[] = $s % 2 ? ['pure', 'native'] : ['native', 'pure']; } +$raw = []; +foreach ($cases as $index => $case) { + [$op, $source, $path, $value, $literal, $semantic] = $case; + $samplesByBackend = ['native' => [], 'pure' => []]; + foreach ($order as $seriesIndex => $backendOrder) { + foreach ($backendOrder as $name) { + $backend = $backends[$name]; + $invoke = static function () use ($operations, $op, $backend, $source, $path, $value) { return $operations[$op]($backend, $source, $path, $value); }; + checkResult($name, $index, $invoke(), $literal, $semantic); + for ($i = 0; $i < $warmups; ++$i) { $invoke(); } + $start = hrtime(true); + $lastOutput = null; + for ($j = 0; $j < $repetitions; ++$j) { $lastOutput = $invoke(); } + $samplesByBackend[$name][] = hrtime(true) - $start; + // Validate the actual final output from this timed batch, outside timing. + checkResult($name, $index, $lastOutput, $literal, $semantic); + } + } + foreach ($samplesByBackend as $name => $samples) { + $means = array_map(static function ($n) use ($repetitions) { return $n / $repetitions; }, $samples); + $raw[] = ['backend' => $name, 'operation' => $case[0], 'fixture' => $index, 'raw_total_mutation_ns' => $samples, 'raw_mean_mutation_ns' => $means, 'median_mutation_ns' => benchmarkMedian($means), 'warmups' => $warmups, 'repetitions_per_sample' => $repetitions, 'series' => $series]; + } +} +echo json_encode(['php_version' => PHP_VERSION, 'os' => PHP_OS . ' ' . php_uname('r'), 'method' => 'mutation-only timed regions; alternating backend order by series; literal bytes and independent semantic oracle checked before and after measurement; descriptive only, no speedup claim', 'raw_results' => $raw], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), "\n"; From 3665dcfd6afb30b69ac5afb0ed8b4ce147df6325 Mon Sep 17 00:00:00 2001 From: Marat K Date: Tue, 27 Oct 2026 16:21:51 +0100 Subject: [PATCH 2/3] fix(bench): expect leading-zero string Keep strict independent semantic assertions and report actual values on mismatches. Co-Authored-By: Claude Sonnet 5.5 --- bench/mutations.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bench/mutations.php b/bench/mutations.php index d2ea251..6d5f630 100644 --- a/bench/mutations.php +++ b/bench/mutations.php @@ -33,7 +33,17 @@ function checkResult(string $backend, int $fixture, string $actual, string $lite if ($actual !== $literal) { throw new RuntimeException("$backend fixture $fixture literal bytes differ"); } if ($backend === 'native') { $parsed = Ktav::loads($actual); } else { $parsed = PureBackend::loads($actual); } - if ($parsed !== $semantic) { throw new RuntimeException("$backend fixture $fixture semantic oracle differs"); } + if ($parsed !== $semantic) { + throw new RuntimeException(sprintf( + "%s fixture %d semantic oracle differs: actual=%s (%s), expected=%s (%s)", + $backend, + $fixture, + var_export($parsed, true), + gettype($parsed), + var_export($semantic, true), + gettype($semantic) + )); + } } $warmups = boundedBenchmarkInt(getenv('BENCH_WARMUPS'), 'warmups', 5, 0); @@ -42,7 +52,7 @@ function checkResult(string $backend, int $fixture, string $actual, string $lite if (!extension_loaded('ffi')) { fwrite(STDERR, "FFI extension required\n"); exit(2); } $cases = [ - ['insert', "cfg: {old: +01}\n", ['cfg', 'new'], 2, "cfg: {old: +01, new: 2}\n", ['cfg' => ['old' => 1, 'new' => 2]]], + ['insert', "cfg: {old: +01}\n", ['cfg', 'new'], 2, "cfg: {old: +01, new: 2}\n", ['cfg' => ['old' => '+01', 'new' => 2]]], ['remove', "obj: {a: 1}\r\n", ['obj', 'a'], null, "obj: {}\r\n", ['obj' => []]], ['rename', "cfg: {\"a.b\": 1, c: 2}\n", ['cfg', 'a.b'], 'd', "cfg: {d: 1, c: 2}\n", ['cfg' => ['d' => 1, 'c' => 2]]], ['insert', "\xEF\xBB\xBFfirst: 1\r\ncfg: [{a: 1,}]\nlast: 2\r", ['cfg', '0', 'b'], 2, "\xEF\xBB\xBFfirst: 1\r\ncfg: [{a: 1, b: 2,}]\nlast: 2\r", ['first' => 1, 'cfg' => [['a' => 1, 'b' => 2]], 'last' => 2]], From 52dd602dd166b3607a1c8a29e5c76fa5af48955d Mon Sep 17 00:00:00 2001 From: Marat K Date: Tue, 27 Oct 2026 16:43:58 +0100 Subject: [PATCH 3/3] perf(pure): reuse inline trim bounds Avoid duplicate boundary scanning during insertion and compare base and candidate on the same CI runner with exact and semantic oracles. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/benchmark.yml | 38 +++++++++- bench/compare-mutations.php | 116 ++++++++++++++++++++++++++++++ src/Pure/Edit/EditSplice.php | 3 +- tests/Pure/InsertMutationSpec.php | 11 +++ 4 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 bench/compare-mutations.php diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7d10402..f1541b8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -3,7 +3,7 @@ name: Mutation benchmark (manual) on: pull_request: paths: - - 'bench/mutations.php' + - 'bench/**' - '.github/workflows/benchmark.yml' - 'src/**' workflow_dispatch: @@ -27,6 +27,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive + fetch-depth: 0 - uses: dtolnay/rust-toolchain@stable - name: Build native release library run: cargo build --release -p ktav-cabi @@ -38,7 +39,7 @@ jobs: ini-values: ffi.enable=1 - name: Install Composer dependencies and generate autoloader run: composer install --no-interaction --no-progress --prefer-dist - - name: Run mutation benchmark + - name: Run existing native/pure mutation benchmark env: KTAV_LIB_PATH: ${{ github.workspace }}/target/release/libktav_cabi.so BENCH_WARMUPS: ${{ inputs.warmups }} @@ -48,10 +49,41 @@ jobs: run: | set -euo pipefail php -d ffi.enable=1 bench/mutations.php | tee mutation-benchmark.json + - name: Prepare immutable benchmark source trees + shell: bash + env: + BASE_REF: 3665dcfd6afb30b69ac5afb0ed8b4ce147df6325 + CANDIDATE_REF: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + set -euo pipefail + [[ "$BASE_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$CANDIDATE_REF" =~ ^[0-9a-f]{40}$ ]] + git cat-file -e "$BASE_REF^{commit}" + git cat-file -e "$CANDIDATE_REF^{commit}" + mkdir -p "$RUNNER_TEMP/benchmark-base/src" "$RUNNER_TEMP/benchmark-candidate/src" + git archive "$BASE_REF" src | tar -x -C "$RUNNER_TEMP/benchmark-base" + git archive "$CANDIDATE_REF" src | tar -x -C "$RUNNER_TEMP/benchmark-candidate" + test -d "$RUNNER_TEMP/benchmark-base/src" + test -d "$RUNNER_TEMP/benchmark-candidate/src" + - name: Run same-runner pure comparison + env: + BENCH_BASE_SRC: ${{ runner.temp }}/benchmark-base/src + BENCH_CANDIDATE_SRC: ${{ runner.temp }}/benchmark-candidate/src + BENCH_BASE_REF: 3665dcfd6afb30b69ac5afb0ed8b4ce147df6325 + BENCH_CANDIDATE_REF: ${{ github.event.pull_request.head.sha || github.sha }} + BENCH_WARMUPS: ${{ inputs.warmups }} + BENCH_REPETITIONS: ${{ inputs.repetitions }} + BENCH_SERIES: '7' + shell: bash + run: | + set -euo pipefail + php -d ffi.enable=1 bench/compare-mutations.php | tee mutation-comparison.json - name: Publish benchmark results if: always() uses: actions/upload-artifact@v4 with: name: mutation-benchmark-results - path: mutation-benchmark.json + path: | + mutation-benchmark.json + mutation-comparison.json if-no-files-found: warn diff --git a/bench/compare-mutations.php b/bench/compare-mutations.php new file mode 100644 index 0000000..a03376d --- /dev/null +++ b/bench/compare-mutations.php @@ -0,0 +1,116 @@ + 10000) { + throw new RuntimeException("$name must be between $minimum and 10000"); + } + return (int)$value; +} + +function sameRunBenchmark(string $srcRoot, string $label, int $warmups, int $repetitions): array +{ + spl_autoload_register(static function (string $class) use ($srcRoot): void { + $prefix = 'Ktav\\'; + if (strncmp($class, $prefix, strlen($prefix)) !== 0) { return; } + $file = $srcRoot . '/' . str_replace('\\', '/', substr($class, strlen($prefix))) . '.php'; + if (!is_file($file)) { throw new RuntimeException("Class file missing from selected source tree: $class"); } + require $file; + }, true, true); + if (!class_exists('Ktav\\PureBackend')) { throw new RuntimeException("PureBackend missing from selected source tree: $srcRoot"); } + $cases = [ + ['insert', "cfg: {old: +01}\n", ['cfg', 'new'], 2, "cfg: {old: +01, new: 2}\n", ['cfg' => ['old' => '+01', 'new' => 2]]], + ['remove', "obj: {a: 1}\r\n", ['obj', 'a'], null, "obj: {}\r\n", ['obj' => []]], + ['rename', "cfg: {\"a.b\": 1, c: 2}\n", ['cfg', 'a.b'], 'd', "cfg: {d: 1, c: 2}\n", ['cfg' => ['d' => 1, 'c' => 2]]], + ['insert', "\xEF\xBB\xBFfirst: 1\r\ncfg: [{a: 1,}]\nlast: 2\r", ['cfg', '0', 'b'], 2, "\xEF\xBB\xBFfirst: 1\r\ncfg: [{a: 1, b: 2,}]\nlast: 2\r", ['first' => 1, 'cfg' => [['a' => 1, 'b' => 2]], 'last' => 2]], + ]; + $ops = [ + 'insert' => static function ($src, $path, $value) { return PureBackend::insertValueSegments($src, $path, $value); }, + 'remove' => static function ($src, $path) { return PureBackend::removeValueSegments($src, $path); }, + 'rename' => static function ($src, $path, $value) { return PureBackend::renameKeySegments($src, $path, $value); }, + ]; + $results = []; + foreach ($cases as $index => $case) { + [$op, $source, $path, $value, $literal, $semantic] = $case; + $invoke = static function () use ($ops, $op, $source, $path, $value) { return $ops[$op]($source, $path, $value); }; + sameRunCheck($label, $index, $invoke(), $literal, $semantic); + for ($i = 0; $i < $warmups; ++$i) { $invoke(); } + $start = hrtime(true); + $last = null; + for ($i = 0; $i < $repetitions; ++$i) { $last = $invoke(); } + $elapsed = hrtime(true) - $start; + sameRunCheck($label, $index, $last, $literal, $semantic); + $results[] = ['backend' => $label, 'operation' => $op, 'fixture' => $index, 'raw_total_mutation_ns' => $elapsed, 'raw_mean_mutation_ns' => $elapsed / $repetitions, 'warmups' => $warmups, 'repetitions' => $repetitions]; + } + return $results; +} + +if (getenv('BENCH_CHILD') === '1') { + try { + $src = getenv('BENCH_SRC'); + if (!is_string($src) || !is_dir($src) || !in_array(getenv('BENCH_LABEL'), ['base', 'candidate'], true)) { throw new RuntimeException('Invalid child source/label'); } + $warmups = sameRunInt('BENCH_WARMUPS', 5, 0); + $repetitions = sameRunInt('BENCH_REPETITIONS', 30, 1); + $series = sameRunInt('BENCH_SERIES', 7, 1); + if ($series !== 1) { throw new RuntimeException('Child BENCH_SERIES must be 1'); } + echo json_encode(['raw_results' => sameRunBenchmark($src, getenv('BENCH_LABEL'), $warmups, $repetitions)], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), "\n"; + } catch (Throwable $e) { fwrite(STDERR, $e->getMessage() . "\n"); exit(1); } + exit(0); +} + +function runPureChild(string $src, string $label, string $baseRef, string $candidateRef, int $warmups, int $repetitions): array +{ + $env = array_merge(getenv(), ['BENCH_CHILD' => '1', 'BENCH_SRC' => $src, 'BENCH_LABEL' => $label, 'BENCH_WARMUPS' => (string)$warmups, 'BENCH_REPETITIONS' => (string)$repetitions, 'BENCH_SERIES' => '1', 'BENCH_BASE_REF' => $baseRef, 'BENCH_CANDIDATE_REF' => $candidateRef]); + $stdoutFile = tmpfile(); + $stderrFile = tmpfile(); + if ($stdoutFile === false || $stderrFile === false) { throw new RuntimeException('Unable to create child output streams'); } + $proc = proc_open([PHP_BINARY, __FILE__], [0 => ['pipe', 'r'], 1 => $stdoutFile, 2 => $stderrFile], $pipes, null, $env); + if (!is_resource($proc)) { fclose($stdoutFile); fclose($stderrFile); throw new RuntimeException('Unable to start child'); } + fclose($pipes[0]); + $status = proc_close($proc); + rewind($stdoutFile); + rewind($stderrFile); + $stdout = stream_get_contents($stdoutFile); + $stderr = stream_get_contents($stderrFile); + fclose($stdoutFile); + fclose($stderrFile); + if ($status !== 0) { throw new RuntimeException("$label child failed: $stderr"); } + $data = json_decode($stdout, true); + if (!is_array($data) || !isset($data['raw_results']) || !is_array($data['raw_results'])) { throw new RuntimeException("Invalid $label child output"); } + return $data['raw_results']; +} + +try { + $base = getenv('BENCH_BASE_SRC'); + $candidate = getenv('BENCH_CANDIDATE_SRC'); + $baseRef = getenv('BENCH_BASE_REF'); + $candidateRef = getenv('BENCH_CANDIDATE_REF'); + if (!is_string($base) || !is_dir($base) || !is_string($candidate) || !is_dir($candidate) + || $baseRef !== '3665dcfd6afb30b69ac5afb0ed8b4ce147df6325' + || !is_string($candidateRef) || !preg_match('/^[0-9a-f]{40}$/', $candidateRef)) { throw new RuntimeException('Invalid/missing refs or source trees'); } + $warmups = sameRunInt('BENCH_WARMUPS', 5, 0); + $repetitions = sameRunInt('BENCH_REPETITIONS', 30, 1); + $series = sameRunInt('BENCH_SERIES', 7, 1); + $runs = []; + for ($round = 0; $round < $series; ++$round) { + foreach (($round % 2 === 0 ? ['base', 'candidate'] : ['candidate', 'base']) as $which) { + $runs[] = ['ref' => $which === 'base' ? $baseRef : $candidateRef, 'round' => $round, 'results' => runPureChild($which === 'base' ? $base : $candidate, $which, $baseRef, $candidateRef, $warmups, $repetitions)]; + } + } + echo json_encode(['php_version' => PHP_VERSION, 'base_ref' => $baseRef, 'candidate_ref' => $candidateRef, 'method' => 'one child process per source per round; alternating order; strict literal and semantic oracle; no timing gate', 'raw_runs' => $runs], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), "\n"; +} catch (Throwable $e) { fwrite(STDERR, $e->getMessage() . "\n"); exit(1); } diff --git a/src/Pure/Edit/EditSplice.php b/src/Pure/Edit/EditSplice.php index 56d56a8..ea76c07 100644 --- a/src/Pure/Edit/EditSplice.php +++ b/src/Pure/Edit/EditSplice.php @@ -451,7 +451,8 @@ private static function insertSplice(string $src, $original, array $parentSegmen && (strpos($c['v'], "\n") !== false || strpos($c['v'], "\r") !== false || PureText::stringNeedsMultiline($c['v'])) )) && self::blockEncodable($c); - if (PureScanner::trimBounds($inner, 0, strlen($inner))[0] === PureScanner::trimBounds($inner, 0, strlen($inner))[1]) { + list($innerStart, $innerEnd) = PureScanner::trimBounds($inner, 0, strlen($inner)); + if ($innerStart === $innerEnd) { $spliceFrom = $from + 1; $spliceTo = $to - 1; $insert = $pair; diff --git a/tests/Pure/InsertMutationSpec.php b/tests/Pure/InsertMutationSpec.php index 7902d7e..b283c45 100644 --- a/tests/Pure/InsertMutationSpec.php +++ b/tests/Pure/InsertMutationSpec.php @@ -46,6 +46,17 @@ ->toBe("cfg: {a: 1, b: 2, }\n"); }); + it('preserves inline empty, whitespace, trailing-comma, and semantic exact-byte fixtures', function () use ($insert) { + expect($insert("a: {}\n", 'a.k', 'v'))->toBe("a: {k: v}\n"); + expect($insert("a: { }\n", 'a.k', 'v'))->toBe("a: {k: v}\n"); + expect($insert("a: {x: 1, }\n", 'a.k', 2))->toBe("a: {x: 1, k: 2, }\n"); + + $source = "a: {x: +01, }\n"; + $mutated = $insert($source, 'a.k', 2); + expect($mutated)->toBe("a: {x: +01, k: 2, }\n"); + expect(PureBackend::loads($mutated)['a'])->toBe(['x' => '+01', 'k' => 2]); + }); + it('preserves mixed EOLs and BOM while mutating an Object nested in an Array', function () use ($insert) { $source = "\xEF\xBB\xBFfirst: 1\r\ncfg: [{a: 1,}]\nlast: 2\r"; expect($insert($source, 'cfg.0.b', 2))