Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 89 additions & 0 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
name: Mutation benchmark (manual)

on:
pull_request:
paths:
- 'bench/**'
- '.github/workflows/benchmark.yml'
- 'src/**'
workflow_dispatch:
inputs:
warmups:
description: Warmup mutation iterations per fixture/backend
required: false
default: '5'
repetitions:
description: Timed mutation iterations per fixture/backend
required: false
default: '30'

permissions:
contents: read

jobs:
benchmark:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
fetch-depth: 0
- uses: dtolnay/rust-toolchain@stable
- name: Build native release library
run: cargo build --release -p ktav-cabi
- name: Setup PHP 7.4 with FFI
uses: shivammathur/setup-php@v2
with:
php-version: '7.4'
extensions: ffi, json, mbstring
ini-values: ffi.enable=1
- name: Install Composer dependencies and generate autoloader
run: composer install --no-interaction --no-progress --prefer-dist
- name: Run existing native/pure mutation benchmark
env:
KTAV_LIB_PATH: ${{ github.workspace }}/target/release/libktav_cabi.so
BENCH_WARMUPS: ${{ inputs.warmups }}
BENCH_REPETITIONS: ${{ inputs.repetitions }}
BENCH_SERIES: '7'
shell: bash
run: |
set -euo pipefail
php -d ffi.enable=1 bench/mutations.php | tee mutation-benchmark.json
- name: Prepare immutable benchmark source trees
shell: bash
env:
BASE_REF: 3665dcfd6afb30b69ac5afb0ed8b4ce147df6325
CANDIDATE_REF: ${{ github.event.pull_request.head.sha || github.sha }}
run: |
set -euo pipefail
[[ "$BASE_REF" =~ ^[0-9a-f]{40}$ ]]
[[ "$CANDIDATE_REF" =~ ^[0-9a-f]{40}$ ]]
git cat-file -e "$BASE_REF^{commit}"
git cat-file -e "$CANDIDATE_REF^{commit}"
mkdir -p "$RUNNER_TEMP/benchmark-base/src" "$RUNNER_TEMP/benchmark-candidate/src"
git archive "$BASE_REF" src | tar -x -C "$RUNNER_TEMP/benchmark-base"
git archive "$CANDIDATE_REF" src | tar -x -C "$RUNNER_TEMP/benchmark-candidate"
test -d "$RUNNER_TEMP/benchmark-base/src"
test -d "$RUNNER_TEMP/benchmark-candidate/src"
- name: Run same-runner pure comparison
env:
BENCH_BASE_SRC: ${{ runner.temp }}/benchmark-base/src
BENCH_CANDIDATE_SRC: ${{ runner.temp }}/benchmark-candidate/src
BENCH_BASE_REF: 3665dcfd6afb30b69ac5afb0ed8b4ce147df6325
BENCH_CANDIDATE_REF: ${{ github.event.pull_request.head.sha || github.sha }}
BENCH_WARMUPS: ${{ inputs.warmups }}
BENCH_REPETITIONS: ${{ inputs.repetitions }}
BENCH_SERIES: '7'
shell: bash
run: |
set -euo pipefail
php -d ffi.enable=1 bench/compare-mutations.php | tee mutation-comparison.json
- name: Publish benchmark results
if: always()
uses: actions/upload-artifact@v4
with:
name: mutation-benchmark-results
path: |
mutation-benchmark.json
mutation-comparison.json
if-no-files-found: warn
116 changes: 116 additions & 0 deletions bench/compare-mutations.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
<?php

declare(strict_types=1);

use Ktav\PureBackend;

function sameRunCheck(string $label, int $fixture, string $actual, string $literal, array $semantic): void
{
if ($actual !== $literal) { throw new RuntimeException("$label fixture $fixture literal bytes differ"); }
$parsed = PureBackend::loads($actual);
if ($parsed !== $semantic) {
throw new RuntimeException(sprintf('%s fixture %d semantic oracle differs: actual=%s expected=%s', $label, $fixture, var_export($parsed, true), var_export($semantic, true)));
}
}

function sameRunInt(string $name, int $default, int $minimum): int
{
$value = getenv($name);
if ($value === false || $value === '') { return $default; }
if (!preg_match('/^[0-9]{1,5}$/', $value) || (int)$value < $minimum || (int)$value > 10000) {
throw new RuntimeException("$name must be between $minimum and 10000");
}
return (int)$value;
}

function sameRunBenchmark(string $srcRoot, string $label, int $warmups, int $repetitions): array
{
spl_autoload_register(static function (string $class) use ($srcRoot): void {
$prefix = 'Ktav\\';
if (strncmp($class, $prefix, strlen($prefix)) !== 0) { return; }
$file = $srcRoot . '/' . str_replace('\\', '/', substr($class, strlen($prefix))) . '.php';
if (!is_file($file)) { throw new RuntimeException("Class file missing from selected source tree: $class"); }
require $file;
}, true, true);
if (!class_exists('Ktav\\PureBackend')) { throw new RuntimeException("PureBackend missing from selected source tree: $srcRoot"); }
$cases = [
['insert', "cfg: {old: +01}\n", ['cfg', 'new'], 2, "cfg: {old: +01, new: 2}\n", ['cfg' => ['old' => '+01', 'new' => 2]]],
['remove', "obj: {a: 1}\r\n", ['obj', 'a'], null, "obj: {}\r\n", ['obj' => []]],
['rename', "cfg: {\"a.b\": 1, c: 2}\n", ['cfg', 'a.b'], 'd', "cfg: {d: 1, c: 2}\n", ['cfg' => ['d' => 1, 'c' => 2]]],
['insert', "\xEF\xBB\xBFfirst: 1\r\ncfg: [{a: 1,}]\nlast: 2\r", ['cfg', '0', 'b'], 2, "\xEF\xBB\xBFfirst: 1\r\ncfg: [{a: 1, b: 2,}]\nlast: 2\r", ['first' => 1, 'cfg' => [['a' => 1, 'b' => 2]], 'last' => 2]],
];
$ops = [
'insert' => static function ($src, $path, $value) { return PureBackend::insertValueSegments($src, $path, $value); },
'remove' => static function ($src, $path) { return PureBackend::removeValueSegments($src, $path); },
'rename' => static function ($src, $path, $value) { return PureBackend::renameKeySegments($src, $path, $value); },
];
$results = [];
foreach ($cases as $index => $case) {
[$op, $source, $path, $value, $literal, $semantic] = $case;
$invoke = static function () use ($ops, $op, $source, $path, $value) { return $ops[$op]($source, $path, $value); };
sameRunCheck($label, $index, $invoke(), $literal, $semantic);
for ($i = 0; $i < $warmups; ++$i) { $invoke(); }
$start = hrtime(true);
$last = null;
for ($i = 0; $i < $repetitions; ++$i) { $last = $invoke(); }
$elapsed = hrtime(true) - $start;
sameRunCheck($label, $index, $last, $literal, $semantic);
$results[] = ['backend' => $label, 'operation' => $op, 'fixture' => $index, 'raw_total_mutation_ns' => $elapsed, 'raw_mean_mutation_ns' => $elapsed / $repetitions, 'warmups' => $warmups, 'repetitions' => $repetitions];
}
return $results;
}

if (getenv('BENCH_CHILD') === '1') {
try {
$src = getenv('BENCH_SRC');
if (!is_string($src) || !is_dir($src) || !in_array(getenv('BENCH_LABEL'), ['base', 'candidate'], true)) { throw new RuntimeException('Invalid child source/label'); }
$warmups = sameRunInt('BENCH_WARMUPS', 5, 0);
$repetitions = sameRunInt('BENCH_REPETITIONS', 30, 1);
$series = sameRunInt('BENCH_SERIES', 7, 1);
if ($series !== 1) { throw new RuntimeException('Child BENCH_SERIES must be 1'); }
echo json_encode(['raw_results' => sameRunBenchmark($src, getenv('BENCH_LABEL'), $warmups, $repetitions)], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), "\n";
} catch (Throwable $e) { fwrite(STDERR, $e->getMessage() . "\n"); exit(1); }
exit(0);
}

function runPureChild(string $src, string $label, string $baseRef, string $candidateRef, int $warmups, int $repetitions): array
{
$env = array_merge(getenv(), ['BENCH_CHILD' => '1', 'BENCH_SRC' => $src, 'BENCH_LABEL' => $label, 'BENCH_WARMUPS' => (string)$warmups, 'BENCH_REPETITIONS' => (string)$repetitions, 'BENCH_SERIES' => '1', 'BENCH_BASE_REF' => $baseRef, 'BENCH_CANDIDATE_REF' => $candidateRef]);
$stdoutFile = tmpfile();
$stderrFile = tmpfile();
if ($stdoutFile === false || $stderrFile === false) { throw new RuntimeException('Unable to create child output streams'); }
$proc = proc_open([PHP_BINARY, __FILE__], [0 => ['pipe', 'r'], 1 => $stdoutFile, 2 => $stderrFile], $pipes, null, $env);
if (!is_resource($proc)) { fclose($stdoutFile); fclose($stderrFile); throw new RuntimeException('Unable to start child'); }
fclose($pipes[0]);
$status = proc_close($proc);
rewind($stdoutFile);
rewind($stderrFile);
$stdout = stream_get_contents($stdoutFile);
$stderr = stream_get_contents($stderrFile);
fclose($stdoutFile);
fclose($stderrFile);
if ($status !== 0) { throw new RuntimeException("$label child failed: $stderr"); }
$data = json_decode($stdout, true);
if (!is_array($data) || !isset($data['raw_results']) || !is_array($data['raw_results'])) { throw new RuntimeException("Invalid $label child output"); }
return $data['raw_results'];
}

try {
$base = getenv('BENCH_BASE_SRC');
$candidate = getenv('BENCH_CANDIDATE_SRC');
$baseRef = getenv('BENCH_BASE_REF');
$candidateRef = getenv('BENCH_CANDIDATE_REF');
if (!is_string($base) || !is_dir($base) || !is_string($candidate) || !is_dir($candidate)
|| $baseRef !== '3665dcfd6afb30b69ac5afb0ed8b4ce147df6325'
|| !is_string($candidateRef) || !preg_match('/^[0-9a-f]{40}$/', $candidateRef)) { throw new RuntimeException('Invalid/missing refs or source trees'); }
$warmups = sameRunInt('BENCH_WARMUPS', 5, 0);
$repetitions = sameRunInt('BENCH_REPETITIONS', 30, 1);
$series = sameRunInt('BENCH_SERIES', 7, 1);
$runs = [];
for ($round = 0; $round < $series; ++$round) {
foreach (($round % 2 === 0 ? ['base', 'candidate'] : ['candidate', 'base']) as $which) {
$runs[] = ['ref' => $which === 'base' ? $baseRef : $candidateRef, 'round' => $round, 'results' => runPureChild($which === 'base' ? $base : $candidate, $which, $baseRef, $candidateRef, $warmups, $repetitions)];
}
}
echo json_encode(['php_version' => PHP_VERSION, 'base_ref' => $baseRef, 'candidate_ref' => $candidateRef, 'method' => 'one child process per source per round; alternating order; strict literal and semantic oracle; no timing gate', 'raw_runs' => $runs], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), "\n";
} catch (Throwable $e) { fwrite(STDERR, $e->getMessage() . "\n"); exit(1); }
91 changes: 91 additions & 0 deletions bench/mutations.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
<?php

declare(strict_types=1);

require dirname(__DIR__) . '/vendor/autoload.php';

use Ktav\Ktav;
use Ktav\PureBackend;

function benchmarkMedian(array $values): float
{
sort($values, SORT_NUMERIC);
$count = count($values);
$middle = intdiv($count, 2);
return $count % 2 ? (float)$values[$middle] : ((float)$values[$middle - 1] + (float)$values[$middle]) / 2;
}

function boundedBenchmarkInt($value, string $label, int $default, int $minimum): int
{
if ($value === false || $value === '') { return $default; }
if (!is_string($value) || !preg_match('/^[0-9]{1,5}$/', $value)) {
fwrite(STDERR, "$label must be an integer between $minimum and 10000\n"); exit(2);
}
$number = (int)$value;
if ($number < $minimum || $number > 10000) {
fwrite(STDERR, "$label must be an integer between $minimum and 10000\n"); exit(2);
}
return $number;
}

function checkResult(string $backend, int $fixture, string $actual, string $literal, array $semantic): void
{
if ($actual !== $literal) { throw new RuntimeException("$backend fixture $fixture literal bytes differ"); }
if ($backend === 'native') { $parsed = Ktav::loads($actual); }
else { $parsed = PureBackend::loads($actual); }
if ($parsed !== $semantic) {
throw new RuntimeException(sprintf(
"%s fixture %d semantic oracle differs: actual=%s (%s), expected=%s (%s)",
$backend,
$fixture,
var_export($parsed, true),
gettype($parsed),
var_export($semantic, true),
gettype($semantic)
));
}
}

$warmups = boundedBenchmarkInt(getenv('BENCH_WARMUPS'), 'warmups', 5, 0);
$repetitions = boundedBenchmarkInt(getenv('BENCH_REPETITIONS'), 'repetitions', 30, 1);
$series = boundedBenchmarkInt(getenv('BENCH_SERIES'), 'series', 7, 1);
if (!extension_loaded('ffi')) { fwrite(STDERR, "FFI extension required\n"); exit(2); }

$cases = [
['insert', "cfg: {old: +01}\n", ['cfg', 'new'], 2, "cfg: {old: +01, new: 2}\n", ['cfg' => ['old' => '+01', 'new' => 2]]],
['remove', "obj: {a: 1}\r\n", ['obj', 'a'], null, "obj: {}\r\n", ['obj' => []]],
['rename', "cfg: {\"a.b\": 1, c: 2}\n", ['cfg', 'a.b'], 'd', "cfg: {d: 1, c: 2}\n", ['cfg' => ['d' => 1, 'c' => 2]]],
['insert', "\xEF\xBB\xBFfirst: 1\r\ncfg: [{a: 1,}]\nlast: 2\r", ['cfg', '0', 'b'], 2, "\xEF\xBB\xBFfirst: 1\r\ncfg: [{a: 1, b: 2,}]\nlast: 2\r", ['first' => 1, 'cfg' => [['a' => 1, 'b' => 2]], 'last' => 2]],
];
$operations = [
'insert' => static function ($backend, $src, $path, $value) { return $backend::insertValueSegments($src, $path, $value); },
'remove' => static function ($backend, $src, $path) { return $backend::removeValueSegments($src, $path); },
'rename' => static function ($backend, $src, $path, $value) { return $backend::renameKeySegments($src, $path, $value); },
];
$backends = ['native' => Ktav::class, 'pure' => PureBackend::class];
$order = [];
for ($s = 0; $s < $series; ++$s) { $order[] = $s % 2 ? ['pure', 'native'] : ['native', 'pure']; }
$raw = [];
foreach ($cases as $index => $case) {
[$op, $source, $path, $value, $literal, $semantic] = $case;
$samplesByBackend = ['native' => [], 'pure' => []];
foreach ($order as $seriesIndex => $backendOrder) {
foreach ($backendOrder as $name) {
$backend = $backends[$name];
$invoke = static function () use ($operations, $op, $backend, $source, $path, $value) { return $operations[$op]($backend, $source, $path, $value); };
checkResult($name, $index, $invoke(), $literal, $semantic);
for ($i = 0; $i < $warmups; ++$i) { $invoke(); }
$start = hrtime(true);
$lastOutput = null;
for ($j = 0; $j < $repetitions; ++$j) { $lastOutput = $invoke(); }
$samplesByBackend[$name][] = hrtime(true) - $start;
// Validate the actual final output from this timed batch, outside timing.
checkResult($name, $index, $lastOutput, $literal, $semantic);
}
}
foreach ($samplesByBackend as $name => $samples) {
$means = array_map(static function ($n) use ($repetitions) { return $n / $repetitions; }, $samples);
$raw[] = ['backend' => $name, 'operation' => $case[0], 'fixture' => $index, 'raw_total_mutation_ns' => $samples, 'raw_mean_mutation_ns' => $means, 'median_mutation_ns' => benchmarkMedian($means), 'warmups' => $warmups, 'repetitions_per_sample' => $repetitions, 'series' => $series];
}
}
echo json_encode(['php_version' => PHP_VERSION, 'os' => PHP_OS . ' ' . php_uname('r'), 'method' => 'mutation-only timed regions; alternating backend order by series; literal bytes and independent semantic oracle checked before and after measurement; descriptive only, no speedup claim', 'raw_results' => $raw], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), "\n";
3 changes: 2 additions & 1 deletion src/Pure/Edit/EditSplice.php
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,8 @@ private static function insertSplice(string $src, $original, array $parentSegmen
&& (strpos($c['v'], "\n") !== false || strpos($c['v'], "\r") !== false
|| PureText::stringNeedsMultiline($c['v']))
)) && self::blockEncodable($c);
if (PureScanner::trimBounds($inner, 0, strlen($inner))[0] === PureScanner::trimBounds($inner, 0, strlen($inner))[1]) {
list($innerStart, $innerEnd) = PureScanner::trimBounds($inner, 0, strlen($inner));
if ($innerStart === $innerEnd) {
$spliceFrom = $from + 1;
$spliceTo = $to - 1;
$insert = $pair;
Expand Down
11 changes: 11 additions & 0 deletions tests/Pure/InsertMutationSpec.php
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,17 @@
->toBe("cfg: {a: 1, b: 2, }\n");
});

it('preserves inline empty, whitespace, trailing-comma, and semantic exact-byte fixtures', function () use ($insert) {
expect($insert("a: {}\n", 'a.k', 'v'))->toBe("a: {k: v}\n");
expect($insert("a: { }\n", 'a.k', 'v'))->toBe("a: {k: v}\n");
expect($insert("a: {x: 1, }\n", 'a.k', 2))->toBe("a: {x: 1, k: 2, }\n");

$source = "a: {x: +01, }\n";
$mutated = $insert($source, 'a.k', 2);
expect($mutated)->toBe("a: {x: +01, k: 2, }\n");
expect(PureBackend::loads($mutated)['a'])->toBe(['x' => '+01', 'k' => 2]);
});

it('preserves mixed EOLs and BOM while mutating an Object nested in an Array', function () use ($insert) {
$source = "\xEF\xBB\xBFfirst: 1\r\ncfg: [{a: 1,}]\nlast: 2\r";
expect($insert($source, 'cfg.0.b', 2))
Expand Down
Loading