From d2fcce2d86d45a701f93c9aee7c1e7aa1897a307 Mon Sep 17 00:00:00 2001 From: Marat K Date: Tue, 27 Oct 2026 15:29:36 +0100 Subject: [PATCH 1/3] bench: measure lossless source mutations Compare baseline and candidate on the same CI runner using identical inputs, paired samples, and independent source and semantic oracles. Record the compiler and lockfile; do not gate on noisy wall-clock timing. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/source-mutations-perf.yml | 55 +++++ Cargo.toml | 4 + benches/source_mutations.rs | 232 ++++++++++++++++++++ scripts/bench-mutations.py | 115 ++++++++++ 4 files changed, 406 insertions(+) create mode 100644 .github/workflows/source-mutations-perf.yml create mode 100644 benches/source_mutations.rs create mode 100644 scripts/bench-mutations.py diff --git a/.github/workflows/source-mutations-perf.yml b/.github/workflows/source-mutations-perf.yml new file mode 100644 index 0000000..ecd4f1f --- /dev/null +++ b/.github/workflows/source-mutations-perf.yml @@ -0,0 +1,55 @@ +name: Source mutation performance + +on: + pull_request: + paths: + - 'src/document/parser/edit/**' + - 'benches/source_mutations.rs' + - 'scripts/bench-mutations.py' + - '.github/workflows/source-mutations-perf.yml' + +permissions: + contents: read + +jobs: + compare: + name: Mutation baseline / candidate + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: baseline + persist-credentials: false + + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: candidate + persist-credentials: false + + - uses: dtolnay/rust-toolchain@stable + + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Record compared revisions + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: printf 'baseline=%s\ncandidate=%s\n' "$BASE_SHA" "$HEAD_SHA" + + - name: Compare identical mutation workloads + env: + CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS: fallback + run: python -I candidate/scripts/bench-mutations.py baseline candidate mutation-results + + - name: Preserve raw samples and environment + if: always() + uses: actions/upload-artifact@v4 + with: + name: source-mutation-performance + path: mutation-results/ + if-no-files-found: warn diff --git a/Cargo.toml b/Cargo.toml index 9d47906..7f997a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,3 +111,7 @@ harness = false [[bench]] name = "runtime_paths" harness = false + +[[bench]] +name = "source_mutations" +harness = false diff --git a/benches/source_mutations.rs b/benches/source_mutations.rs new file mode 100644 index 0000000..1c4e86b --- /dev/null +++ b/benches/source_mutations.rs @@ -0,0 +1,232 @@ +//! Ограниченный CI-прогон mutations; оракулы проверяются вне таймера. + +use std::fmt::Write; +use std::hint::black_box; +use std::time::{Duration, Instant}; + +use ktav::edit::{ + insert_value, insert_value_segments, remove_value, remove_value_segments, rename_key, + rename_key_segments, EditError, +}; +use ktav::{ObjectMap, Value}; + +const SAMPLE: Duration = Duration::from_millis(60); +const WARMUP: Duration = Duration::from_millis(5); + +#[derive(Clone, Copy)] +enum Operation { + Insert, + Remove, + Rename, +} + +struct Case { + name: String, + operation: Operation, + source: String, + path: String, + segments: Vec, + expected: String, + semantic: Value, +} + +impl Case { + fn run(&self, segments: bool) -> Result { + let source = black_box(self.source.as_str()); + let value = Value::Integer("1".into()); + match (self.operation, segments) { + (Operation::Insert, false) => insert_value(source, black_box(&self.path), &value), + (Operation::Insert, true) => { + insert_value_segments(source, black_box(&self.segments), &value) + } + (Operation::Remove, false) => remove_value(source, black_box(&self.path)), + (Operation::Remove, true) => remove_value_segments(source, black_box(&self.segments)), + (Operation::Rename, false) => rename_key(source, black_box(&self.path), "renamed"), + (Operation::Rename, true) => { + rename_key_segments(source, black_box(&self.segments), "renamed") + } + } + } + + fn validate(&self, segments: bool) -> usize { + let output = self.run(segments).expect("benchmark mutation must succeed"); + assert_eq!(output, self.expected, "{}: source bytes", self.name); + assert_eq!( + ktav::parse(&output).expect("benchmark result must parse"), + self.semantic, + "{}: semantic tree", + self.name, + ); + output.len() + } +} + +fn object(entries: impl IntoIterator) -> Value { + Value::Object(entries.into_iter().map(|(k, v)| (k.into(), v)).collect()) +} + +fn integer() -> Value { + Value::Integer("1".into()) +} + +fn flat_cases(rows: usize) -> Vec { + let mut source = String::new(); + let mut original = ObjectMap::default(); + for i in 0..rows { + let key = format!("key{i}"); + writeln!(source, "{key}: old").expect("String writes cannot fail"); + original.insert(key.into(), Value::String("old".into())); + } + let target = format!("key{}", rows / 2); + let line = format!("{target}: old\n"); + let mut inserted = original.clone(); + inserted.insert("added".into(), integer()); + let mut removed = original.clone(); + removed.shift_remove(target.as_str()); + let renamed: ObjectMap = original + .into_iter() + .map(|(key, value)| { + if key == target.as_str() { + ("renamed".into(), value) + } else { + (key, value) + } + }) + .collect(); + vec![ + Case { + name: format!("insert_flat_{rows}"), + operation: Operation::Insert, + expected: format!("{source}added: 1\n"), + source: source.clone(), + path: "added".into(), + segments: vec!["added".into()], + semantic: Value::Object(inserted), + }, + Case { + name: format!("remove_flat_{rows}"), + operation: Operation::Remove, + expected: source.replacen(&line, "", 1), + source: source.clone(), + path: target.clone(), + segments: vec![target.clone()], + semantic: Value::Object(removed), + }, + Case { + name: format!("rename_flat_{rows}"), + operation: Operation::Rename, + expected: source.replacen(&line, "renamed: old\n", 1), + source, + path: target.clone(), + segments: vec![target], + semantic: Value::Object(renamed), + }, + ] +} + +fn cases() -> Vec { + let mut cases = Vec::new(); + for rows in [4, 4096, 32768] { + cases.extend(flat_cases(rows)); + } + cases.extend([ + Case { + name: "insert_inline_unicode".into(), + operation: Operation::Insert, + source: "\u{feff} {ключ: old,}\r\n".into(), + path: "added".into(), + segments: vec!["added".into()], + expected: "\u{feff} {ключ: old, added: 1,}\r\n".into(), + semantic: object([("ключ", Value::String("old".into())), ("added", integer())]), + }, + Case { + name: "insert_array_item".into(), + operation: Operation::Insert, + source: "items: [{x: 1}]\n".into(), + path: "items.0.added".into(), + segments: vec!["items".into(), "0".into(), "added".into()], + expected: "items: [{x: 1, added: 1}]\n".into(), + semantic: object([( + "items", + Value::Array(vec![object([("x", integer()), ("added", integer())])]), + )]), + }, + Case { + name: "remove_dotted_materialize".into(), + operation: Operation::Remove, + source: "a.b.c: 1\r\nkeep: 1\r\n".into(), + path: "a.b.c".into(), + segments: vec!["a".into(), "b".into(), "c".into()], + expected: "a.b: {}\r\nkeep: 1\r\n".into(), + semantic: object([("a", object([("b", object([]))])), ("keep", integer())]), + }, + Case { + name: "remove_array_element".into(), + operation: Operation::Remove, + source: "items: [1, 2, 3]\n".into(), + path: "items.1".into(), + segments: vec!["items".into(), "1".into()], + expected: "items: [1, 3]\n".into(), + semantic: object([( + "items", + Value::Array(vec![integer(), Value::Integer("3".into())]), + )]), + }, + Case { + name: "rename_escaped_key".into(), + operation: Operation::Rename, + source: "\"a.b\": 1\nkeep: 1\n".into(), + path: "a\\.b".into(), + segments: vec!["a.b".into()], + expected: "renamed: 1\nkeep: 1\n".into(), + semantic: object([("renamed", integer()), ("keep", integer())]), + }, + Case { + name: "rename_noop".into(), + operation: Operation::Rename, + source: "renamed:: 0007\r\n".into(), + path: "renamed".into(), + segments: vec!["renamed".into()], + expected: "renamed:: 0007\r\n".into(), + semantic: object([("renamed", Value::String("0007".into()))]), + }, + ]); + cases +} + +fn main() { + let label = std::env::args() + .nth(1) + .expect("baseline or candidate label"); + assert!(matches!(label.as_str(), "baseline" | "candidate")); + println!("label,case,path,iterations,elapsed_ns,ns_per_op,output_len"); + for case in cases() { + for segments in [false, true] { + let len = case.validate(segments); + let warmup = Instant::now(); + while warmup.elapsed() < WARMUP { + drop(black_box( + case.run(segments).expect("validated benchmark case"), + )); + } + let start = Instant::now(); + let mut iterations = 0u64; + loop { + drop(black_box( + case.run(segments).expect("validated benchmark case"), + )); + iterations += 1; + if start.elapsed() >= SAMPLE { + break; + } + } + let elapsed = start.elapsed().as_nanos(); + let path = if segments { "segments" } else { "text" }; + println!( + "{label},{},{path},{iterations},{elapsed},{:.1},{len}", + case.name, + elapsed as f64 / iterations as f64, + ); + } + } +} diff --git a/scripts/bench-mutations.py b/scripts/bench-mutations.py new file mode 100644 index 0000000..7e8ac16 --- /dev/null +++ b/scripts/bench-mutations.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Compare mutation benchmarks on one runner; timings are not a CI gate.""" + +import csv +import io +import json +import os +from pathlib import Path +import shutil +import statistics +import subprocess +import sys + + +BENCH = "source_mutations" +FIELDS = ["label", "case", "path", "iterations", "elapsed_ns", "ns_per_op", "output_len"] + + +def command(args, cwd, timeout=900): + result = subprocess.run( + args, cwd=cwd, check=True, text=True, stdout=subprocess.PIPE, timeout=timeout + ) + return result.stdout + + +def executable(root): + output = command( + ["cargo", "bench", "--locked", "--bench", BENCH, "--no-run", "--message-format=json"], + root, + ) + matches = [] + for line in output.splitlines(): + entry = json.loads(line) + if ( + entry.get("reason") == "compiler-artifact" + and entry.get("target", {}).get("name") == BENCH + and entry.get("executable") + ): + matches.append(entry["executable"]) + if len(matches) != 1: + raise RuntimeError(f"expected one {BENCH} executable, got {matches!r}") + return matches[0] + + +def main(): + baseline, candidate, output = (Path(arg).resolve() for arg in sys.argv[1:]) + output.mkdir(parents=True, exist_ok=False) + shutil.copyfile(candidate / "benches" / f"{BENCH}.rs", baseline / "benches" / f"{BENCH}.rs") + manifest = baseline / "Cargo.toml" + text = manifest.read_text() + if f'name = "{BENCH}"' not in text: + manifest.write_text(text + f'\n[[bench]]\nname = "{BENCH}"\nharness = false\n') + command(["cargo", "generate-lockfile"], candidate) + shutil.copyfile(candidate / "Cargo.lock", baseline / "Cargo.lock") + shutil.copyfile(candidate / "Cargo.lock", output / "Cargo.lock") + versions = command(["rustc", "--version", "--verbose"], candidate) + versions += command(["cargo", "--version"], candidate) + versions += f"RUSTFLAGS={os.environ.get('RUSTFLAGS', '')}\n" + versions += "features=default; profile=bench; execution=serial\n" + (output / "environment.txt").write_text(versions) + print(versions, flush=True) + roots = {"baseline": baseline, "candidate": candidate} + binaries = {label: executable(root) for label, root in roots.items()} + samples = {label: {} for label in roots} + expected_keys = None + lengths = {} + for repetition in range(3): + order = ["baseline", "candidate"] if repetition % 2 == 0 else ["candidate", "baseline"] + for label in order: + raw = command([binaries[label], label], roots[label], timeout=30) + (output / f"{label}-{repetition}.csv").write_text(raw) + reader = csv.DictReader(io.StringIO(raw)) + if reader.fieldnames != FIELDS: + raise RuntimeError(f"unexpected CSV header: {reader.fieldnames!r}") + keys = set() + for row in reader: + key = (row["case"], row["path"]) + if row["label"] != label or key in keys: + raise RuntimeError(f"bad or duplicate benchmark row: {row!r}") + keys.add(key) + timing = float(row["ns_per_op"]) + if timing <= 0 or int(row["iterations"]) <= 0: + raise RuntimeError(f"invalid timing: {row!r}") + length = int(row["output_len"]) + if lengths.setdefault(key, length) != length: + raise RuntimeError(f"output length changed for {key!r}") + samples[label].setdefault(key, []).append(timing) + if not keys or (expected_keys is not None and keys != expected_keys): + raise RuntimeError("benchmark case sets differ") + expected_keys = keys + report = [ + "## Source mutation benchmark", + "", + "Same runner/compiler/lockfile; median of three alternating runs. Positive change means faster.", + "Shared-runner wall-clock results are informational, not a pass/fail gate.", + "Exact bytes and independent semantic trees are asserted outside each timed sample.", + "", + "| Case | Path | Baseline ns/op | Candidate ns/op | Change |", + "| --- | --- | ---: | ---: | ---: |", + ] + for key in sorted(expected_keys): + before = statistics.median(samples["baseline"][key]) + after = statistics.median(samples["candidate"][key]) + change = (before - after) / before * 100 + report.append(f"| {key[0]} | {key[1]} | {before:.1f} | {after:.1f} | {change:+.1f}% |") + summary = "\n".join(report) + "\n" + (output / "summary.md").write_text(summary) + print(summary) + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as destination: + destination.write(summary) + + +if __name__ == "__main__": + main() From d25f2d56c206ec1f284ccc34d35110870a320e63 Mon Sep 17 00:00:00 2001 From: Marat K Date: Tue, 27 Oct 2026 15:35:37 +0100 Subject: [PATCH 2/3] perf(edit): move values when renaming keys Rebuild the expected ordered Object by moving entries rather than cloning entire sibling values. Preserve the semantic verification and ordering contract; measure the change with a large-sibling workload. Co-Authored-By: Claude Sonnet 5.5 --- benches/source_mutations.rs | 13 ++++++++++++ src/document/parser/edit/splice/rename.rs | 14 +++++------- tests/suite/api/text/edit_mutate.rs | 26 +++++++++++++++++++++++ 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/benches/source_mutations.rs b/benches/source_mutations.rs index 1c4e86b..c7e0643 100644 --- a/benches/source_mutations.rs +++ b/benches/source_mutations.rs @@ -129,6 +129,19 @@ fn cases() -> Vec { for rows in [4, 4096, 32768] { cases.extend(flat_cases(rows)); } + let payload = "x".repeat(128 * 1024); + cases.push(Case { + name: "rename_large_sibling".into(), + operation: Operation::Rename, + source: format!("payload: {payload}\ntarget: 1\n"), + path: "target".into(), + segments: vec!["target".into()], + expected: format!("payload: {payload}\nrenamed: 1\n"), + semantic: object([ + ("payload", Value::String(payload.into())), + ("renamed", integer()), + ]), + }); cases.extend([ Case { name: "insert_inline_unicode".into(), diff --git a/src/document/parser/edit/splice/rename.rs b/src/document/parser/edit/splice/rename.rs index 23b04bd..aa2630c 100644 --- a/src/document/parser/edit/splice/rename.rs +++ b/src/document/parser/edit/splice/rename.rs @@ -113,18 +113,14 @@ pub(super) fn apply>( let parent = value_at(&mut expected, parent_segs)?; match parent { Value::Object(o) => { - let renamed: Vec<(crate::value::Scalar, Value)> = o - .iter() + let entries = std::mem::take(o); + *o = entries + .into_iter() .map(|(k, v)| { - let k = if k.as_str() == key { - new_key.into() - } else { - k.clone() - }; - (k, v.clone()) + let k = if k.as_str() == key { new_key.into() } else { k }; + (k, v) }) .collect(); - *o = renamed.into_iter().collect(); } _ => return Err(unsupported("internal: parent value is not an object")), } diff --git a/tests/suite/api/text/edit_mutate.rs b/tests/suite/api/text/edit_mutate.rs index b852039..d7af462 100644 --- a/tests/suite/api/text/edit_mutate.rs +++ b/tests/suite/api/text/edit_mutate.rs @@ -34,6 +34,32 @@ fn keys_of(v: &Value) -> Vec<&str> { } } +#[test] +fn rename_preserves_order_and_nested_sibling_values() { + let source = "before: {nested: [1, 2]}\ntarget: 3\nafter: {v: keep}\n"; + let expected = "before: {nested: [1, 2]}\nrenamed: 3\nafter: {v: keep}\n"; + let tree = obj(&[ + ( + "before", + obj(&[( + "nested", + Value::Array(vec![Value::Integer("1".into()), Value::Integer("2".into())]), + )]), + ), + ("renamed", Value::Integer("3".into())), + ("after", obj(&[("v", s("keep"))])), + ]); + for output in [ + rename_key(source, "target", "renamed").unwrap(), + rename_key_segments(source, &["target"], "renamed").unwrap(), + ] { + assert_eq!(output, expected); + let parsed = parse(&output).unwrap(); + assert_eq!(parsed, tree); + assert_eq!(keys_of(&parsed), ["before", "renamed", "after"]); + } +} + // --- inline root после ведущих trivia ---------------------------------------- #[test] From 1f7c717597cceecb8da84aaf8f47e62945b1db98 Mon Sep 17 00:00:00 2001 From: Marat K Date: Tue, 27 Oct 2026 16:34:37 +0100 Subject: [PATCH 3/3] fix(edit): preserve verbatim data during reindent Keep multiline String data intact when reindenting inserted or replacement compounds, while adjusting the structural closing marker. Cover LF and CRLF with exact and semantic assertions. Co-Authored-By: Claude Sonnet 5.5 --- src/document/parser/edit/splice.rs | 9 ++++--- tests/suite/api/text/edit_mutate.rs | 26 ++++++++++++++++++ tests/suite/api/text/edit_value.rs | 41 ++++++++++++++++++++++++++++- 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/src/document/parser/edit/splice.rs b/src/document/parser/edit/splice.rs index 15f1e59..83299f4 100644 --- a/src/document/parser/edit/splice.rs +++ b/src/document/parser/edit/splice.rs @@ -864,17 +864,20 @@ fn reindent(doc: &str, indent: &str, eol: &str) -> String { let mut out = String::with_capacity(doc.len()); let mut in_verbatim = false; for (i, line) in doc.lines().enumerate() { - let line = line.strip_prefix(" ").unwrap_or(line); let verbatim_open = line.ends_with("(("); let verbatim_close = line.trim_matches(|c: char| c == ' ' || c == '\t') == "))"; if i > 0 { out.push_str(eol); - let skip_prefix = in_verbatim || line.is_empty(); + let skip_prefix = (in_verbatim && !verbatim_close) || line.is_empty(); if !skip_prefix { out.push_str(indent); } } - out.push_str(line); + if !in_verbatim || verbatim_close { + out.push_str(line.strip_prefix(" ").unwrap_or(line)); + } else { + out.push_str(line); + } if verbatim_open { in_verbatim = true; } else if verbatim_close { diff --git a/tests/suite/api/text/edit_mutate.rs b/tests/suite/api/text/edit_mutate.rs index d7af462..f04ed64 100644 --- a/tests/suite/api/text/edit_mutate.rs +++ b/tests/suite/api/text/edit_mutate.rs @@ -450,6 +450,32 @@ fn insert_inline_numeric_looking_string_gets_provenance() { ); } +#[test] +fn insert_nested_object_multiline_string_lf_exact_and_semantic() { + let source = "a: {\n x: 1\n}\n"; + let value = obj(&[("text", s(" four\n six"))]); + let output = insert_value(source, "a.value", &value).unwrap(); + assert_eq!(output, "a: {\n x: 1\n value: {\n text: ((\n four\n six\n ))\n }\n}\n"); + let expected = obj(&[( + "a", + obj(&[("x", Value::Integer("1".into())), ("value", value)]), + )]); + assert_eq!(parse(&output).unwrap(), expected); +} + +#[test] +fn insert_nested_object_multiline_string_crlf_exact_and_semantic() { + let source = "a: {\r\n x: 1\r\n}\r\n"; + let value = obj(&[("text", s(" four\n six"))]); + let output = insert_value(source, "a.value", &value).unwrap(); + assert_eq!(output, "a: {\r\n x: 1\r\n value: {\r\n text: ((\r\n four\r\n six\r\n ))\r\n }\r\n}\r\n"); + let expected = obj(&[( + "a", + obj(&[("x", Value::Integer("1".into())), ("value", value)]), + )]); + assert_eq!(parse(&output).unwrap(), expected); +} + #[test] fn insert_multiline_string_expands_inline_parent_to_block() { let t = "a: {x: 1}\n"; diff --git a/tests/suite/api/text/edit_value.rs b/tests/suite/api/text/edit_value.rs index 1645a7e..cfe88e1 100644 --- a/tests/suite/api/text/edit_value.rs +++ b/tests/suite/api/text/edit_value.rs @@ -7,7 +7,7 @@ mod layout; use ktav::edit::{edit_value, edit_value_segments, EditError}; use ktav::parse; -use ktav::value::Value; +use ktav::value::{ObjectMap, Value}; fn s(v: &str) -> Value { Value::String(v.into()) @@ -15,6 +15,45 @@ fn s(v: &str) -> Value { // --- скаляры, все типы ------------------------------------------------------ +#[test] +fn nested_object_multiline_string_replacement_lf_exact_and_semantic() { + let source = "a: {\n value: old\n}\n"; + let value = Value::Object({ + let mut map = ObjectMap::default(); + map.insert("text".into(), s(" four\n six")); + map + }); + let output = edit_value(source, "a.value", &value).unwrap(); + assert_eq!( + output, + "a: {\n value: {\n text: ((\n four\n six\n ))\n }\n}\n" + ); + let mut nested = ObjectMap::default(); + nested.insert("text".into(), s(" four\n six")); + let mut a = ObjectMap::default(); + a.insert("value".into(), Value::Object(nested)); + let mut expected = ObjectMap::default(); + expected.insert("a".into(), Value::Object(a)); + assert_eq!(parse(&output).unwrap(), Value::Object(expected)); +} + +#[test] +fn nested_object_multiline_string_replacement_crlf_exact_and_semantic() { + let source = "a: {\r\n value: old\r\n}\r\n"; + let mut value_map = ObjectMap::default(); + value_map.insert("text".into(), s(" four\n six")); + let value = Value::Object(value_map); + let output = edit_value(source, "a.value", &value).unwrap(); + assert_eq!(output, "a: {\r\n value: {\r\n text: ((\r\n four\r\n six\r\n ))\r\n }\r\n}\r\n"); + let mut nested = ObjectMap::default(); + nested.insert("text".into(), s(" four\n six")); + let mut a = ObjectMap::default(); + a.insert("value".into(), Value::Object(nested)); + let mut expected = ObjectMap::default(); + expected.insert("a".into(), Value::Object(a)); + assert_eq!(parse(&output).unwrap(), Value::Object(expected)); +} + #[test] fn string_replacement_plain() { let t = "port: 8080\nhost: db\n";