From 42ba13a72ef7ce360de018cb770a0fc282495d23 Mon Sep 17 00:00:00 2001 From: Marat K Date: Sat, 24 Oct 2026 20:56:35 +0200 Subject: [PATCH 1/2] =?UTF-8?q?feat(error)!:=20=D1=81=D0=BE=D1=85=D1=80?= =?UTF-8?q?=D0=B0=D0=BD=D1=8F=D1=82=D1=8C=20=D0=BF=D0=BE=D0=BB=D0=BD=D1=8B?= =?UTF-8?q?=D0=B5=20=D0=BA=D0=BE=D0=BE=D1=80=D0=B4=D0=B8=D0=BD=D0=B0=D1=82?= =?UTF-8?q?=D1=8B=20=D0=BE=D1=88=D0=B8=D0=B1=D0=BE=D0=BA=20=D0=B2=200.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Добавить line/span в InvalidUtf8 и строку opener в UnclosedCompound. Использовать общий расчёт исходных byte offsets в Rust API, C ABI, Span::line_col и line_text публичного envelope для LF/CR/CRLF. Сохранить UTF-8 priority, I/O и writer contracts и JSON schema. Закрепить draft spec corpus 0.9 с 72 новыми diagnostic fixtures. Проверять oracles через source entry points, envelope и C ABI; переключить corpus runners на 0.9 и запрещать missing-corpus skips в CI. Описать breaking migration на EN/RU/ZH. Path-editing API остаётся частью готовящейся 0.9.0. Публикация релиза не выполняется. BREAKING CHANGE: InvalidUtf8 теперь содержит valid_up_to, line и span; UnclosedCompound — kind, line и span. Downstream-конструкторы и полные field patterns требуют миграции. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 + CHANGELOG.md | 4 + CHANGELOG.ru.md | 4 + CHANGELOG.zh.md | 4 + Cargo.toml | 4 +- README.md | 31 +++- README.ru.md | 33 +++- README.zh.md | 26 +++- SECURITY.md | 2 +- SECURITY.ru.md | 2 +- SECURITY.zh.md | 2 +- root-docs/CHANGELOG/unreleased/body-1.md | 12 ++ root-docs/README/error-migration/body-1.md | 44 ++++++ root-docs/README/error-migration/meta.js | 7 + root-docs/README/installation/body-1.md | 15 +- root-docs/README/intro/body-1.md | 34 ++-- root-docs/README/manifest.js | 1 + spec | 2 +- src/cabi/envelope.rs | 93 ++--------- src/document/parser/edit/tree.rs | 3 + src/document/parser/fmt_parser/mod.rs | 4 +- src/document/parser/fmt_parser/positioned.rs | 5 +- src/document/parser/parser/mod.rs | 5 +- src/document/parser/parser/parse_str.rs | 4 +- src/document/parser/tests/inline_values.rs | 2 +- src/error/envelope.rs | 18 +-- src/error/error.rs | 49 +++--- src/error/error/display.rs | 2 +- src/error/error/traits.rs | 27 ++-- src/error/mod.rs | 1 + src/error/source.rs | 54 +++++++ src/lib.rs | 5 +- .../thin/event_parser/counter_tests.rs | 4 +- src/serde_bridge/thin/event_parser/lines.rs | 6 +- .../format/format_canonical_equivalence.rs | 2 +- .../docs/format/format_comments_preserved.rs | 2 +- .../format/format_explicit_root_unclosed.rs | 2 +- .../api/text/docs/format/format_idempotent.rs | 2 +- .../api/thin/scan/top_level_inline_thin.rs | 2 +- tests/suite/api/value/cabi_corpus.rs | 2 +- tests/suite/errors/envelope/spans.rs | 13 +- tests/suite/errors/surface/error_accessors.rs | 10 +- tests/suite/errors/surface/invalid_utf8.rs | 8 +- .../suite/errors/surface/structured_errors.rs | 9 +- tests/suite/spec.rs | 2 + tests/suite/spec/diagnostics.rs | 146 ++++++++++++++++++ tests/suite/spec/support.rs | 63 ++++++-- 47 files changed, 532 insertions(+), 244 deletions(-) create mode 100644 root-docs/README/error-migration/body-1.md create mode 100644 root-docs/README/error-migration/meta.js create mode 100644 src/error/source.rs create mode 100644 tests/suite/spec/diagnostics.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12b59db..d711cd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,6 +75,8 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: cargo test + env: + KTAV_REQUIRE_SPEC: '1' run: cargo test --all-features --no-fail-fast msrv: @@ -113,4 +115,6 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: cargo test + env: + KTAV_REQUIRE_SPEC: '1' run: cargo test --locked --all-features --no-fail-fast diff --git a/CHANGELOG.md b/CHANGELOG.md index 5836c3d..78cc08d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ For the format specification's own history, see the ## [Unreleased] +### Changed + +- **Breaking in 0.9.0:** `InvalidUtf8` gains `line`/`span`; `UnclosedCompound` gains the opener's `line`. Downstream constructors and exhaustive field patterns must migrate. Direct Rust errors, public envelopes and C ABI source operations share raw-byte locations; UTF-8 spans are non-empty and EOF spans cover the innermost opener through EOF. LF, lone CR, CRLF and BOM offsets are preserved, including `Span::line_col` and envelope `line_text`. The ten-field JSON schema and I/O/writer location rules are unchanged. New regression oracles live only in the draft spec `versions/0.9/tests` and run across source entry points; CI refuses to skip a missing corpus. Version 0.9.0 is in preparation, not published. + ### Added - Rust-only source-editing API: `ktav::edit::edit_value` accepts an escaped dotted path, and `edit_value_segments` accepts a slice/vector of literal segments. Both replace an existing value without reformatting the document, preserve surrounding bytes and comments, and return a new `String`. diff --git a/CHANGELOG.ru.md b/CHANGELOG.ru.md index 173b530..30841dd 100644 --- a/CHANGELOG.ru.md +++ b/CHANGELOG.ru.md @@ -12,6 +12,10 @@ Cargo-конвенцией: до 1.0 bump MINOR считается ломающ ## [Не выпущено] +### Изменено + +- **Breaking в 0.9.0:** в `InvalidUtf8` добавлены `line`/`span`, в `UnclosedCompound` — `line` открывающего маркера. Downstream-конструкторы и patterns, перечисляющие все поля, требуют миграции. Прямые Rust-ошибки, публичный envelope и C ABI source operations используют общие координаты исходных байтов; UTF-8 spans непустые, EOF spans охватывают самый внутренний незакрытый opener до EOF. LF, одиночный CR, CRLF и BOM учитываются без нормализации, в том числе в `Span::line_col` и envelope `line_text`. Схема JSON из десяти полей и отсутствие позиций у I/O/writer не меняются. Новые regression oracles находятся только в draft спеки `versions/0.9/tests` и проверяются через source entry points; CI запрещает пропуск отсутствующего corpus. Версия 0.9.0 готовится, но не опубликована. + ### Добавлено - Rust API редактирования исходного текста: `ktav::edit::edit_value` принимает экранированный dotted-путь, `edit_value_segments` — слайс/вектор буквальных сегментов. Обе функции заменяют существующее значение без форматирования документа, сохраняют окружающие байты и комментарии и возвращают новый `String`. diff --git a/CHANGELOG.zh.md b/CHANGELOG.zh.md index e6951ce..211edb7 100644 --- a/CHANGELOG.zh.md +++ b/CHANGELOG.zh.md @@ -12,6 +12,10 @@ ## [尚未发布] +### 变更 + +- **0.9.0 破坏性变更:** `InvalidUtf8` 新增 `line`/`span`,`UnclosedCompound` 新增起始标记的 `line`。下游构造器及枚举全部字段的模式需要迁移。直接 Rust 错误、公开 envelope 和 C ABI 源操作共用原始字节坐标;UTF-8 区间非空,EOF 区间覆盖最内层未闭合起始标记至 EOF。LF、单独 CR、CRLF 及 BOM 均不经规范化,`Span::line_col` 和 envelope 的 `line_text` 同样遵守该规则。十字段 JSON 格式及 I/O/writer 无源位置的规则不变。新增回归 oracle 仅位于规范草案 `versions/0.9/tests`,由各源入口执行;CI 不允许因语料缺失而跳过。0.9.0 尚在准备中,未发布。 + ### 新增 - Rust 源文本编辑 API:`ktav::edit::edit_value` 接受带转义的点分路径,`edit_value_segments` 接受字面路径段切片/向量。两者均在不重新格式化文档的情况下替换已有值,保留周围字节与注释,并返回新的 `String`。 diff --git a/Cargo.toml b/Cargo.toml index f83b091..97d17de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ktav" -version = "0.8.1" +version = "0.9.0" edition = "2021" description = "Ktav — a plain configuration format. Three rules, zero indentation, zero quoting. Serde-native." license = "MIT OR Apache-2.0" @@ -38,7 +38,7 @@ exclude = [ # Which version of the Ktav format specification this crate implements. # See https://github.com/ktav-lang/spec for the formal document. [package.metadata.ktav] -spec-version = "0.8.1" +spec-version = "0.9.0" [dependencies] serde = { version = "1", features = ["derive"] } diff --git a/README.md b/README.md index ca174df..35e44aa 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,11 @@ **Playground:** convert JSON / YAML / TOML / INI ⇄ Ktav in your browser at **[ktav-lang.github.io](https://ktav-lang.github.io/)**. -**Specification:** this crate implements **Ktav 0.8.0**, the version -named by `[package.metadata.ktav] spec-version` in `Cargo.toml`. The -format is versioned and maintained independently of this crate — the two -numbers move apart on purpose, since a crate release that changes no -format behaviour leaves `spec-version` where it was. See -[`ktav-lang/spec`](https://github.com/ktav-lang/spec) for the formal -document, and [`CHANGELOG.md`](CHANGELOG.md) for this crate's history. +**Version in preparation: 0.9.0, not published.** This branch combines +lossless source editing by path and the new source-error location model. +`Cargo.toml` targets the draft spec corpus `versions/0.9/tests`; published +0.8.1 remains unchanged. See [`ktav-lang/spec`](https://github.com/ktav-lang/spec) +for the specification and [`CHANGELOG.md`](CHANGELOG.md) for the migration. --- @@ -503,6 +501,20 @@ the target numeric type; a value that arrived as a string (e.g. forced with `::`) is still accepted via `FromStr`. `NaN` / `±Infinity` are rejected by the serializer (Ktav does not represent them). +## Error model migration (0.9) + +**0.9 migration (breaking):** `Error::InvalidUtf8` now contains +`{ valid_up_to, line, span }`; `ErrorKind::UnclosedCompound` contains +`{ kind, line, span }`. Update downstream constructors with the new fields; +use `..` in patterns when only selected fields matter. `line()` now returns +the source line for both errors. UTF-8 spans cover original malformed bytes; +EOF diagnostics cover the innermost opener through EOF and report its line. +Coordinates preserve BOM and LF/CR/CRLF; `Span::line_col` and envelope +`line_text` recognize all three terminators. I/O and writer failures retain +absent source positions. The ten-field JSON schema below is unchanged. +The new location oracles live in the draft `versions/0.9/tests` spec corpus; +this working version has not been published. + ## Examples: Ktav → JSON5 JSON5 is on the right because it reads like ordinary JavaScript, allows @@ -961,9 +973,12 @@ their parent module. ## Installation +The published dependency below is 0.8.1; it does not include the new path +editing or 0.9 error model. Version 0.9.0 currently requires this source checkout. + ```toml [dependencies] -ktav = "0.8.0" +ktav = "0.8.1" serde = { version = "1", features = ["derive"] } ``` diff --git a/README.ru.md b/README.ru.md index 9f7d247..e672086 100644 --- a/README.ru.md +++ b/README.ru.md @@ -14,13 +14,12 @@ **Песочница:** конвертация JSON / YAML / TOML / INI ⇄ Ktav прямо в браузере — **[ktav-lang.github.io](https://ktav-lang.github.io/)**. -**Спецификация:** этот crate реализует **Ktav 0.8.0** — версию, -указанную в `[package.metadata.ktav] spec-version` в `Cargo.toml`. -Формат версионируется и поддерживается независимо от crate-а, и два -номера намеренно расходятся: выпуск crate-а, не меняющий поведение -формата, оставляет `spec-version` на месте. См. -[`ktav-lang/spec`](https://github.com/ktav-lang/spec) для канонического -документа и [`CHANGELOG.ru.md`](CHANGELOG.ru.md) — историю crate-а. +**Готовится версия 0.9.0, ещё не опубликована.** Эта ветка объединяет +lossless-редактирование исходного текста по пути и новую модель координат +ошибок. `Cargo.toml` указывает на draft corpus спеки `versions/0.9/tests`; +опубликованная 0.8.1 не меняется. Спецификация — +[`ktav-lang/spec`](https://github.com/ktav-lang/spec), миграция — +[`CHANGELOG.ru.md`](CHANGELOG.ru.md). --- @@ -508,6 +507,21 @@ assert_eq!(keys, ["port", "host"]); через `FromStr`. `NaN` / `±Infinity` отвергаются сериализатором (Ktav их не представляет). +## Миграция модели ошибок (0.9) + +**Миграция 0.9 (breaking):** `Error::InvalidUtf8` теперь содержит +`{ valid_up_to, line, span }`, `ErrorKind::UnclosedCompound` — +`{ kind, line, span }`. В downstream-конструкторах нужно заполнить новые +поля; в patterns используйте `..`, если важна только часть полей. +`line()` теперь возвращает исходную строку обеих ошибок. UTF-8 span +покрывает исходные ошибочные байты; EOF-диагностика охватывает самый +внутренний незакрытый opener до EOF и указывает его строку. Координаты +сохраняют BOM и LF/CR/CRLF; `Span::line_col` и envelope `line_text` +учитывают все три терминатора. Ошибки I/O и writer по-прежнему не имеют +позиции исходника. Схема JSON из десяти полей ниже не меняется. Новые +oracles координат находятся в draft corpus спеки `versions/0.9/tests`; +эта рабочая версия ещё не опубликована. + ## Примеры: Ktav → JSON5 JSON5 справа потому, что читается как обычный JavaScript, допускает @@ -971,9 +985,12 @@ ktav/ ## Установка +Ниже указана опубликованная 0.8.1: в ней нет новых методов редактирования +по пути и модели ошибок 0.9. Для готовящейся 0.9.0 пока нужен этот source checkout. + ```toml [dependencies] -ktav = "0.8.0" +ktav = "0.8.1" serde = { version = "1", features = ["derive"] } ``` diff --git a/README.zh.md b/README.zh.md index b11ae61..24ca8aa 100644 --- a/README.zh.md +++ b/README.zh.md @@ -13,11 +13,10 @@ **演练场:** 在浏览器中互转 JSON / YAML / TOML / INI ⇄ Ktav — **[ktav-lang.github.io](https://ktav-lang.github.io/)**。 -**规范:** 本 crate 实现 **Ktav 0.8.0**,即 `Cargo.toml` 中 -`[package.metadata.ktav] spec-version` 所指定的版本。格式与 crate 彼此 -独立地版本化与维护,两个号码会有意分开——不改变格式行为的 crate 发布会 -让 `spec-version` 保持原样。规范正文见 -[`ktav-lang/spec`](https://github.com/ktav-lang/spec),crate 的历史见 +**0.9.0 正在准备中,尚未发布。** 本分支同时包含按路径无损编辑源文本 +及新的源错误坐标模型。`Cargo.toml` 指向规范草案语料 `versions/0.9/tests`; +已发布的 0.8.1 保持不变。规范见 +[`ktav-lang/spec`](https://github.com/ktav-lang/spec),迁移说明见 [`CHANGELOG.zh.md`](CHANGELOG.zh.md)。 --- @@ -466,6 +465,18 @@ Rust 数值类型(`u8`..`u128`、`i8`..`i128`、`usize`、`isize`、`f32`、 (例如用 `::` 强制的)仍通过 `FromStr` 接受。`NaN` / `±Infinity` 会被 序列化器拒绝(Ktav 不表示这些值)。 +## 错误模型迁移(0.9) + +**0.9 迁移(破坏性变更):** `Error::InvalidUtf8` 现在包含 +`{ valid_up_to, line, span }`,`ErrorKind::UnclosedCompound` 包含 +`{ kind, line, span }`。下游构造器需要填写新增字段;只关注部分字段时, +模式应使用 `..`。两种错误的 `line()` 都返回源行号。UTF-8 区间覆盖原始 +错误字节;EOF 诊断覆盖最内层未闭合起始标记至 EOF,并报告起始标记所在行。 +坐标保留 BOM 及 LF/CR/CRLF;`Span::line_col` 和 envelope 的 `line_text` +识别全部三种换行形式。I/O 及 writer 错误仍不携带源位置。下述十字段 JSON +格式不变。新增位置 oracle 位于规范草案的 `versions/0.9/tests`; +当前工作版本尚未发布。 + ## 示例:Ktav → JSON5 右侧使用 JSON5,因为它读起来像普通 JavaScript,允许注释,并且 @@ -899,9 +910,12 @@ ktav/ ## 安装 +下面的依赖为已发布的 0.8.1,不包含新增路径编辑 API 或 0.9 错误模型。 +准备中的 0.9.0 目前需要使用本源代码 checkout。 + ```toml [dependencies] -ktav = "0.8.0" +ktav = "0.8.1" serde = { version = "1", features = ["derive"] } ``` diff --git a/SECURITY.md b/SECURITY.md index 8189b2d..9e1314d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -10,7 +10,7 @@ release within a few days. | Version | Supported | |---------|--------------------| -| 0.8.x | ✅ | +| 0.9.x | ✅ | | older | ❌ — upgrade first | ## Reporting a vulnerability diff --git a/SECURITY.ru.md b/SECURITY.ru.md index e6a4736..bcc1b47 100644 --- a/SECURITY.ru.md +++ b/SECURITY.ru.md @@ -10,7 +10,7 @@ | Версия | Поддерживается | |---------|------------------------| -| 0.8.x | ✅ | +| 0.9.x | ✅ | | старше | ❌ — сперва обновитесь | ## Сообщение об уязвимости diff --git a/SECURITY.zh.md b/SECURITY.zh.md index 7405bbd..d2913f0 100644 --- a/SECURITY.zh.md +++ b/SECURITY.zh.md @@ -9,7 +9,7 @@ | 版本 | 支持 | |---------|------------------------| -| 0.8.x | ✅ | +| 0.9.x | ✅ | | 更早 | ❌ —— 请先升级 | ## 上报漏洞 diff --git a/root-docs/CHANGELOG/unreleased/body-1.md b/root-docs/CHANGELOG/unreleased/body-1.md index 7e78f6c..d8c578f 100644 --- a/root-docs/CHANGELOG/unreleased/body-1.md +++ b/root-docs/CHANGELOG/unreleased/body-1.md @@ -1,6 +1,10 @@ >>>>> lang=en ## [Unreleased] +### Changed + +- **Breaking in 0.9.0:** `InvalidUtf8` gains `line`/`span`; `UnclosedCompound` gains the opener's `line`. Downstream constructors and exhaustive field patterns must migrate. Direct Rust errors, public envelopes and C ABI source operations share raw-byte locations; UTF-8 spans are non-empty and EOF spans cover the innermost opener through EOF. LF, lone CR, CRLF and BOM offsets are preserved, including `Span::line_col` and envelope `line_text`. The ten-field JSON schema and I/O/writer location rules are unchanged. New regression oracles live only in the draft spec `versions/0.9/tests` and run across source entry points; CI refuses to skip a missing corpus. Version 0.9.0 is in preparation, not published. + ### Added - Rust-only source-editing API: `ktav::edit::edit_value` accepts an escaped dotted path, and `edit_value_segments` accepts a slice/vector of literal segments. Both replace an existing value without reformatting the document, preserve surrounding bytes and comments, and return a new `String`. @@ -17,6 +21,10 @@ >>>>> lang=ru ## [Не выпущено] +### Изменено + +- **Breaking в 0.9.0:** в `InvalidUtf8` добавлены `line`/`span`, в `UnclosedCompound` — `line` открывающего маркера. Downstream-конструкторы и patterns, перечисляющие все поля, требуют миграции. Прямые Rust-ошибки, публичный envelope и C ABI source operations используют общие координаты исходных байтов; UTF-8 spans непустые, EOF spans охватывают самый внутренний незакрытый opener до EOF. LF, одиночный CR, CRLF и BOM учитываются без нормализации, в том числе в `Span::line_col` и envelope `line_text`. Схема JSON из десяти полей и отсутствие позиций у I/O/writer не меняются. Новые regression oracles находятся только в draft спеки `versions/0.9/tests` и проверяются через source entry points; CI запрещает пропуск отсутствующего corpus. Версия 0.9.0 готовится, но не опубликована. + ### Добавлено - Rust API редактирования исходного текста: `ktav::edit::edit_value` принимает экранированный dotted-путь, `edit_value_segments` — слайс/вектор буквальных сегментов. Обе функции заменяют существующее значение без форматирования документа, сохраняют окружающие байты и комментарии и возвращают новый `String`. @@ -33,6 +41,10 @@ >>>>> lang=zh ## [尚未发布] +### 变更 + +- **0.9.0 破坏性变更:** `InvalidUtf8` 新增 `line`/`span`,`UnclosedCompound` 新增起始标记的 `line`。下游构造器及枚举全部字段的模式需要迁移。直接 Rust 错误、公开 envelope 和 C ABI 源操作共用原始字节坐标;UTF-8 区间非空,EOF 区间覆盖最内层未闭合起始标记至 EOF。LF、单独 CR、CRLF 及 BOM 均不经规范化,`Span::line_col` 和 envelope 的 `line_text` 同样遵守该规则。十字段 JSON 格式及 I/O/writer 无源位置的规则不变。新增回归 oracle 仅位于规范草案 `versions/0.9/tests`,由各源入口执行;CI 不允许因语料缺失而跳过。0.9.0 尚在准备中,未发布。 + ### 新增 - Rust 源文本编辑 API:`ktav::edit::edit_value` 接受带转义的点分路径,`edit_value_segments` 接受字面路径段切片/向量。两者均在不重新格式化文档的情况下替换已有值,保留周围字节与注释,并返回新的 `String`。 diff --git a/root-docs/README/error-migration/body-1.md b/root-docs/README/error-migration/body-1.md new file mode 100644 index 0000000..6609260 --- /dev/null +++ b/root-docs/README/error-migration/body-1.md @@ -0,0 +1,44 @@ +>>>>> lang=en +## Error model migration (0.9) + +**0.9 migration (breaking):** `Error::InvalidUtf8` now contains +`{ valid_up_to, line, span }`; `ErrorKind::UnclosedCompound` contains +`{ kind, line, span }`. Update downstream constructors with the new fields; +use `..` in patterns when only selected fields matter. `line()` now returns +the source line for both errors. UTF-8 spans cover original malformed bytes; +EOF diagnostics cover the innermost opener through EOF and report its line. +Coordinates preserve BOM and LF/CR/CRLF; `Span::line_col` and envelope +`line_text` recognize all three terminators. I/O and writer failures retain +absent source positions. The ten-field JSON schema below is unchanged. +The new location oracles live in the draft `versions/0.9/tests` spec corpus; +this working version has not been published. + +>>>>> lang=ru +## Миграция модели ошибок (0.9) + +**Миграция 0.9 (breaking):** `Error::InvalidUtf8` теперь содержит +`{ valid_up_to, line, span }`, `ErrorKind::UnclosedCompound` — +`{ kind, line, span }`. В downstream-конструкторах нужно заполнить новые +поля; в patterns используйте `..`, если важна только часть полей. +`line()` теперь возвращает исходную строку обеих ошибок. UTF-8 span +покрывает исходные ошибочные байты; EOF-диагностика охватывает самый +внутренний незакрытый opener до EOF и указывает его строку. Координаты +сохраняют BOM и LF/CR/CRLF; `Span::line_col` и envelope `line_text` +учитывают все три терминатора. Ошибки I/O и writer по-прежнему не имеют +позиции исходника. Схема JSON из десяти полей ниже не меняется. Новые +oracles координат находятся в draft corpus спеки `versions/0.9/tests`; +эта рабочая версия ещё не опубликована. + +>>>>> lang=zh +## 错误模型迁移(0.9) + +**0.9 迁移(破坏性变更):** `Error::InvalidUtf8` 现在包含 +`{ valid_up_to, line, span }`,`ErrorKind::UnclosedCompound` 包含 +`{ kind, line, span }`。下游构造器需要填写新增字段;只关注部分字段时, +模式应使用 `..`。两种错误的 `line()` 都返回源行号。UTF-8 区间覆盖原始 +错误字节;EOF 诊断覆盖最内层未闭合起始标记至 EOF,并报告起始标记所在行。 +坐标保留 BOM 及 LF/CR/CRLF;`Span::line_col` 和 envelope 的 `line_text` +识别全部三种换行形式。I/O 及 writer 错误仍不携带源位置。下述十字段 JSON +格式不变。新增位置 oracle 位于规范草案的 `versions/0.9/tests`; +当前工作版本尚未发布。 + diff --git a/root-docs/README/error-migration/meta.js b/root-docs/README/error-migration/meta.js new file mode 100644 index 0000000..eb86416 --- /dev/null +++ b/root-docs/README/error-migration/meta.js @@ -0,0 +1,7 @@ +export default { + "kind": "frontmatter", + "number": null, + "level": null, + "title": null, + "bodyParts": 1 +} diff --git a/root-docs/README/installation/body-1.md b/root-docs/README/installation/body-1.md index 958dc83..ea55cb6 100644 --- a/root-docs/README/installation/body-1.md +++ b/root-docs/README/installation/body-1.md @@ -1,9 +1,12 @@ >>>>> lang=en ## Installation +The published dependency below is 0.8.1; it does not include the new path +editing or 0.9 error model. Version 0.9.0 currently requires this source checkout. + ```toml [dependencies] -ktav = "0.8.0" +ktav = "0.8.1" serde = { version = "1", features = ["derive"] } ``` @@ -18,9 +21,12 @@ ktav-fmt --check config.ktav >>>>> lang=ru ## Установка +Ниже указана опубликованная 0.8.1: в ней нет новых методов редактирования +по пути и модели ошибок 0.9. Для готовящейся 0.9.0 пока нужен этот source checkout. + ```toml [dependencies] -ktav = "0.8.0" +ktav = "0.8.1" serde = { version = "1", features = ["derive"] } ``` @@ -35,9 +41,12 @@ ktav-fmt --check config.ktav >>>>> lang=zh ## 安装 +下面的依赖为已发布的 0.8.1,不包含新增路径编辑 API 或 0.9 错误模型。 +准备中的 0.9.0 目前需要使用本源代码 checkout。 + ```toml [dependencies] -ktav = "0.8.0" +ktav = "0.8.1" serde = { version = "1", features = ["derive"] } ``` diff --git a/root-docs/README/intro/body-1.md b/root-docs/README/intro/body-1.md index cdec20e..029f0d6 100644 --- a/root-docs/README/intro/body-1.md +++ b/root-docs/README/intro/body-1.md @@ -15,13 +15,11 @@ **Playground:** convert JSON / YAML / TOML / INI ⇄ Ktav in your browser at **[ktav-lang.github.io](https://ktav-lang.github.io/)**. -**Specification:** this crate implements **Ktav 0.8.0**, the version -named by `[package.metadata.ktav] spec-version` in `Cargo.toml`. The -format is versioned and maintained independently of this crate — the two -numbers move apart on purpose, since a crate release that changes no -format behaviour leaves `spec-version` where it was. See -[`ktav-lang/spec`](https://github.com/ktav-lang/spec) for the formal -document, and [`CHANGELOG.md`](CHANGELOG.md) for this crate's history. +**Version in preparation: 0.9.0, not published.** This branch combines +lossless source editing by path and the new source-error location model. +`Cargo.toml` targets the draft spec corpus `versions/0.9/tests`; published +0.8.1 remains unchanged. See [`ktav-lang/spec`](https://github.com/ktav-lang/spec) +for the specification and [`CHANGELOG.md`](CHANGELOG.md) for the migration. --- @@ -42,13 +40,12 @@ document, and [`CHANGELOG.md`](CHANGELOG.md) for this crate's history. **Песочница:** конвертация JSON / YAML / TOML / INI ⇄ Ktav прямо в браузере — **[ktav-lang.github.io](https://ktav-lang.github.io/)**. -**Спецификация:** этот crate реализует **Ktav 0.8.0** — версию, -указанную в `[package.metadata.ktav] spec-version` в `Cargo.toml`. -Формат версионируется и поддерживается независимо от crate-а, и два -номера намеренно расходятся: выпуск crate-а, не меняющий поведение -формата, оставляет `spec-version` на месте. См. -[`ktav-lang/spec`](https://github.com/ktav-lang/spec) для канонического -документа и [`CHANGELOG.ru.md`](CHANGELOG.ru.md) — историю crate-а. +**Готовится версия 0.9.0, ещё не опубликована.** Эта ветка объединяет +lossless-редактирование исходного текста по пути и новую модель координат +ошибок. `Cargo.toml` указывает на draft corpus спеки `versions/0.9/tests`; +опубликованная 0.8.1 не меняется. Спецификация — +[`ktav-lang/spec`](https://github.com/ktav-lang/spec), миграция — +[`CHANGELOG.ru.md`](CHANGELOG.ru.md). --- @@ -68,11 +65,10 @@ document, and [`CHANGELOG.md`](CHANGELOG.md) for this crate's history. **演练场:** 在浏览器中互转 JSON / YAML / TOML / INI ⇄ Ktav — **[ktav-lang.github.io](https://ktav-lang.github.io/)**。 -**规范:** 本 crate 实现 **Ktav 0.8.0**,即 `Cargo.toml` 中 -`[package.metadata.ktav] spec-version` 所指定的版本。格式与 crate 彼此 -独立地版本化与维护,两个号码会有意分开——不改变格式行为的 crate 发布会 -让 `spec-version` 保持原样。规范正文见 -[`ktav-lang/spec`](https://github.com/ktav-lang/spec),crate 的历史见 +**0.9.0 正在准备中,尚未发布。** 本分支同时包含按路径无损编辑源文本 +及新的源错误坐标模型。`Cargo.toml` 指向规范草案语料 `versions/0.9/tests`; +已发布的 0.8.1 保持不变。规范见 +[`ktav-lang/spec`](https://github.com/ktav-lang/spec),迁移说明见 [`CHANGELOG.zh.md`](CHANGELOG.zh.md)。 --- diff --git a/root-docs/README/manifest.js b/root-docs/README/manifest.js index 621b84c..9eb6d17 100644 --- a/root-docs/README/manifest.js +++ b/root-docs/README/manifest.js @@ -7,6 +7,7 @@ export default [ "compounds", "corners", "rust-usage", + "error-migration", "examples", "round-trip", "formatting", diff --git a/spec b/spec index 3889821..a452756 160000 --- a/spec +++ b/spec @@ -1 +1 @@ -Subproject commit 388982118c0d8f767bf283356ac9c9bb50fdc3f5 +Subproject commit a45275688557dc9ecc24a82afb491dd46e615d4d diff --git a/src/cabi/envelope.rs b/src/cabi/envelope.rs index 61bd3f6..e2497fe 100644 --- a/src/cabi/envelope.rs +++ b/src/cabi/envelope.rs @@ -1,96 +1,21 @@ -use crate::{Error, ErrorEnvelope, Span}; +use crate::{Error, ErrorEnvelope}; + +#[cfg(test)] +pub(super) use crate::error::source::line_of_offset; pub(super) fn envelope(err: &Error, source: &str) -> String { - let mut env = ErrorEnvelope::from_error(err, source); - // Spec § 6: every source-content parse error carries a 1-based line. - // `from_error` takes the line from `Error::line()`, which is `None` - // for an EOF-detected `UnclosedCompound` even though the error - // carries an opener..EOF span. The C ABI layer has the source, so - // the line is derived from the span's start (the opener line) here - // rather than shipping a null. Errors with no source position (Io, - // writer rejections) have no span and keep their honest nulls. - if env.line.is_none() && !source.is_empty() { - if let Some(span) = env.span { - env.line = Some(line_of_offset(source.as_bytes(), span.start as usize)); - } - } - env.to_json() + ErrorEnvelope::from_error(err, source).to_json() } -/// Wrap a non-`ktav::Error` diagnostic (a serde_json failure, the -/// top-level check) so it leaves through the same envelope channel as -/// everything else. +/// Неструктурированная диагностика использует тот же JSON-канал. pub(super) fn message_envelope(text: impl Into, source: &str) -> String { envelope(&Error::Message(text.into()), source) } -/// The raw-`InvalidUtf8` envelope for a source-bytes entry point whose -/// input failed `std::str::from_utf8` (spec § 3.1/§ 6.15): class -/// `InvalidUtf8` — never `Message` — with the § 6 location computed -/// over the original bytes. The span covers the offending sequence -/// (`Utf8Error::error_len`), or runs to end-of-input when the sequence -/// is truncated at EOF (`error_len` is `None`); the line counts § 3.2 -/// terminators over the raw bytes, before any BOM removal or trimming. +/// Координаты UTF-8 совпадают с прямым Rust API, без ABI-only исправлений. pub(super) fn not_utf8_envelope(src: &[u8], err: std::str::Utf8Error) -> String { - let valid_up_to = err.valid_up_to(); - let mut end = valid_up_to + err.error_len().unwrap_or(src.len() - valid_up_to); - // An illegal lead byte is reported by `std` as a one-byte sequence, - // but any following continuation bytes belong to the same ill-formed - // subsequence; the spec (§ 6.15) fixes only the span's start, so the - // span swallows them to cover the offending region as a whole. - if err.error_len() == Some(1) { - while src.get(end).is_some_and(|b| (0x80..=0xBF).contains(b)) { - end += 1; - } - } - let mut env = ErrorEnvelope::from_error( - &Error::InvalidUtf8 { valid_up_to }, - // No valid source text exists, so `line_text` honestly comes out - // null; `line` and `span` are filled in from the raw bytes below. - "", - ); - // `Span` is u32-based; a >4 GiB document is beyond what `Span` can - // address anywhere in this crate, so saturate (same trade-off as - // `Error::span`). - let start = u32::try_from(valid_up_to).unwrap_or(u32::MAX); - let end = u32::try_from(end).unwrap_or(u32::MAX); - env.line = Some(line_of_offset(src, valid_up_to)); - env.span = Some(Span::new(start, end)); + let error = crate::error::source::invalid_utf8(src, err); + let mut env = ErrorEnvelope::from_error(&error, ""); env.body = Some(format!("input is not valid UTF-8: {err}")); env.to_json() } - -/// 1-based line number of raw byte offset `pos`, counting each LF, lone -/// CR and CR LF pair as exactly one terminator (spec § 3.2). A whole -/// terminator counts only when it ends at or before `pos`, so an offset -/// pointing at a terminator belongs to the line that terminator closes. -/// Counted over the raw bytes, before BOM removal or trimming (§ 6.15); -/// `pos` beyond the end clamps to the end. -pub(super) fn line_of_offset(bytes: &[u8], pos: usize) -> u32 { - let mut line: u32 = 1; - let mut i = 0; - let bound = pos.min(bytes.len()); - while i < bound { - match bytes[i] { - b'\n' => { - line += 1; - i += 1; - } - b'\r' => { - if bytes.get(i + 1) == Some(&b'\n') { - // CRLF is ONE terminator and counts only for offsets - // strictly after its LF byte. - if i + 2 <= pos { - line += 1; - } - i += 2; - } else { - line += 1; - i += 1; - } - } - _ => i += 1, - } - } - line -} diff --git a/src/document/parser/edit/tree.rs b/src/document/parser/edit/tree.rs index 381e425..8e50d9e 100644 --- a/src/document/parser/edit/tree.rs +++ b/src/document/parser/edit/tree.rs @@ -989,18 +989,21 @@ impl<'a> Builder<'a> { if self.collecting.is_some() { return Err(Error::Structured(ErrorKind::UnclosedCompound { kind: CompoundKind::MultilineVerbatim, + line: 1, span: Span::new(0, eof_offset), })); } if self.stack.len() > 1 { return Err(Error::Structured(ErrorKind::UnclosedCompound { kind: CompoundKind::Object, + line: 1, span: Span::new(0, eof_offset), })); } if self.root_is_explicit && !self.root_consumed { return Err(Error::Structured(ErrorKind::UnclosedCompound { kind: CompoundKind::Object, + line: 1, span: Span::new(0, eof_offset), })); } diff --git a/src/document/parser/fmt_parser/mod.rs b/src/document/parser/fmt_parser/mod.rs index d86bc35..cf2e9b2 100644 --- a/src/document/parser/fmt_parser/mod.rs +++ b/src/document/parser/fmt_parser/mod.rs @@ -74,7 +74,7 @@ pub(crate) fn parse_with_trivia(text: &str) -> Result { } line_start = end + 1; } - return parser.finish(bytes.len() as u32); + return parser.finish(bytes.len() as u32, text); } let mut line_start: usize = start; @@ -105,5 +105,5 @@ pub(crate) fn parse_with_trivia(text: &str) -> Result { parser.handle_line(line, line_num, line_start as u32)?; line_start = next_start; } - parser.finish(bytes.len() as u32) + parser.finish(bytes.len() as u32, text) } diff --git a/src/document/parser/fmt_parser/positioned.rs b/src/document/parser/fmt_parser/positioned.rs index 3eda181..4f29538 100644 --- a/src/document/parser/fmt_parser/positioned.rs +++ b/src/document/parser/fmt_parser/positioned.rs @@ -83,7 +83,7 @@ impl<'a> PositionedParser<'a> { .push(TriviaLine::Comment(trimmed.to_string())); } - pub(super) fn finish(mut self, eof_offset: u32) -> Result { + pub(super) fn finish(mut self, eof_offset: u32, source: &str) -> Result { if let Some(c) = &self.collecting { let kind = match c.mode { MultilineMode::Stripped => CompoundKind::MultilineStripped, @@ -92,6 +92,7 @@ impl<'a> PositionedParser<'a> { let start = self.multiline_opener.unwrap_or(eof_offset); return Err(Error::Structured(ErrorKind::UnclosedCompound { kind, + line: crate::error::source::line_of_offset(source.as_bytes(), start as usize), span: Span::new(start, eof_offset), })); } @@ -103,6 +104,7 @@ impl<'a> PositionedParser<'a> { let start = *self.opener_offsets.last().unwrap(); return Err(Error::Structured(ErrorKind::UnclosedCompound { kind, + line: crate::error::source::line_of_offset(source.as_bytes(), start as usize), span: Span::new(start, eof_offset), })); } @@ -119,6 +121,7 @@ impl<'a> PositionedParser<'a> { let start = *self.opener_offsets.last().unwrap(); return Err(Error::Structured(ErrorKind::UnclosedCompound { kind, + line: crate::error::source::line_of_offset(source.as_bytes(), start as usize), span: Span::new(start, eof_offset), })); } diff --git a/src/document/parser/parser/mod.rs b/src/document/parser/parser/mod.rs index 86e4de4..7a80946 100644 --- a/src/document/parser/parser/mod.rs +++ b/src/document/parser/parser/mod.rs @@ -66,7 +66,7 @@ impl<'a> Parser<'a> { } } - pub(super) fn finish(mut self, eof_offset: u32) -> Result { + pub(super) fn finish(mut self, eof_offset: u32, source: &str) -> Result { if let Some(c) = &self.collecting { let kind = match c.mode { MultilineMode::Stripped => CompoundKind::MultilineStripped, @@ -75,6 +75,7 @@ impl<'a> Parser<'a> { let start = self.multiline_opener.unwrap_or(eof_offset); return Err(Error::Structured(ErrorKind::UnclosedCompound { kind, + line: crate::error::source::line_of_offset(source.as_bytes(), start as usize), span: Span::new(start, eof_offset), })); } @@ -86,6 +87,7 @@ impl<'a> Parser<'a> { let start = *self.opener_offsets.last().unwrap(); return Err(Error::Structured(ErrorKind::UnclosedCompound { kind, + line: crate::error::source::line_of_offset(source.as_bytes(), start as usize), span: Span::new(start, eof_offset), })); } @@ -102,6 +104,7 @@ impl<'a> Parser<'a> { let start = *self.opener_offsets.last().unwrap(); return Err(Error::Structured(ErrorKind::UnclosedCompound { kind, + line: crate::error::source::line_of_offset(source.as_bytes(), start as usize), span: Span::new(start, eof_offset), })); } diff --git a/src/document/parser/parser/parse_str.rs b/src/document/parser/parser/parse_str.rs index 542bef8..355adbf 100644 --- a/src/document/parser/parser/parse_str.rs +++ b/src/document/parser/parser/parse_str.rs @@ -57,7 +57,7 @@ fn parse_str_impl(text: &str, strict: bool) -> Result { } line_start = end + 1; } - return parser.finish(bytes.len() as u32); + return parser.finish(bytes.len() as u32, text); } let mut line_start: usize = start; @@ -96,5 +96,5 @@ fn parse_str_impl(text: &str, strict: bool) -> Result { parser.handle_line(line, line_num, line_start as u32)?; line_start = next_start; } - parser.finish(bytes.len() as u32) + parser.finish(bytes.len() as u32, text) } diff --git a/src/document/parser/tests/inline_values.rs b/src/document/parser/tests/inline_values.rs index 43c1284..e7ebace 100644 --- a/src/document/parser/tests/inline_values.rs +++ b/src/document/parser/tests/inline_values.rs @@ -409,7 +409,7 @@ fn parse_top_level_explicit_array() { fn expect_unclosed_root(src: &str, kind: crate::error::CompoundKind, start: u32, end: u32) { let err = crate::parse(src).unwrap_err(); match err { - crate::Error::Structured(crate::ErrorKind::UnclosedCompound { kind: k, span }) => { + crate::Error::Structured(crate::ErrorKind::UnclosedCompound { kind: k, span, .. }) => { assert_eq!(k, kind, "kind mismatch for {src:?}"); assert_eq!( span, diff --git a/src/error/envelope.rs b/src/error/envelope.rs index a9fd1f5..584f8fd 100644 --- a/src/error/envelope.rs +++ b/src/error/envelope.rs @@ -288,19 +288,11 @@ fn line_text_at(source: &str, pos: usize) -> Option { return None; } let bytes = source.as_bytes(); - let line_start = bytes[..pos] - .iter() - .rposition(|&b| b == b'\n') - .map_or(0, |i| i + 1); - let line_end = bytes[pos..] - .iter() - .position(|&b| b == b'\n') - .map_or(source.len(), |i| pos + i); - let mut line = &source[line_start..line_end]; - if line.ends_with('\r') { - line = &line[..line.len() - 1]; - } - Some(line.to_string()) + let (_, column) = crate::error::source::line_col(bytes, pos); + let line_start = pos - column as usize; + let line_end = memchr::memchr2(b'\r', b'\n', &bytes[line_start..]) + .map_or(source.len(), |offset| line_start + offset); + Some(source[line_start..line_end].to_string()) } /// Decode the raw key-path text carried by `DuplicateKey` / diff --git a/src/error/error.rs b/src/error/error.rs index 27553f0..eafb10b 100644 --- a/src/error/error.rs +++ b/src/error/error.rs @@ -83,26 +83,14 @@ impl Span { Some(&input[start..end]) } - /// Compute the 1-based line and 0-based column corresponding to - /// this span's `start` offset within `input`. Counts `\n` as a - /// line break; `\r` is treated as part of the previous line. + /// Compute the 1-based line and 0-based byte column at `start`. + /// LF, lone CR and CRLF each count as one line terminator. /// Out-of-range start clamps to `input.len()`. /// /// The returned column is a **0-based byte column** (bytes since /// the last line break), not a character or UTF-16 column. pub fn line_col(&self, input: &str) -> (u32, u32) { - let start = (self.start as usize).min(input.len()); - let bytes = input.as_bytes(); - let mut line: u32 = 1; - let mut last_nl: usize = 0; - for (i, &b) in bytes.iter().enumerate().take(start) { - if b == b'\n' { - line += 1; - last_nl = i + 1; - } - } - let col = (start - last_nl) as u32; - (line, col) + crate::error::source::line_col(input.as_bytes(), self.start as usize) } } @@ -162,20 +150,16 @@ pub enum Error { /// Decoded key path from the root to the offending pair's key. path: Vec, }, - /// A byte-level input (currently only [`crate::from_file`]) is not - /// valid UTF-8 (spec 0.7 § 6.15). This is a top-level variant rather - /// than an [`ErrorKind`] case because the check runs before any - /// line-oriented processing — there is no meaningful line number, - /// exactly as for `Unrepresentable`. The `&str`-taking entry points - /// (`parse`, `parse_strict`, `from_str`, `parse_events`) cannot - /// produce this error: a Rust `&str` is valid UTF-8 by construction. + /// Byte-level input is not valid UTF-8; validation precedes grammar. + /// Locations refer to original bytes, including BOM and line endings. + /// Entry points accepting `&str` cannot produce this error. InvalidUtf8 { - /// Byte offset of the first invalid UTF-8 sequence — the input - /// is valid UTF-8 up to this point - /// (`std::str::Utf8Error::valid_up_to`). The spec's error-span - /// SHOULD points here; [`Error::span`] reports it as an - /// insertion-point [`Span`]. + /// Byte offset of the first invalid UTF-8 sequence. valid_up_to: usize, + /// 1-based line of the invalid sequence. + line: u32, + /// Non-empty byte range covering the invalid sequence. + span: Span, }, } @@ -286,10 +270,13 @@ pub enum ErrorKind { /// A key contained an invalid byte (whitespace, `[`, `]`, `{`, `}`, /// `:`, `#`) or was empty after segmentation. InvalidKey { line: u32, key: String, span: Span }, - /// The document ended while a compound (object, array, multi-line - /// string) was still open. `span` covers from the opener through - /// EOF (or `Span::EMPTY` if the opener wasn't tracked). - UnclosedCompound { kind: CompoundKind, span: Span }, + /// EOF before a compound closed. `span` covers opener..EOF; + /// `line` is the 1-based line containing that opener. + UnclosedCompound { + kind: CompoundKind, + line: u32, + span: Span, + }, /// A closer (`}` or `]`) appeared without a matching open compound, /// or with the wrong shape relative to the open one. /// `expected` records what was actually open (or could be opened), diff --git a/src/error/error/display.rs b/src/error/error/display.rs index 75826e7..3a3ca47 100644 --- a/src/error/error/display.rs +++ b/src/error/error/display.rs @@ -159,7 +159,7 @@ impl Display for Error { } Ok(()) } - Error::InvalidUtf8 { valid_up_to } => write!( + Error::InvalidUtf8 { valid_up_to, .. } => write!( f, "InvalidUtf8: input is not valid UTF-8; first invalid byte sequence at byte offset {}", valid_up_to diff --git a/src/error/error/traits.rs b/src/error/error/traits.rs index c8484c1..5d3543d 100644 --- a/src/error/error/traits.rs +++ b/src/error/error/traits.rs @@ -31,12 +31,12 @@ impl Error { /// Returns the 1-based line number associated with the error, if /// available. `None` for [`Error::Io`], [`Error::Message`], free- /// form [`Error::Syntax`], [`Error::Unrepresentable`], - /// [`Error::UnrepresentableAt`], [`Error::InvalidUtf8`], - /// EOF-detected `UnclosedCompound`, and - /// the parser-internal `Other` variants that lack a line number. + /// [`Error::UnrepresentableAt`] and parser-internal `Other` errors + /// that lack a line number. pub fn line(&self) -> Option { match self { Error::Structured(k) => k.line(), + Error::InvalidUtf8 { line, .. } => Some(*line), _ => None, } } @@ -44,20 +44,14 @@ impl Error { /// Returns the byte-offset span associated with the error, if /// available. `None` for [`Error::Io`], [`Error::Message`], /// [`Error::Syntax`], [`Error::Unrepresentable`] and - /// [`Error::UnrepresentableAt`]. [`Error::InvalidUtf8`] returns an - /// insertion-point span at the byte offset of the first invalid - /// UTF-8 sequence. May return `Some(Span::EMPTY)` for an + /// [`Error::UnrepresentableAt`]. [`Error::InvalidUtf8`] covers the + /// first invalid UTF-8 sequence. May return `Some(Span::EMPTY)` for an /// internal-state structured error that has no meaningful source /// range. pub fn span(&self) -> Option { match self { Error::Structured(k) => Some(k.span()), - Error::InvalidUtf8 { valid_up_to } => { - // `Span` is u32-based; a >4 GiB document is beyond what - // `Span` can address anywhere in this crate, so saturate. - let offset = u32::try_from(*valid_up_to).unwrap_or(u32::MAX); - Some(Span::new(offset, offset)) - } + Error::InvalidUtf8 { span, .. } => Some(*span), _ => None, } } @@ -77,15 +71,14 @@ impl Error { /// [`Error::InvalidUtf8`] error, else `None`. pub fn valid_up_to(&self) -> Option { match self { - Error::InvalidUtf8 { valid_up_to } => Some(*valid_up_to), + Error::InvalidUtf8 { valid_up_to, .. } => Some(*valid_up_to), _ => None, } } } impl ErrorKind { - /// 1-based line number, or `None` for variants where the failure - /// is detected at EOF or carries no line context. + /// 1-based line number; `None` only for internal errors without context. pub fn line(&self) -> Option { match self { ErrorKind::MissingSeparatorSpace { line, .. } @@ -102,8 +95,8 @@ impl ErrorKind { | ErrorKind::UnterminatedQuotedKey { line, .. } | ErrorKind::MalformedInlineCompound { line, .. } | ErrorKind::BadEscapeSequence { line, .. } - | ErrorKind::OrphanLineAfterTopLevelInline { line, .. } => Some(*line), - ErrorKind::UnclosedCompound { .. } => None, + | ErrorKind::OrphanLineAfterTopLevelInline { line, .. } + | ErrorKind::UnclosedCompound { line, .. } => Some(*line), ErrorKind::Other { line, .. } => *line, } } diff --git a/src/error/mod.rs b/src/error/mod.rs index 82ccf66..e1f3e33 100644 --- a/src/error/mod.rs +++ b/src/error/mod.rs @@ -3,6 +3,7 @@ pub mod envelope; pub mod error; pub mod result; +pub(crate) mod source; pub use envelope::ErrorEnvelope; pub use error::{CompoundKind, ConflictKind, Error, ErrorKind, ReasonCode, Span}; diff --git a/src/error/source.rs b/src/error/source.rs new file mode 100644 index 0000000..cddf977 --- /dev/null +++ b/src/error/source.rs @@ -0,0 +1,54 @@ +use super::{Error, Span}; + +/// Координаты исходных байтов; CRLF считается одним терминатором. +pub(crate) fn line_col(bytes: &[u8], pos: usize) -> (u32, u32) { + let bound = pos.min(bytes.len()); + let mut line = 1u32; + let mut line_start = 0usize; + let mut i = 0; + while i < bound { + match bytes[i] { + b'\r' if bytes.get(i + 1) == Some(&b'\n') => { + if i + 2 > bound { + break; + } + i += 2; + line = line.saturating_add(1); + line_start = i; + } + b'\r' | b'\n' => { + i += 1; + line = line.saturating_add(1); + line_start = i; + } + _ => i += 1, + } + } + (line, u32::try_from(bound - line_start).unwrap_or(u32::MAX)) +} + +pub(crate) fn line_of_offset(bytes: &[u8], pos: usize) -> u32 { + line_col(bytes, pos).0 +} + +/// UTF-8 проверяется до грамматики; диапазон сохраняет исходные байты. +pub(crate) fn invalid_utf8(source: &[u8], error: std::str::Utf8Error) -> Error { + let valid_up_to = error.valid_up_to(); + let mut end = valid_up_to + error.error_len().unwrap_or(source.len() - valid_up_to); + if error.error_len() == Some(1) { + while source + .get(end) + .is_some_and(|byte| (0x80..=0xBF).contains(byte)) + { + end += 1; + } + } + Error::InvalidUtf8 { + valid_up_to, + line: line_of_offset(source, valid_up_to), + span: Span::new( + u32::try_from(valid_up_to).unwrap_or(u32::MAX), + u32::try_from(end).unwrap_or(u32::MAX), + ), + } +} diff --git a/src/lib.rs b/src/lib.rs index ad2a8ef..df51228 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -207,9 +207,8 @@ pub fn from_str(s: &str) -> Result { /// failures (file not found, permission denied) remain [`Error::Io`]. pub fn from_file>(path: P) -> Result { let bytes = fs::read(path)?; - let text = std::str::from_utf8(&bytes).map_err(|e| Error::InvalidUtf8 { - valid_up_to: e.valid_up_to(), - })?; + let text = std::str::from_utf8(&bytes) + .map_err(|error| crate::error::source::invalid_utf8(&bytes, error))?; from_str(text) } diff --git a/src/serde_bridge/thin/event_parser/counter_tests.rs b/src/serde_bridge/thin/event_parser/counter_tests.rs index d4451bc..94cd6ac 100644 --- a/src/serde_bridge/thin/event_parser/counter_tests.rs +++ b/src/serde_bridge/thin/event_parser/counter_tests.rs @@ -32,7 +32,7 @@ fn parse_counters(doc: &str) -> (usize, usize, usize) { } line_start = end + 1; } - p.finish(bytes.len() as u32, &mut events).unwrap(); + p.finish(bytes.len() as u32, doc, &mut events).unwrap(); (p.dbg_node_allocs, p.dbg_index_probes, p.dbg_entry_compares) } @@ -61,7 +61,7 @@ fn registration_scans(doc: &str) -> usize { } line_start = end + 1; } - p.finish(bytes.len() as u32, &mut events).unwrap(); + p.finish(bytes.len() as u32, doc, &mut events).unwrap(); p.dbg_reg_scans } diff --git a/src/serde_bridge/thin/event_parser/lines.rs b/src/serde_bridge/thin/event_parser/lines.rs index 09eca2f..8a35e49 100644 --- a/src/serde_bridge/thin/event_parser/lines.rs +++ b/src/serde_bridge/thin/event_parser/lines.rs @@ -86,7 +86,7 @@ pub(crate) fn parse_events<'a>(text: &'a str, bump: &'a Bump) -> Result<(EventSt } } - p.finish(bytes.len() as u32, &mut events)?; + p.finish(bytes.len() as u32, text, &mut events)?; Ok((events, p.reopens)) } // --------------------------------------------------------------------------- @@ -97,6 +97,7 @@ impl<'a> EventParser<'a> { pub(crate) fn finish>( &mut self, eof_offset: u32, + source: &str, events: &mut S, ) -> Result<()> { if let Some(c) = &self.collecting { @@ -107,6 +108,7 @@ impl<'a> EventParser<'a> { let start = self.multiline_opener.unwrap_or(eof_offset); return Err(Error::Structured(ErrorKind::UnclosedCompound { kind, + line: crate::error::source::line_of_offset(source.as_bytes(), start as usize), span: Span::new(start, eof_offset), })); } @@ -118,6 +120,7 @@ impl<'a> EventParser<'a> { let start = *self.opener_offsets.last().unwrap(); return Err(Error::Structured(ErrorKind::UnclosedCompound { kind, + line: crate::error::source::line_of_offset(source.as_bytes(), start as usize), span: Span::new(start, eof_offset), })); } @@ -133,6 +136,7 @@ impl<'a> EventParser<'a> { let start = *self.opener_offsets.last().unwrap(); return Err(Error::Structured(ErrorKind::UnclosedCompound { kind, + line: crate::error::source::line_of_offset(source.as_bytes(), start as usize), span: Span::new(start, eof_offset), })); } diff --git a/tests/suite/api/text/docs/format/format_canonical_equivalence.rs b/tests/suite/api/text/docs/format/format_canonical_equivalence.rs index 693f8ee..96696d1 100644 --- a/tests/suite/api/text/docs/format/format_canonical_equivalence.rs +++ b/tests/suite/api/text/docs/format/format_canonical_equivalence.rs @@ -14,7 +14,7 @@ use std::fs; use std::path::{Path, PathBuf}; -const SPEC_VERSION: &str = "0.8"; +const SPEC_VERSION: &str = "0.9"; fn resolve_spec_root() -> Option { let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); diff --git a/tests/suite/api/text/docs/format/format_comments_preserved.rs b/tests/suite/api/text/docs/format/format_comments_preserved.rs index 87f3aa3..f0bbc3f 100644 --- a/tests/suite/api/text/docs/format/format_comments_preserved.rs +++ b/tests/suite/api/text/docs/format/format_comments_preserved.rs @@ -173,7 +173,7 @@ fn no_comments_inside_multiline_string_block_are_not_comments() { // Corpus fixtures // --------------------------------------------------------------------------- -const SPEC_VERSION: &str = "0.8"; +const SPEC_VERSION: &str = "0.9"; fn resolve_spec_root() -> Option { let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); diff --git a/tests/suite/api/text/docs/format/format_explicit_root_unclosed.rs b/tests/suite/api/text/docs/format/format_explicit_root_unclosed.rs index f90c6bd..eaf56b5 100644 --- a/tests/suite/api/text/docs/format/format_explicit_root_unclosed.rs +++ b/tests/suite/api/text/docs/format/format_explicit_root_unclosed.rs @@ -13,7 +13,7 @@ fn expect_unclosed(src: &str, kind: CompoundKind, start: u32, end: u32) { Err(e) => e, }; match &err { - Error::Structured(ErrorKind::UnclosedCompound { kind: k, span }) => { + Error::Structured(ErrorKind::UnclosedCompound { kind: k, span, .. }) => { assert_eq!(*k, kind, "kind mismatch for {src:?}"); assert_eq!(*span, Span::new(start, end), "span mismatch for {src:?}"); } diff --git a/tests/suite/api/text/docs/format/format_idempotent.rs b/tests/suite/api/text/docs/format/format_idempotent.rs index 9217ef0..cf2ae78 100644 --- a/tests/suite/api/text/docs/format/format_idempotent.rs +++ b/tests/suite/api/text/docs/format/format_idempotent.rs @@ -16,7 +16,7 @@ use std::fs; use std::path::{Path, PathBuf}; -const SPEC_VERSION: &str = "0.8"; +const SPEC_VERSION: &str = "0.9"; fn resolve_spec_root() -> Option { let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); diff --git a/tests/suite/api/thin/scan/top_level_inline_thin.rs b/tests/suite/api/thin/scan/top_level_inline_thin.rs index f7452da..b7886cb 100644 --- a/tests/suite/api/thin/scan/top_level_inline_thin.rs +++ b/tests/suite/api/thin/scan/top_level_inline_thin.rs @@ -178,7 +178,7 @@ fn explicit_root_unclosed_at_eof_is_unclosed_compound() { // event stream, the owned parser, and serde `from_str` alike. let err = collect("{\n a: 1\n").unwrap_err(); match &err { - Error::Structured(ErrorKind::UnclosedCompound { kind, span }) => { + Error::Structured(ErrorKind::UnclosedCompound { kind, span, .. }) => { assert_eq!(*kind, CompoundKind::Object); assert_eq!(*span, Span::new(0, 11)); // opener `{` .. EOF } diff --git a/tests/suite/api/value/cabi_corpus.rs b/tests/suite/api/value/cabi_corpus.rs index a32652e..778b40f 100644 --- a/tests/suite/api/value/cabi_corpus.rs +++ b/tests/suite/api/value/cabi_corpus.rs @@ -12,7 +12,7 @@ use std::fs; use std::path::{Path, PathBuf}; -const SPEC_VERSION: &str = "0.8"; +const SPEC_VERSION: &str = "0.9"; fn resolve_spec_root() -> Option { let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); diff --git a/tests/suite/errors/envelope/spans.rs b/tests/suite/errors/envelope/spans.rs index 4b69996..7b03965 100644 --- a/tests/suite/errors/envelope/spans.rs +++ b/tests/suite/errors/envelope/spans.rs @@ -48,13 +48,17 @@ fn line_text_edges() { assert_eq!(e.line_text, None); assert_eq!(e.span, None); - // InvalidUtf8: insertion-point span at valid_up_to, no line - let err = Error::InvalidUtf8 { valid_up_to: 7 }; + // Координаты сохраняются при публичном преобразовании в envelope. + let err = Error::InvalidUtf8 { + valid_up_to: 7, + line: 1, + span: Span::new(7, 8), + }; let e = env(&err, "0123456789"); assert_eq!(e.error, "InvalidUtf8"); - assert_eq!(e.span, Some(Span::new(7, 7))); + assert_eq!(e.span, Some(Span::new(7, 8))); assert_eq!(e.line_text, Some("0123456789".to_string())); - assert_eq!(e.line, None); + assert_eq!(e.line, Some(1)); assert_eq!(e.spec_section, Some("§6.15".to_string())); } @@ -103,6 +107,7 @@ fn every_error_kind_maps_onto_envelope() { }, ErrorKind::UnclosedCompound { kind: ktav::CompoundKind::Object, + line: 1, span, }, ErrorKind::UnbalancedBracket { diff --git a/tests/suite/errors/surface/error_accessors.rs b/tests/suite/errors/surface/error_accessors.rs index e4dd50a..ffaabce 100644 --- a/tests/suite/errors/surface/error_accessors.rs +++ b/tests/suite/errors/surface/error_accessors.rs @@ -91,11 +91,9 @@ fn invalid_key_has_line_and_span() { } #[test] -fn unclosed_compound_has_no_line_but_has_span() { - // EOF-detected: no line is meaningful, but the span covers - // opener..EOF. +fn unclosed_compound_has_opener_line_and_span() { let e = parse_err("obj: {\n a: 1\n"); - assert_eq!(e.line(), None); + assert_eq!(e.line(), Some(1)); let s = e.span().expect("span"); assert!(s.end >= s.start); } @@ -135,10 +133,10 @@ fn missing_separator_has_line_and_span() { // --------------------------------------------------------------------------- #[test] -fn error_kind_line_is_none_for_unclosed_compound() { +fn error_kind_line_is_opener_line_for_unclosed_compound() { let e = parse_err("obj: {\n"); if let Error::Structured(k) = &e { - assert_eq!(k.line(), None); + assert_eq!(k.line(), Some(1)); // Span is always populated, even if EMPTY. let _ = k.span(); } else { diff --git a/tests/suite/errors/surface/invalid_utf8.rs b/tests/suite/errors/surface/invalid_utf8.rs index 52801ea..9c5ce9f 100644 --- a/tests/suite/errors/surface/invalid_utf8.rs +++ b/tests/suite/errors/surface/invalid_utf8.rs @@ -27,13 +27,13 @@ fn from_file_lone_continuation_byte_reports_invalid_utf8() { .expect_err("must reject invalid UTF-8"); let _ = std::fs::remove_file(&path); - let Error::InvalidUtf8 { valid_up_to } = &err else { + let Error::InvalidUtf8 { valid_up_to, .. } = &err else { panic!("expected Error::InvalidUtf8, got {err:?}"); }; assert_eq!(*valid_up_to, 8); assert_eq!(err.valid_up_to(), Some(8)); - assert_eq!(err.span(), Some(Span::new(8, 8))); - assert_eq!(err.line(), None); + assert_eq!(err.span(), Some(Span::new(8, 9))); + assert_eq!(err.line(), Some(2)); assert!(err.to_string().starts_with("InvalidUtf8")); } @@ -44,7 +44,7 @@ fn from_file_truncated_multibyte_sequence_reports_invalid_utf8() { .expect_err("must reject truncated sequence"); let _ = std::fs::remove_file(&path); - let Error::InvalidUtf8 { valid_up_to: 3 } = &err else { + let Error::InvalidUtf8 { valid_up_to: 3, .. } = &err else { panic!("expected Error::InvalidUtf8 {{ valid_up_to: 3 }}, got {err:?}"); }; } diff --git a/tests/suite/errors/surface/structured_errors.rs b/tests/suite/errors/surface/structured_errors.rs index fd1b103..d561ce5 100644 --- a/tests/suite/errors/surface/structured_errors.rs +++ b/tests/suite/errors/surface/structured_errors.rs @@ -16,13 +16,10 @@ use std::path::{Path, PathBuf}; use ktav::{CompoundKind, Error, ErrorKind, Span}; -const SPEC_VERSION: &str = "0.8"; +const SPEC_VERSION: &str = "0.9"; -/// Floor on the number of fixtures the walk must reach. The 0.7 corpus -/// holds 74; this is deliberately exact-ish rather than "> 0", because -/// the failure mode being guarded against is running a *smaller* corpus -/// than intended and calling it a pass. -const MIN_INVALID_FIXTURES: usize = 74; +/// Не допускать неполный или устаревший corpus 0.9. +const MIN_INVALID_FIXTURES: usize = 162; fn resolve_spec_root() -> Option { let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); diff --git a/tests/suite/spec.rs b/tests/suite/spec.rs index 354d205..bca4e7c 100644 --- a/tests/suite/spec.rs +++ b/tests/suite/spec.rs @@ -23,6 +23,8 @@ //! 4. if none contains a `versions/` dir, the test logs and returns — //! it does not fail, so CI without the spec checkout stays green. +#[path = "spec/diagnostics.rs"] +mod diagnostics; #[path = "spec/invalid.rs"] mod invalid; #[path = "spec/manifest.rs"] diff --git a/tests/suite/spec/diagnostics.rs b/tests/suite/spec/diagnostics.rs new file mode 100644 index 0000000..8e27ba4 --- /dev/null +++ b/tests/suite/spec/diagnostics.rs @@ -0,0 +1,146 @@ +use super::support::{collect_invalid_fixtures, resolve_spec_root, InvalidFixture}; +use ktav::{Error, ErrorEnvelope, Span}; +use serde_json::Value as Json; + +fn check_location(fixture: &InvalidFixture, line: Option, span: Option) { + let expected = fixture.location.as_ref().expect("location oracle"); + assert_eq!(line, Some(expected.line), "{}: source line", fixture.rel); + let actual = span.unwrap_or_else(|| panic!("{}: missing span", fixture.rel)); + assert_eq!(actual.start, expected.start, "{}: span start", fixture.rel); + assert!( + actual.start < actual.end && actual.end as usize <= fixture.bytes.len(), + "{}: invalid span {actual:?}", + fixture.rel + ); + if let Some(end) = expected.end { + assert_eq!(actual.end, end, "{}: span end", fixture.rel); + } + if let Some(minimum_end) = expected.minimum_end { + assert!( + actual.end >= minimum_end, + "{}: incomplete covering span", + fixture.rel + ); + } +} + +fn check_error(fixture: &InvalidFixture, error: &Error, source: Option<&str>) { + let category = match error { + Error::Structured(kind) => kind.code_name(), + Error::InvalidUtf8 { + valid_up_to, span, .. + } => { + assert_eq!( + *valid_up_to, span.start as usize, + "{}: UTF-8 offset", + fixture.rel + ); + "InvalidUtf8" + } + other => panic!("{}: unexpected error {other:?}", fixture.rel), + }; + assert_eq!(category, fixture.expected, "{}: category", fixture.rel); + check_location(fixture, error.line(), error.span()); + let envelope = ErrorEnvelope::from_error(error, source.unwrap_or("")); + assert_eq!(envelope.error, fixture.expected); + assert_eq!(envelope.line, error.line()); + assert_eq!(envelope.span, error.span()); + if let Some(source) = source { + assert_eq!( + error.span().unwrap().line_col(source).0, + fixture.location.as_ref().unwrap().line, + "{}: Span::line_col", + fixture.rel + ); + let normalized = source.replace("\r\n", "\n").replace('\r', "\n"); + let line = fixture.location.as_ref().unwrap().line as usize; + assert_eq!( + envelope.line_text.as_deref(), + normalized.split('\n').nth(line - 1), + "{}: line_text", + fixture.rel + ); + } else { + assert!(envelope.line_text.is_none()); + } + let wire: Json = serde_json::from_str(&envelope.to_json()).unwrap(); + assert_eq!(wire["line"].as_u64(), error.line().map(u64::from)); + assert_eq!( + wire["span"]["start"].as_u64(), + error.span().map(|span| u64::from(span.start)) + ); + assert_eq!( + wire["span"]["end"].as_u64(), + error.span().map(|span| u64::from(span.end)) + ); +} + +#[test] +fn source_diagnostics_match_versioned_fixture_oracles() { + let Some(root) = resolve_spec_root() else { + eprintln!("skipping diagnostics: spec corpus unavailable outside required CI"); + return; + }; + let fixtures = collect_invalid_fixtures(&root); + let mut checked = 0; + for fixture in fixtures.iter().filter(|fixture| fixture.location.is_some()) { + checked += 1; + let source = std::str::from_utf8(&fixture.bytes).ok(); + if let Some(text) = source { + check_error(fixture, &ktav::parse(text).unwrap_err(), source); + check_error(fixture, &ktav::parse_strict(text).unwrap_err(), source); + check_error(fixture, &ktav::format_str(text).unwrap_err(), source); + let error = ktav::parse_events(text, |_event: ktav::ParseEvent<'_>| ()).unwrap_err(); + check_error(fixture, &error, source); + check_error( + fixture, + &ktav::from_str::(text).unwrap_err(), + source, + ); + } + let error = ktav::from_file::(&fixture.path).unwrap_err(); + check_error(fixture, &error, source); + } + assert!( + checked > 0, + "versioned diagnostic fixtures must not be skipped" + ); + eprintln!("spec diagnostics: {checked} fixture locations verified across Rust entry points"); +} + +#[cfg(feature = "cabi")] +#[test] +fn source_cabi_diagnostics_match_same_fixture_oracles() { + let Some(root) = resolve_spec_root() else { + eprintln!("skipping C ABI diagnostics: spec corpus unavailable outside required CI"); + return; + }; + let operations = [ + ktav::cabi::loads as fn(&[u8]) -> Result, String>, + ktav::cabi::loads_strict, + ktav::cabi::format, + ktav::cabi::canonical_from_source, + ]; + let fixtures = collect_invalid_fixtures(&root); + let mut checked = 0; + for fixture in fixtures.iter().filter(|fixture| fixture.location.is_some()) { + checked += 1; + for operation in operations { + let error = operation(&fixture.bytes).unwrap_err(); + let wire: Json = serde_json::from_str(&error).expect("C ABI error envelope"); + assert_eq!(wire["error"].as_str(), Some(fixture.expected.as_str())); + let line = wire["line"] + .as_u64() + .map(|value| u32::try_from(value).unwrap()); + let span = Span::new( + u32::try_from(wire["span"]["start"].as_u64().expect("span start")).unwrap(), + u32::try_from(wire["span"]["end"].as_u64().expect("span end")).unwrap(), + ); + check_location(fixture, line, Some(span)); + } + } + assert!(checked > 0, "C ABI diagnostic fixtures must not be skipped"); + eprintln!( + "spec diagnostics: {checked} fixture locations verified across C ABI source operations" + ); +} diff --git a/tests/suite/spec/support.rs b/tests/suite/spec/support.rs index a8a55d4..6fcaed0 100644 --- a/tests/suite/spec/support.rs +++ b/tests/suite/spec/support.rs @@ -8,7 +8,7 @@ use super::manifest::RunnerManifest; use ktav::ReasonCode; use serde_json::Value as JsonValue; -pub(crate) const SPEC_VERSION: &str = "0.8"; +pub(crate) const SPEC_VERSION: &str = "0.9"; pub(crate) fn resolve_spec_root() -> Option { let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); @@ -26,7 +26,14 @@ pub(crate) fn resolve_spec_root() -> Option { // gracefully skipping the suite. let root = candidates .into_iter() - .find(|p| p.join("versions").join(SPEC_VERSION).is_dir())?; + .find(|p| p.join("versions").join(SPEC_VERSION).is_dir()); + if std::env::var_os("KTAV_REQUIRE_SPEC").is_some() { + assert!( + root.is_some(), + "required spec {SPEC_VERSION} corpus is missing" + ); + } + let root = root?; // § 8.5: load and enforce the corpus manifest BEFORE any fixture is // enumerated. Every runner in this suite reaches its fixtures through @@ -181,13 +188,14 @@ fn validate_error_fixture(fixture: &JsonValue, rel: impl std::fmt::Display) -> & let map = fixture .as_object() .unwrap_or_else(|| panic!("fixture {rel}: top-level JSON is not an object")); - if map.len() != 2 { - panic!( - "fixture {rel}: expected exactly the fields expected_error/note, \ - found {} field(s)", - map.len() - ); - } + assert!( + (map.len() == 2 || map.len() == 3) + && map.keys().all(|key| matches!( + key.as_str(), + "expected_error" | "note" | "expected_location" + )), + "fixture {rel}: unexpected invalid oracle fields" + ); for required in ["expected_error", "note"] { if !map.contains_key(required) { panic!("fixture {rel}: missing required field {required:?}"); @@ -208,6 +216,15 @@ fn validate_error_fixture(fixture: &JsonValue, rel: impl std::fmt::Display) -> & expected } +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ExpectedLocation { + pub(crate) line: u32, + pub(crate) start: u32, + pub(crate) end: Option, + pub(crate) minimum_end: Option, +} + /// An invalid fixture with its validated oracle. `bytes` are read once /// here so both the owned-API and thin-API passes walk the identical /// corpus. @@ -218,6 +235,7 @@ pub(crate) struct InvalidFixture { pub(crate) bytes: Vec, /// Validated `expected_error` category name. pub(crate) expected: String, + pub(crate) location: Option, } /// Collect every invalid fixture with its validated oracle. Panics if @@ -259,11 +277,38 @@ pub(crate) fn collect_invalid_fixtures(spec_root: &Path) -> Vec Ok(b) => b, Err(e) => panic!("fixture {}: unreadable `.ktav` file: {}", rel, e), }; + let location = fixture.get("expected_location").map(|value| { + let map = value + .as_object() + .expect("expected_location must be an object"); + assert_eq!( + map.len(), + 3, + "fixture {rel}: location requires three fields" + ); + let location: ExpectedLocation = serde_json::from_value(value.clone()) + .unwrap_or_else(|error| panic!("fixture {rel}: invalid location: {error}")); + assert!( + location.end.is_some() ^ location.minimum_end.is_some(), + "fixture {rel}: endpoint must be exclusive" + ); + let end = location.end.or(location.minimum_end).unwrap(); + assert!( + location.line > 0 && location.start < end && end as usize <= bytes.len(), + "fixture {rel}: invalid location bounds" + ); + location + }); + assert!( + !rel.replace('\\', "/").starts_with("diagnostics09/") || location.is_some(), + "fixture {rel}: required location oracle is missing" + ); fixtures.push(InvalidFixture { rel, path: ktav_path.clone(), bytes, expected, + location, }); } fixtures From 8f8dd875f6c36744b779864f6c9db3a66396c248 Mon Sep 17 00:00:00 2001 From: Marat K Date: Sun, 25 Oct 2026 00:21:40 +0200 Subject: [PATCH 2/2] =?UTF-8?q?perf(runtime):=20=D1=83=D1=81=D0=BA=D0=BE?= =?UTF-8?q?=D1=80=D0=B8=D1=82=D1=8C=20=D0=B4=D0=B8=D0=B0=D0=B3=D0=BD=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D0=B8=D0=BA=D1=83=20=D0=B8=20JSON?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Искать строку ошибки обратно от span без сканирования всего префикса. Выделять дочерние merge-буферы и seen-списки только при необходимости. Проверять JSON-escapes безопасными восьмибайтовыми операциями; копировать строки без escapes целиком и выделить ASCII encoder, сохранив прежний Unicode-путь и точные выходные байты. Добавить проверки координат, JSON-oracles, packed-byte classifier и отсутствия arena-аллокации у пустого дочернего буфера. Добавить короткий парный benchmark с общим внешним лимитом batch, обновить changelog EN/RU/ZH. Публичный API, версия, spec pin и зависимости не меняются; нового unsafe нет. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + CHANGELOG.ru.md | 1 + CHANGELOG.zh.md | 1 + Cargo.toml | 4 + benches/runtime_paths.rs | 157 ++++++++++++++++++ root-docs/CHANGELOG/unreleased/body-1.md | 3 + src/error/envelope.rs | 126 +++++++++++++- src/serde_bridge/thin/merge/reopen.rs | 25 ++- src/serde_bridge/thin/merge/table.rs | 7 +- tests/suite/errors/envelope.rs | 2 + .../envelope/performance_regressions.rs | 78 +++++++++ 11 files changed, 386 insertions(+), 19 deletions(-) create mode 100644 benches/runtime_paths.rs create mode 100644 tests/suite/errors/envelope/performance_regressions.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 78cc08d..f3b27c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ For the format specification's own history, see the - Write nested inline replacements into one buffer; avoid temporary character vectors and a second copy of parsed textual path segments. Keep semantic verification and byte-preserving behavior. Add short paired benchmarks with separate compilation and an external 15-second process limit. - The positional pass shares structural classifiers with the parser without constructing unused values or multiline bodies. Verify the edited target and untouched tree without cloning the replacement; render borrowed block compounds without a temporary root wrapper. Preserve validation, writer error paths and byte-for-byte output, with focused regression tests and expanded paired benchmarks. - Borrow literal path slices and unescaped textual segments instead of copying each segment. Assemble raw-marker prefixes and unchanged replacement bodies directly into the result; render verbatim replacements in one buffer without an intermediate EOL-converted string. Preserve exact output and full semantic verification; measure text and segment paths separately in short paired benchmarks. +- Locate diagnostic line text by reverse terminator search instead of rescanning the document prefix. Probe JSON strings with safe eight-byte operations, bulk-copy unescaped UTF-8, specialize escaped ASCII and retain the Unicode encoder. Allocate reopen-merge child buffers and seen-key storage lazily. Preserve exact bytes and locations; add allocation/escaping regressions and paired runtime benchmarks with one external 15-second limit for the entire process batch. ## [0.8.1] — 2026-10-21 diff --git a/CHANGELOG.ru.md b/CHANGELOG.ru.md index 30841dd..88ecc4c 100644 --- a/CHANGELOG.ru.md +++ b/CHANGELOG.ru.md @@ -28,6 +28,7 @@ Cargo-конвенцией: до 1.0 bump MINOR считается ломающ - Вложенные inline-замены пишутся в один буфер, без временного вектора символов и повторного копирования разобранных сегментов текстового пути. Семантическая проверка и сохранение исходных байтов остаются. Добавлены короткие парные benchmarks с отдельной компиляцией и внешним лимитом 15 секунд на процесс. - Позиционный проход использует общие с parser структурные классификаторы без построения ненужных значений и multiline-тел. Проверка цели и неизменной части дерева обходится без клонирования replacement; блочные контейнеры рендерятся по заимствованию без временной корневой обёртки. Валидация, пути writer-ошибок и точные выходные байты сохранены; добавлены регрессии и расширены парные benchmarks. - Буквальные сегменты пути и сегменты текстового пути без escape заимствуются без копирования каждого сегмента. Префиксы raw-маркера и тела замен, не требующие преобразования, переносятся непосредственно в результат; verbatim-замены рендерятся в один буфер без промежуточной строки с преобразованными EOL. Точные выходные байты и полная семантическая проверка сохранены; короткие парные benchmarks отдельно измеряют text- и segments-пути. +- Строка диагностики находится обратным поиском терминатора без повторного прохода по префиксу документа. JSON-строки проверяются безопасными восьмибайтовыми операциями; UTF-8 без escapes переносится целиком, экранируемый ASCII обрабатывается специализированно, Unicode encoder сохраняется. Дочерние буферы и хранилища seen-ключей reopen-merge выделяются лениво. Точные байты и координаты сохранены; добавлены регрессии аллокаций/экранирования и парные runtime benchmarks с единым внешним лимитом 15 секунд на весь batch процессов. ## [0.8.1] — 2026-10-21 diff --git a/CHANGELOG.zh.md b/CHANGELOG.zh.md index 211edb7..2eb54d7 100644 --- a/CHANGELOG.zh.md +++ b/CHANGELOG.zh.md @@ -28,6 +28,7 @@ - 将嵌套行内替换直接写入同一缓冲区,避免临时字符向量及文本路径段的二次复制。保留语义验证和原始字节保护。新增短时配对基准,编译独立进行,每个进程由外部工具限制为 15 秒。 - 位置扫描与解析器共用结构分类,不再构造无用值或多行正文。验证目标和未修改的树时不再克隆替换值;以借用方式渲染块容器,避免临时根包装。保留验证、writer 错误路径和逐字节输出,并增加专项回归测试及配对基准场景。 - 借用字面路径段及不含转义的文本路径段,不再逐段复制。将 raw 标记前缀及无需转换的替换正文直接写入结果;verbatim 替换使用单一缓冲区,不再生成转换 EOL 的中间字符串。保留精确输出与完整语义验证;短时配对基准分别测量文本路径和路径段 API。 +- 通过反向查找换行符定位诊断行,不再重复扫描文档前缀。用安全的八字节运算检查 JSON 字符串,整体复制无需转义的 UTF-8,专门处理需转义的 ASCII,并保留 Unicode encoder。按需分配 reopen-merge 子缓冲区和 seen 键存储。保留精确字节与坐标;增加分配及转义回归测试,并提供配对 runtime 基准,整个进程 batch 共用一个外部 15 秒限制。 ## [0.8.1] — 2026-10-21 diff --git a/Cargo.toml b/Cargo.toml index 97d17de..9d47906 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,3 +107,7 @@ harness = false [[bench]] name = "source_edit" harness = false + +[[bench]] +name = "runtime_paths" +harness = false diff --git a/benches/runtime_paths.rs b/benches/runtime_paths.rs new file mode 100644 index 0000000..8e8c230 --- /dev/null +++ b/benches/runtime_paths.rs @@ -0,0 +1,157 @@ +//! Парный прогон; общий внешний caps 15 охватывает все процессы. +use std::hint::black_box; +use std::time::{Duration, Instant}; + +use ktav::{Error, ErrorEnvelope, ErrorKind, Span}; +use serde_json::Value as Json; + +const SAMPLE: Duration = Duration::from_millis(50); +const WARMUP: Duration = Duration::from_millis(5); + +enum Work { + Line { + source: String, + error: Error, + line: u32, + }, + Json { + envelope: ErrorEnvelope, + expected: String, + }, + Parse { + source: String, + expected: Json, + }, +} + +struct Case { + name: &'static str, + work: Work, +} + +impl Case { + fn validate(&self) { + match &self.work { + Work::Line { + source, + error, + line, + } => { + let envelope = ErrorEnvelope::from_error(error, source); + assert_eq!(envelope.line, Some(*line)); + assert_eq!(envelope.line_text.as_deref(), Some(": bad")); + assert_eq!(envelope.span, error.span()); + } + Work::Json { envelope, expected } => assert_eq!(&envelope.to_json(), expected), + Work::Parse { source, expected } => { + assert_eq!(&ktav::from_str::(source).unwrap(), expected); + } + } + } + + fn run(&self) { + match &self.work { + Work::Line { source, error, .. } => { + drop(black_box(ErrorEnvelope::from_error( + black_box(error), + black_box(source), + ))); + } + Work::Json { envelope, .. } => { + drop(black_box(black_box(envelope).to_json())); + } + Work::Parse { source, .. } => { + drop(black_box( + ktav::from_str::(black_box(source)).unwrap(), + )); + } + } + } +} + +fn line_case(name: &'static str, tail: bool) -> Case { + let mut prefix = String::from("anchor: ok\n"); + if tail { + prefix.push_str(&"## note\n".repeat(200_000)); + } + let start = u32::try_from(prefix.len()).unwrap(); + let line = if tail { 200_002 } else { 2 }; + let source = format!("{prefix}: bad\n"); + let error = Error::Structured(ErrorKind::EmptyKey { + line, + span: Span::new(start, start + 1), + }); + Case { + name, + work: Work::Line { + source, + error, + line, + }, + } +} + +fn json_case(name: &'static str, payload: String) -> Case { + let quoted = serde_json::to_string(&payload).unwrap(); + let expected = format!( + "{{\"error\":\"Message\",\"reason\":null,\"line\":null,\"line_text\":null,\"span\":null,\"path\":null,\"body\":{quoted},\"canonical\":null,\"spec_section\":null,\"message\":{quoted}}}" + ); + let envelope = ErrorEnvelope::from_error(&Error::Message(payload), ""); + Case { + name, + work: Work::Json { envelope, expected }, + } +} + +fn parse_case(name: &'static str, reopen: bool) -> Case { + let count = 2048; + let mut source = String::from("rows: [\n"); + for _ in 0..count { + source.push_str("[1]\n"); + } + source.push_str("]\n"); + if reopen { + source.push_str("a: {x: 1}\nb: 0\na.y: 2\n"); + } else { + source.push_str("a: {x: 1, y: 2}\nb: 0\n"); + } + let expected = serde_json::json!({"rows": vec![vec![1]; count], "a": {"x": 1, "y": 2}, "b": 0}); + Case { + name, + work: Work::Parse { source, expected }, + } +} + +fn main() { + let label = std::env::args().nth(1).unwrap_or_default(); + assert!(matches!(label.as_str(), "baseline" | "candidate")); + let cases = [ + line_case("line_tail_1600k", true), + line_case("line_head_control", false), + json_case("json_plain_16k", "ordinary text 123".repeat(1024)), + json_case("json_unicode_16k", "שלום café 中文 😀".repeat(512)), + json_case("json_escaped_control", "\"\\\n\r\t\u{0001}".repeat(2048)), + parse_case("one_reopen_tiny_arrays", true), + parse_case("no_reopen_control", false), + ]; + println!("label,case,iterations,elapsed_ns,ns_per_op"); + for case in &cases { + case.validate(); + let warmup = Instant::now(); + while warmup.elapsed() < WARMUP { + case.run(); + } + let start = Instant::now(); + let mut iterations = 0u64; + while start.elapsed() < SAMPLE { + case.run(); + iterations += 1; + } + let elapsed = start.elapsed().as_nanos(); + println!( + "{label},{},{iterations},{elapsed},{:.1}", + case.name, + elapsed as f64 / iterations as f64 + ); + } +} diff --git a/root-docs/CHANGELOG/unreleased/body-1.md b/root-docs/CHANGELOG/unreleased/body-1.md index d8c578f..60bbb64 100644 --- a/root-docs/CHANGELOG/unreleased/body-1.md +++ b/root-docs/CHANGELOG/unreleased/body-1.md @@ -17,6 +17,7 @@ - Write nested inline replacements into one buffer; avoid temporary character vectors and a second copy of parsed textual path segments. Keep semantic verification and byte-preserving behavior. Add short paired benchmarks with separate compilation and an external 15-second process limit. - The positional pass shares structural classifiers with the parser without constructing unused values or multiline bodies. Verify the edited target and untouched tree without cloning the replacement; render borrowed block compounds without a temporary root wrapper. Preserve validation, writer error paths and byte-for-byte output, with focused regression tests and expanded paired benchmarks. - Borrow literal path slices and unescaped textual segments instead of copying each segment. Assemble raw-marker prefixes and unchanged replacement bodies directly into the result; render verbatim replacements in one buffer without an intermediate EOL-converted string. Preserve exact output and full semantic verification; measure text and segment paths separately in short paired benchmarks. +- Locate diagnostic line text by reverse terminator search instead of rescanning the document prefix. Probe JSON strings with safe eight-byte operations, bulk-copy unescaped UTF-8, specialize escaped ASCII and retain the Unicode encoder. Allocate reopen-merge child buffers and seen-key storage lazily. Preserve exact bytes and locations; add allocation/escaping regressions and paired runtime benchmarks with one external 15-second limit for the entire process batch. >>>>> lang=ru ## [Не выпущено] @@ -37,6 +38,7 @@ - Вложенные inline-замены пишутся в один буфер, без временного вектора символов и повторного копирования разобранных сегментов текстового пути. Семантическая проверка и сохранение исходных байтов остаются. Добавлены короткие парные benchmarks с отдельной компиляцией и внешним лимитом 15 секунд на процесс. - Позиционный проход использует общие с parser структурные классификаторы без построения ненужных значений и multiline-тел. Проверка цели и неизменной части дерева обходится без клонирования replacement; блочные контейнеры рендерятся по заимствованию без временной корневой обёртки. Валидация, пути writer-ошибок и точные выходные байты сохранены; добавлены регрессии и расширены парные benchmarks. - Буквальные сегменты пути и сегменты текстового пути без escape заимствуются без копирования каждого сегмента. Префиксы raw-маркера и тела замен, не требующие преобразования, переносятся непосредственно в результат; verbatim-замены рендерятся в один буфер без промежуточной строки с преобразованными EOL. Точные выходные байты и полная семантическая проверка сохранены; короткие парные benchmarks отдельно измеряют text- и segments-пути. +- Строка диагностики находится обратным поиском терминатора без повторного прохода по префиксу документа. JSON-строки проверяются безопасными восьмибайтовыми операциями; UTF-8 без escapes переносится целиком, экранируемый ASCII обрабатывается специализированно, Unicode encoder сохраняется. Дочерние буферы и хранилища seen-ключей reopen-merge выделяются лениво. Точные байты и координаты сохранены; добавлены регрессии аллокаций/экранирования и парные runtime benchmarks с единым внешним лимитом 15 секунд на весь batch процессов. >>>>> lang=zh ## [尚未发布] @@ -57,4 +59,5 @@ - 将嵌套行内替换直接写入同一缓冲区,避免临时字符向量及文本路径段的二次复制。保留语义验证和原始字节保护。新增短时配对基准,编译独立进行,每个进程由外部工具限制为 15 秒。 - 位置扫描与解析器共用结构分类,不再构造无用值或多行正文。验证目标和未修改的树时不再克隆替换值;以借用方式渲染块容器,避免临时根包装。保留验证、writer 错误路径和逐字节输出,并增加专项回归测试及配对基准场景。 - 借用字面路径段及不含转义的文本路径段,不再逐段复制。将 raw 标记前缀及无需转换的替换正文直接写入结果;verbatim 替换使用单一缓冲区,不再生成转换 EOL 的中间字符串。保留精确输出与完整语义验证;短时配对基准分别测量文本路径和路径段 API。 +- 通过反向查找换行符定位诊断行,不再重复扫描文档前缀。用安全的八字节运算检查 JSON 字符串,整体复制无需转义的 UTF-8,专门处理需转义的 ASCII,并保留 Unicode encoder。按需分配 reopen-merge 子缓冲区和 seen 键存储。保留精确字节与坐标;增加分配及转义回归测试,并提供配对 runtime 基准,整个进程 batch 共用一个外部 15 秒限制。 diff --git a/src/error/envelope.rs b/src/error/envelope.rs index 584f8fd..892f2c6 100644 --- a/src/error/envelope.rs +++ b/src/error/envelope.rs @@ -288,8 +288,14 @@ fn line_text_at(source: &str, pos: usize) -> Option { return None; } let bytes = source.as_bytes(); - let (_, column) = crate::error::source::line_col(bytes, pos); - let line_start = pos - column as usize; + // Позиция на LF внутри CRLF относится к строке перед всей парой. + let search_end = if pos > 0 && bytes[pos - 1] == b'\r' && bytes.get(pos) == Some(&b'\n') { + pos - 1 + } else { + pos + }; + let line_start = + memchr::memrchr2(b'\r', b'\n', &bytes[..search_end]).map_or(0, |offset| offset + 1); let line_end = memchr::memchr2(b'\r', b'\n', &bytes[line_start..]) .map_or(source.len(), |offset| line_start + offset); Some(source[line_start..line_end].to_string()) @@ -390,12 +396,73 @@ fn push_json_opt_string(out: &mut String, value: Option<&str>) { } } -/// Push `s` as a `"…"`-quoted JSON string. RFC 8259 escaping: -/// control characters as `\u00xx` with lowercase hex digits (matching -/// serde_json's convention); DEL, non-ASCII and non-BMP characters -/// pass through as raw UTF-8, which is valid JSON — a Rust `str` -/// cannot hold lone surrogates, so no surrogate pairing is needed. +fn requires_json_escape(bytes: &[u8]) -> bool { + const LOW: u64 = 0x0101_0101_0101_0101; + const HIGH: u64 = 0x8080_8080_8080_8080; + let mut chunks = bytes.chunks_exact(8); + for chunk in &mut chunks { + let word = u64::from_ne_bytes(chunk.try_into().expect("chunks_exact yields eight bytes")); + let quote = word ^ 0x2222_2222_2222_2222; + let slash = word ^ 0x5c5c_5c5c_5c5c_5c5c; + // Wrapping намеренный: packed-byte predicates дают только any-match. + let controls = word.wrapping_sub(0x2020_2020_2020_2020) & !word; + let quotes = quote.wrapping_sub(LOW) & !quote; + let slashes = slash.wrapping_sub(LOW) & !slash; + if (controls | quotes | slashes) & HIGH != 0 { + return true; + } + } + chunks + .remainder() + .iter() + .any(|&byte| byte < 0x20 || byte == b'"' || byte == b'\\') +} + +/// RFC 8259: строка без escapes переносится целиком, остальные — прежним encoder. fn push_json_string(out: &mut String, s: &str) { + if requires_json_escape(s.as_bytes()) { + if s.is_ascii() { + push_json_ascii_escaped(out, s); + } else { + push_json_escaped(out, s); + } + } else { + out.push('"'); + out.push_str(s); + out.push('"'); + } +} + +// Вызывается только после проверки is_ascii(). +#[inline(never)] +fn push_json_ascii_escaped(out: &mut String, s: &str) { + debug_assert!(s.is_ascii()); + out.push('"'); + for &byte in s.as_bytes() { + match byte { + b'"' => out.push_str("\\\""), + b'\\' => out.push_str("\\\\"), + b'\n' => out.push_str("\\n"), + b'\r' => out.push_str("\\r"), + b'\t' => out.push_str("\\t"), + 8 => out.push_str("\\b"), + 12 => out.push_str("\\f"), + byte if byte < 0x20 => { + const HEX: &[u8; 16] = b"0123456789abcdef"; + out.push_str("\\u00"); + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0xf) as usize] as char); + } + byte if byte.is_ascii() => out.push(char::from(byte)), + _ => unreachable!("ASCII input was checked before encoding"), + } + } + out.push('"'); +} + +// Сохранять escape-цикл отдельно от probe. +#[inline(never)] +fn push_json_escaped(out: &mut String, s: &str) { out.push('"'); for c in s.chars() { match c { @@ -418,3 +485,48 @@ fn push_json_string(out: &mut String, s: &str) { } out.push('"'); } + +#[cfg(test)] +mod escape_probe_tests { + use super::requires_json_escape; + + fn scalar(bytes: &[u8]) -> bool { + bytes + .iter() + .any(|&byte| byte < 32 || matches!(byte, b'"' | b'\\')) + } + + #[test] + fn packed_probe_matches_every_byte_in_every_lane() { + for fill in [b'a', b' ', 0xff] { + for position in 0..8 { + for byte in 0..=255 { + let mut bytes = [fill; 8]; + bytes[position] = byte; + assert_eq!( + requires_json_escape(&bytes), + scalar(&bytes), + "fill={fill}, lane={position}, byte={byte}" + ); + } + } + } + } + + #[test] + fn packed_probe_matches_scalar_across_tail_lengths() { + for length in 0..=24 { + for byte in 0..=255 { + let mut bytes = vec![b'a'; length]; + if let Some(last) = bytes.last_mut() { + *last = byte; + } + assert_eq!( + requires_json_escape(&bytes), + scalar(&bytes), + "length={length}, byte={byte}" + ); + } + } + } +} diff --git a/src/serde_bridge/thin/merge/reopen.rs b/src/serde_bridge/thin/merge/reopen.rs index efef9c2..a3f3608 100644 --- a/src/serde_bridge/thin/merge/reopen.rs +++ b/src/serde_bridge/thin/merge/reopen.rs @@ -36,7 +36,7 @@ fn is_open(ev: &Event<'_>) -> bool { pub(crate) fn merge_reopened<'a>(src: &EventStream<'a>, bump: &'a Bump) -> EventStream<'a> { let mut bufs: Vec> = vec![Buf { items: BumpVec::with_capacity_in(src.len(), bump), - seen: BumpVec::with_capacity_in(8, bump), + seen: BumpVec::new_in(bump), index: None, }]; // Index 0 is the document root, pre-created above. `root_open` @@ -66,7 +66,7 @@ pub(crate) fn merge_reopened<'a>(src: &EventStream<'a>, bump: &'a Bump) -> Event // entry and the `Key` item. ev @ (Event::BeginObject | Event::BeginArray) => { let idx = bufs.len(); - bufs.push(new_buf(bump, src.len())); + bufs.push(new_buf(bump)); bufs[stack_top(&stack)] .items .push(Item::Obj { buf: idx, open: ev }); @@ -83,7 +83,7 @@ pub(crate) fn merge_reopened<'a>(src: &EventStream<'a>, bump: &'a Bump) -> Event // rejected as duplicates by the parser (§ 6.3), // so this is always fresh. No `seen` entry. let idx = bufs.len(); - bufs.push(new_buf(bump, src.len())); + bufs.push(new_buf(bump)); bufs[stack_top(&stack)].items.push(Item::Ev(Event::Key(k))); bufs[stack_top(&stack)] .items @@ -108,7 +108,7 @@ pub(crate) fn merge_reopened<'a>(src: &EventStream<'a>, bump: &'a Bump) -> Event } None => { let idx = bufs.len(); - bufs.push(new_buf(bump, src.len())); + bufs.push(new_buf(bump)); bufs[top].seen_insert(bump, k, idx); bufs[top].items.push(Item::Ev(Event::Key(k))); bufs[top].items.push(Item::Obj { buf: idx, open }); @@ -169,10 +169,21 @@ pub(crate) fn merge_reopened<'a>(src: &EventStream<'a>, bump: &'a Bump) -> Event out } -fn new_buf<'a>(bump: &'a Bump, hint: usize) -> Buf<'a> { +#[cfg(test)] +#[test] +fn empty_child_buffer_has_no_arena_allocation() { + let bump = Bump::new(); + let before = bump.allocated_bytes(); + let buffer = new_buf(&bump); + assert_eq!(bump.allocated_bytes(), before); + assert!(buffer.items.is_empty()); + assert!(buffer.seen.is_empty()); +} + +fn new_buf<'a>(bump: &'a Bump) -> Buf<'a> { Buf { - items: BumpVec::with_capacity_in(hint.min(64), bump), - seen: BumpVec::with_capacity_in(8, bump), + items: BumpVec::new_in(bump), + seen: BumpVec::new_in(bump), index: None, } } diff --git a/src/serde_bridge/thin/merge/table.rs b/src/serde_bridge/thin/merge/table.rs index f727cf8..f60f9a0 100644 --- a/src/serde_bridge/thin/merge/table.rs +++ b/src/serde_bridge/thin/merge/table.rs @@ -27,11 +27,8 @@ pub(super) struct Buf<'a> { pub(super) index: Option>, } -/// Buffers at or below this many keyed children never build the hash -/// index: the linear scan over `seen` (whose preallocation is exactly -/// 8 slots) is cheaper than hashing, the worst case is a bounded 28 -/// comparisons for all misses, and small buffers keep exactly the -/// pre-index behavior. +/// До восьми Object-ключей используется линейный поиск без hash index. +/// `seen` выделяется лениво; число сравнений для всех промахов ≤ 28. const LINEAR_MAX: usize = 8; /// Sentinel `pos` of an empty index slot. Kept on `pos` (not `h`) so a diff --git a/tests/suite/errors/envelope.rs b/tests/suite/errors/envelope.rs index 23719a5..4c4ebbc 100644 --- a/tests/suite/errors/envelope.rs +++ b/tests/suite/errors/envelope.rs @@ -13,6 +13,8 @@ mod helpers; mod message; #[path = "envelope/paths.rs"] mod paths; +#[path = "envelope/performance_regressions.rs"] +mod performance_regressions; #[path = "envelope/shape.rs"] mod shape; #[path = "envelope/spans.rs"] diff --git a/tests/suite/errors/envelope/performance_regressions.rs b/tests/suite/errors/envelope/performance_regressions.rs new file mode 100644 index 0000000..ad0d9fb --- /dev/null +++ b/tests/suite/errors/envelope/performance_regressions.rs @@ -0,0 +1,78 @@ +use ktav::{Error, ErrorEnvelope, ErrorKind, Span}; + +#[test] +fn line_text_and_columns_preserve_terminator_boundaries() { + let source = "a\r\nb\rc\nd"; + let cases = [ + (0, 1, 0, "a"), + (1, 1, 1, "a"), + (2, 1, 2, "a"), + (3, 2, 0, "b"), + (4, 2, 1, "b"), + (5, 3, 0, "c"), + (6, 3, 1, "c"), + (7, 4, 0, "d"), + (8, 4, 1, "d"), + ]; + for (offset, line, column, text) in cases { + let span = Span::new(offset, offset); + let error = Error::Structured(ErrorKind::EmptyKey { line, span }); + let envelope = ErrorEnvelope::from_error(&error, source); + assert_eq!(envelope.line_text.as_deref(), Some(text), "offset {offset}"); + assert_eq!(span.line_col(source), (line, column), "offset {offset}"); + } +} + +#[test] +fn line_text_preserves_bom_utf8_and_defensive_span_handling() { + let source = "\u{FEFF}שלום\r\nx"; + let error = Error::Structured(ErrorKind::EmptyKey { + line: 2, + span: Span::new(13, 14), + }); + let envelope = ErrorEnvelope::from_error(&error, source); + assert_eq!(envelope.line_text.as_deref(), Some("x")); + assert_eq!(Span::new(13, 14).line_col(source), (2, 0)); + + let inside_char = Error::Structured(ErrorKind::EmptyKey { + line: 1, + span: Span::new(4, 4), + }); + assert!(ErrorEnvelope::from_error(&inside_char, source) + .line_text + .is_none()); + let beyond_eof = Error::Structured(ErrorKind::EmptyKey { + line: 2, + span: Span::new(u32::MAX, u32::MAX), + }); + assert_eq!( + ErrorEnvelope::from_error(&beyond_eof, source) + .line_text + .as_deref(), + Some("x") + ); + assert!(ErrorEnvelope::from_error(&error, "").line_text.is_none()); +} + +#[test] +fn json_encoding_matches_independent_escaping_oracle() { + let controls: String = (0u8..=31).map(char::from).collect(); + let ascii: String = (0u8..=127).map(char::from).collect(); + let payloads = [ + String::new(), + ascii, + "ordinary text 123".repeat(2048), + "café שלום 中文 😀".repeat(256), + format!("{controls}\"\\"), + "safe\0שלום\"\\😀".repeat(256), + format!("{}\"שלום\\\u{1}{}", "a".repeat(4096), "z".repeat(4096)), + ]; + for payload in payloads { + let quoted = serde_json::to_string(&payload).unwrap(); + let expected = format!( + "{{\"error\":\"Message\",\"reason\":null,\"line\":null,\"line_text\":null,\"span\":null,\"path\":null,\"body\":{quoted},\"canonical\":null,\"spec_section\":null,\"message\":{quoted}}}" + ); + let actual = ErrorEnvelope::from_error(&Error::Message(payload), "").to_json(); + assert_eq!(actual, expected); + } +}