From f6aa9f2bb35df69d31cf8eb13ca1bcdad71d8865 Mon Sep 17 00:00:00 2001 From: Marat K Date: Sun, 25 Oct 2026 17:40:44 +0100 Subject: [PATCH 1/7] =?UTF-8?q?feat(edit):=20source-edit=20=D0=B8=20=D0=B4?= =?UTF-8?q?=D0=B8=D0=B0=D0=B3=D0=BD=D0=BE=D1=81=D1=82=D0=B8=D0=BA=D0=B0=20?= =?UTF-8?q?0.9=20=D0=B2=20=D0=BE=D0=B1=D0=BE=D0=B8=D1=85=20backend?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Добавить native PyO3 и самостоятельный pure API редактирования по текстовому пути и сегментам. Сохранять исходную форму, проверять результат сплайса и диагностические координаты по corpus 0.9. Закрепить проверенный Rust core, обновить регрессии и документацию EN/RU/ZH без повышения версии пакета. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 3 + .github/workflows/CI.yml | 37 +- CHANGELOG.md | 21 + Cargo.toml | 3 +- README.md | 37 ++ docs/ru/CHANGELOG.ru.md | 22 + docs/ru/README.ru.md | 37 ++ docs/zh/CHANGELOG.zh.md | 18 + docs/zh/README.zh.md | 32 + python/ktav/__init__.py | 134 +++- python/ktav/_core.pyi | 32 + python/ktav/_pure/__init__.py | 75 ++- python/ktav/_pure/_edit.py | 37 ++ python/ktav/_pure/_edit_common.py | 294 +++++++++ python/ktav/_pure/_edit_encode.py | 308 +++++++++ python/ktav/_pure/_edit_layout.py | 278 ++++++++ python/ktav/_pure/_edit_splice.py | 202 ++++++ python/ktav/_pure/_edit_tree.py | 614 ++++++++++++++++++ python/ktav/_pure/errors.py | 29 + python/ktav/_pure/parser.py | 70 +- python/ktav/pure.py | 12 + .../CHANGELOG/unreleased-added/body-3.md | 64 ++ root-docs/CHANGELOG/unreleased-added/meta.js | 2 +- root-docs/README/manifest.js | 2 + root-docs/README/source-edit-status/body-1.md | 9 + root-docs/README/source-edit-status/meta.js | 7 + root-docs/README/source-edit/body-1.md | 103 +++ root-docs/README/source-edit/meta.js | 7 + spec | 2 +- src/lib.rs | 129 ++++ tests/api/test_error_envelope.py | 9 +- tests/conformance_runner.py | 125 ++-- tests/edit/__init__.py | 0 tests/edit/_backends.py | 21 + tests/edit/test_edit_layout.py | 241 +++++++ tests/edit/test_edit_value.py | 440 +++++++++++++ tests/edit/test_native_pin.py | 46 ++ tests/pure/_no_native_probe.py | 11 +- tests/pure/test_pure_api.py | 11 +- tests/serialize/test_format.py | 7 +- tests/test_conformance.py | 6 + 41 files changed, 3445 insertions(+), 92 deletions(-) create mode 100644 python/ktav/_pure/_edit.py create mode 100644 python/ktav/_pure/_edit_common.py create mode 100644 python/ktav/_pure/_edit_encode.py create mode 100644 python/ktav/_pure/_edit_layout.py create mode 100644 python/ktav/_pure/_edit_splice.py create mode 100644 python/ktav/_pure/_edit_tree.py create mode 100644 root-docs/CHANGELOG/unreleased-added/body-3.md create mode 100644 root-docs/README/source-edit-status/body-1.md create mode 100644 root-docs/README/source-edit-status/meta.js create mode 100644 root-docs/README/source-edit/body-1.md create mode 100644 root-docs/README/source-edit/meta.js create mode 100644 tests/edit/__init__.py create mode 100644 tests/edit/_backends.py create mode 100644 tests/edit/test_edit_layout.py create mode 100644 tests/edit/test_edit_value.py create mode 100644 tests/edit/test_native_pin.py diff --git a/.gitattributes b/.gitattributes index 55da72c..a5d53bd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,6 @@ # on all platforms check out with LF — no more CRLF/LF noise in # `git status` from Windows clients. * text=auto eol=lf + +# Polydoc требует пустую строку в конце промежуточных source parts. +root-docs/**/body-*.md whitespace=-blank-at-eof diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8a617e9..fa06707 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -116,10 +116,41 @@ jobs: - name: Install test tooling run: python -m pip install --upgrade pip maturin pytest pytest-cov "tomli>=2.5.0; python_version < '3.11'" # Lock + provenance check before the native build compiles anything. + # The core is an IMMUTABLE git pin (Cargo.toml rev 447a00bd...); + # the verification step below FAILS CLOSED on rev/version drift — + # it never skips unconditionally. - name: Generate Cargo.lock (no --locked) - run: cargo generate-lockfile && cargo update -p ktav --precise 0.8.1 - - name: Check Rust provenance - run: python scripts/check_rust_provenance.py + run: cargo generate-lockfile + - name: Verify ktav core pin (rev/version, fail closed) + shell: python + run: | + import re, sys + from pathlib import Path + + PIN = "447a00bd012e8d3f347c7e1eb59c3f0d9a78332f" + toml = Path("Cargo.toml").read_text(encoding="utf-8") + m = re.search(r'rev\s*=\s*"([0-9a-f]{40})"', toml) + if m is None: + sys.exit("Cargo.toml: ktav dependency has no git rev pin") + if m.group(1) != PIN: + sys.exit(f"Cargo.toml rev {m.group(1)} != immutable pin {PIN}") + lock = Path("Cargo.lock").read_text(encoding="utf-8") + packages = lock.split("[[package]]") + core = [p for p in packages if re.search(r'^name = "ktav"$', p, re.M)] + if len(core) != 1: + sys.exit("Cargo.lock: expected exactly one ktav package") + entry = core[0] + sm = re.search(r'^source = "git\+https://github\.com/ktav-lang/rust\?rev=' + PIN + r'#([0-9a-f]{40})"$', entry, re.M) + if sm is None: + sys.exit("Cargo.lock: ktav is not locked to the pinned git source") + if sm.group(1) != PIN: + sys.exit(f"Cargo.lock rev {sm.group(1)} != immutable pin {PIN}") + vm = re.search(r'version = "([^"]+)"', entry) + if vm is None: + sys.exit("Cargo.lock: ktav entry has no version") + if vm.group(1) != "0.9.0": + sys.exit(f"Cargo.lock: expected ktav core 0.9.0, got {vm.group(1)}") + print(f"ktav core pinned: rev {PIN}, version {vm.group(1)}") - name: Build extension (release) run: maturin develop --release --locked - name: Pytest (both backends; conftest enforces the native scenario) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0559130..8dcd0e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,27 @@ release: a complete pure-Python parser and generator, available as native platform wheels. It installs with no Rust toolchain, compiler or maturin and provides exactly the `ktav.pure` backend. +- `ktav.edit_value(source, path, replacement)` and + `ktav.edit_value_segments(source, segments, replacement)` — lossless + in-place source edits (source-edit 0.9). Only the bytes of the target + value (plus inline wrappers the layout rules force to unfold) change; + comments, key spelling, blank lines, BOM and CR/LF/CRLF survive, and + an equal replacement is a byte-identical no-op. New exception family + `KtavEditError` with `KtavInvalidPathError` (`body` = decimal UTF-8 + byte offset into the path), `KtavNotFoundError`, and + `KtavUnsupportedError`; the same ten-field envelope, `error` is one + of `EditInvalidPath` / `EditNotFound` / `EditUnsupported`. +- Spec 0.9 corpus: the conformance runner now executes the + `versions/0.9/tests` fixtures (162 invalid — 72 new `diagnostics09` + EOF/malformed-UTF-8 diagnostics), validates `expected_location` + metadata (line, raw-byte `start`, exact `end` or `minimum_end`), and + feeds every `raw_bytes`-flagged fixture through the shared byte + boundary, so invalid UTF-8 raises the structured `InvalidUtf8` + envelope in both backends. +- `UnclosedCompound` errors now pin the opener: `line` and `line_text` + name the line that opened the innermost unclosed compound, and + `span` runs from that opener to end of input. + ### Changed - Upgraded to Rust core 0.8.1 and spec 0.8.1 (published as the spec diff --git a/Cargo.toml b/Cargo.toml index e76d6a9..9c53ba2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,8 @@ name = "_core" crate-type = ["cdylib"] [dependencies] -ktav = ">=0.8.1, <0.9" +# Подготовка 0.9: immutable core pin с source-edit; пакет ещё не опубликован. +ktav = { git = "https://github.com/ktav-lang/rust", rev = "447a00bd012e8d3f347c7e1eb59c3f0d9a78332f" } pyo3 = { version = "0.28", features = ["abi3-py39", "extension-module", "indexmap", "generate-import-lib"] } indexmap = "2" rustc-hash = "2" diff --git a/README.md b/README.md index e100d7b..559ccdf 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,43 @@ Pick by what you are holding: `emit_canonical` when you have an object, built in between, `format` when you want the document's own comments and grouping to survive. +> **Unreleased (0.9 preparation):** These editing APIs and updated diagnostics are not included in published 0.8.1 packages. Native backends require the pinned 0.9 core (C ABI 2 for C-FFI); pure backends remain standalone. + +### Source edits — `edit_value` / `edit_value_segments` + +```python +import ktav + +out = ktav.edit_value("port: 8080\nhost: db\n", "port", 9090) +# "port: 9090\nhost: db\n" + +out = ktav.edit_value_segments("a: {x: 1, y: 2}\n", ["a", "y"], {"n": [1]}) +# "a: {x: 1, y: {n: [1]}}\n" +``` + +`edit_value` takes a textual path (`key1.key2.0.key4`): inside a +segment `\.` is a literal dot and `\\` a literal backslash; any other +escape raises `KtavInvalidPathError` whose `body` is the decimal UTF-8 +byte offset of the problem. `edit_value_segments` takes +already-decoded literal segments with no extra unescaping — a numeric +segment is a string key inside an Object and a zero-based index inside +an Array. Only replacement of an existing value is supported; missing +keys, out-of-bounds indexes and descent through a scalar raise +`KtavNotFoundError`. + +The edit is a source splice, not a parse→mutate→serialize of the whole +document: comments, key spelling, order, whitespace, BOM and +CR/LF/CRLF outside the effective change are preserved byte-for-byte, +dotted/inline/block forms are kept, and an equal replacement is a +byte-identical no-op — even for noncanonical spelling. Every +successful patch is re-parsed and semantically verified. When no +lossless patch exists (noncontiguous dotted/reopened targets, internal +comments under structural replacement, a `))` line inside a multiline +replacement) the call raises `KtavUnsupportedError` instead of +guessing. Long or multiline replacements unfold only the touched +inline wrappers; untouched nested branches stay inline. Both the +native and the pure backend implement the identical semantics. + Unlike the JavaScript bindings, `canonical_from_source` buys you no extra numeric fidelity here — Python distinguishes `int` from `float`, so the Ktav Integer/Float distinction survives `loads`, and diff --git a/docs/ru/CHANGELOG.ru.md b/docs/ru/CHANGELOG.ru.md index 8ca0b49..17da52d 100644 --- a/docs/ru/CHANGELOG.ru.md +++ b/docs/ru/CHANGELOG.ru.md @@ -66,6 +66,28 @@ нативными платформенными wheel-ами. Он ставится без Rust toolchain, компилятора и maturin и предоставляет ровно backend `ktav.pure`. +- `ktav.edit_value(source, path, replacement)` и + `ktav.edit_value_segments(source, segments, replacement)` — + lossless-правка исходника по месту (source-edit 0.9). Меняются только + байты целевого значения (плюс inline-обёртки, которые требует + раскрыть layout-правило); комментарии, написание ключей, пустые + строки, BOM и CR/LF/CRLF сохраняются, а совпадающая замена — + byte-identical no-op. Новое семейство исключений `KtavEditError` с + `KtavInvalidPathError` (`body` = десятичный UTF-8 byte offset в + путь), `KtavNotFoundError` и `KtavUnsupportedError`; тот же + десятипольный конверт, `error` — один из `EditInvalidPath` / + `EditNotFound` / `EditUnsupported`. +- Корпус spec 0.9: conformance runner исполняет fixtures из + `versions/0.9/tests` (162 invalid — 72 новых диагностики + `diagnostics09` EOF/битый UTF-8), проверяет метаданные + `expected_location` (line, raw-byte `start`, точный `end` или + `minimum_end`) и прогоняет все `raw_bytes`-fixture через общий + byte-boundary, так что битый UTF-8 даёт структурированный конверт + `InvalidUtf8` в обоих backend-ах. +- Ошибки `UnclosedCompound` теперь указывают открыватель: `line` и + `line_text` называют строку, открывшую ближайший незакрытый + контейнер, а `span` тянется от него до конца ввода. + ### Changed - Обновление до Rust core 0.8.1 и spec 0.8.1 (опубликованы как каталог diff --git a/docs/ru/README.ru.md b/docs/ru/README.ru.md index 270f1ff..2608039 100644 --- a/docs/ru/README.ru.md +++ b/docs/ru/README.ru.md @@ -175,6 +175,43 @@ print(ktav.format(open("config.ktav").read())) значение Python, `format` — когда нужно сохранить собственные комментарии и группировку документа. +> **Ещё не выпущено (подготовка 0.9):** Эти API редактирования и обновлённая диагностика не входят в опубликованные пакеты 0.8.1. Native-backend требует закреплённое ядро 0.9 (C ABI 2 для C-FFI); pure-backend остаётся самостоятельным. + +### Правка исходника — `edit_value` / `edit_value_segments` + +```python +import ktav + +out = ktav.edit_value("port: 8080\nhost: db\n", "port", 9090) +# "port: 9090\nhost: db\n" + +out = ktav.edit_value_segments("a: {x: 1, y: 2}\n", ["a", "y"], {"n": [1]}) +# "a: {x: 1, y: {n: [1]}}\n" +``` + +`edit_value` принимает текстовый путь (`key1.key2.0.key4`): внутри +сегмента `\.` — literal точка, `\\` — literal backslash; любой другой +escape бросает `KtavInvalidPathError`, у которого `body` — десятичный +UTF-8 byte offset проблемного места. `edit_value_segments` принимает +уже декодированные literal-сегменты без повторного unescape — числовой +сегмент это строковый ключ в Object и zero-based индекс в Array. +Поддерживается только замена существующего значения: отсутствующий +ключ, выход за границы массива и спуск сквозь скаляр дают +`KtavNotFoundError`. + +Правка — это splice исходника, а не parse→mutate→serialize всего +документа: комментарии, написание ключей, порядок, пробелы, BOM и +CR/LF/CRLF вне эффективного изменения сохраняются побайтово, +dotted/inline/block-формы держатся, а совпадающая замена — +byte-identical no-op, даже при неканоничном написании. Каждый +успешный патч перепарсится и семантически проверяется. Когда lossless +патча не существует (noncontiguous dotted/reopen-цели, внутренние +комментарии при структурной замене, строка `))` внутри multiline +замены), вызов бросает `KtavUnsupportedError`, а не угадывает. Длинные +и многострочные замены раскрывают только затронутые inline-обёртки; +нетронутые ветки остаются inline. Native и pure backend реализуют +идентичную семантику. + В отличие от биндингов для JavaScript, `canonical_from_source` не даёт здесь дополнительной численной точности: Python различает `int` и `float`, поэтому различие Integer/Float из Ktav переживает `loads`, и diff --git a/docs/zh/CHANGELOG.zh.md b/docs/zh/CHANGELOG.zh.md index f059e88..c1d57a4 100644 --- a/docs/zh/CHANGELOG.zh.md +++ b/docs/zh/CHANGELOG.zh.md @@ -54,6 +54,24 @@ Rust 实现见 [`ktav-lang/rust`](https://github.com/ktav-lang/rust)。 它的安装不需要 Rust 工具链、编译器或 maturin,并且只提供 `ktav.pure` 这一个 backend。 +- `ktav.edit_value(source, path, replacement)` 与 + `ktav.edit_value_segments(source, segments, replacement)` —— + 无损的原地源文本编辑(source-edit 0.9)。只有目标值的字节会改变 + (加上布局规则要求展开的 inline 包装层);注释、键拼写、空行、 + BOM 与 CR/LF/CRLF 全部保留,相同替换是字节级 no-op。新增异常家族 + `KtavEditError`,含 `KtavInvalidPathError`(`body` = 路径参数的 + UTF-8 字节偏移十进制串)、`KtavNotFoundError` 与 + `KtavUnsupportedError`;信封仍是同样的十字段,`error` 取 + `EditInvalidPath` / `EditNotFound` / `EditUnsupported` 之一。 +- spec 0.9 语料:conformance 运行器现在执行 `versions/0.9/tests` + 的 fixture(162 个 invalid —— 其中 72 个是新的 `diagnostics09` + EOF/坏 UTF-8 诊断),校验 `expected_location` 元数据(line、 + 原始字节 `start`、精确 `end` 或 `minimum_end`),并让所有 + `raw_bytes` fixture 走共享的字节边界,因此两个 backend 中坏 + UTF-8 都会抛出结构化的 `InvalidUtf8` 信封。 +- `UnclosedCompound` 错误现在固定 opener:`line` 与 `line_text` + 指向打开最内层未闭合容器的行,`span` 从该 opener 延伸到输入末尾。 + ### Changed - 升级到 Rust core 0.8.1 与 spec 0.8.1(发布为规范仓库的 diff --git a/docs/zh/README.zh.md b/docs/zh/README.zh.md index ae8f8a1..f37863d 100644 --- a/docs/zh/README.zh.md +++ b/docs/zh/README.zh.md @@ -169,6 +169,38 @@ print(ktav.format(open("config.ktav").read())) 构造出 Python 值时用 `canonical_from_source`,想让文档自身的注释与分组 存活下来时用 `format`。 +> **尚未发布(0.9 准备版本):** 这些编辑 API 和更新后的诊断不包含在已发布的 0.8.1 包中。原生后端需要固定的 0.9 核心(C-FFI 使用 C ABI 2);纯实现仍完全独立。 + +### 源文本编辑 —— `edit_value` / `edit_value_segments` + +```python +import ktav + +out = ktav.edit_value("port: 8080\nhost: db\n", "port", 9090) +# "port: 9090\nhost: db\n" + +out = ktav.edit_value_segments("a: {x: 1, y: 2}\n", ["a", "y"], {"n": [1]}) +# "a: {x: 1, y: {n: [1]}}\n" +``` + +`edit_value` 接受文本路径(`key1.key2.0.key4`):段内 `\.` 表示 +literal 点号,`\\` 表示 literal 反斜杠;任何其他转义都会抛出 +`KtavInvalidPathError`,其 `body` 是问题位置的 UTF-8 字节偏移十进制串。 +`edit_value_segments` 接受已解码的 literal 段,不再额外反转义 —— +数字段在 Object 中是字符串键,在 Array 中是零基索引。仅支持替换 +既有值:缺失键、越界索引与穿过标量的下降都会抛出 +`KtavNotFoundError`。 + +编辑是对源文本的 splice,而不是整个文档的 +parse→mutate→serialize:有效修改之外的注释、键拼写、顺序、空白、 +BOM 与 CR/LF/CRLF 按字节保留,dotted/inline/block 形式得以维持, +相同替换是字节级 no-op —— 即使拼写非规范。每个成功的补丁都会 +重新解析并做语义校验。当不存在无损补丁时(noncontiguous 的 +dotted/reopen 目标、结构替换下的内部注释、multiline 替换中的 +`))` 行),调用抛出 `KtavUnsupportedError` 而不是猜测。长或多行 +替换只展开被触及的 inline 包装层;未触及的嵌套分支保持 inline。 +native 与 pure backend 实现完全一致的语义。 + 与 JavaScript 绑定不同,`canonical_from_source` 在这里不会带来额外的 数值精度 —— Python 区分 `int` 与 `float`,因此 Ktav 的 Integer/Float 之分在 `loads` 之后依然保留,并且 `emit_canonical(loads(src))` 产生的 diff --git a/python/ktav/__init__.py b/python/ktav/__init__.py index d4438e0..6abb31a 100644 --- a/python/ktav/__init__.py +++ b/python/ktav/__init__.py @@ -26,8 +26,12 @@ if TYPE_CHECKING: from ktav._core import ( KtavDecodeError, + KtavEditError, KtavEncodeError, KtavError, + KtavInvalidPathError, + KtavNotFoundError, + KtavUnsupportedError, __spec_version__, __version__, ) @@ -40,6 +44,12 @@ from ktav._core import ( dumps_force_strings as _dumps_force_strings, ) + from ktav._core import ( + edit_value as _edit_value, + ) + from ktav._core import ( + edit_value_segments as _edit_value_segments, + ) from ktav._core import ( emit_canonical as _emit_canonical, ) @@ -55,13 +65,19 @@ _CORE_NAMES = ( "KtavDecodeError", + "KtavEditError", "KtavEncodeError", "KtavError", + "KtavInvalidPathError", + "KtavNotFoundError", + "KtavUnsupportedError", "__spec_version__", "__version__", "canonical_from_source", "dumps", "dumps_force_strings", + "edit_value", + "edit_value_segments", "emit_canonical", "format", "loads", @@ -71,23 +87,43 @@ "canonical_from_source", "dumps", "dumps_force_strings", + "edit_value", + "edit_value_segments", "emit_canonical", "format", "loads", "loads_strict", ) +_ERROR_CLASSES = ( + "KtavDecodeError", + "KtavEditError", + "KtavEncodeError", + "KtavError", + "KtavInvalidPathError", + "KtavNotFoundError", + "KtavUnsupportedError", +) _CORE_CHECKED = False _HAS_CORE: bool def _no_native(name: str) -> Any: def stub(*args: Any, **kwargs: Any) -> Any: + if name in ("edit_value", "edit_value_segments"): + raise ImportError( + f"ktav.{name} requires a ktav._core built against the" + " source-edit 0.9 core; the installed extension does not" + " export it (an older native binary never falls back to the" + " pure backend silently). Use ktav.pure.edit_value instead:" + " from ktav import pure." + ) raise ImportError( f"ktav.{name} requires the native ktav._core extension, which is" " not installed; use the pure-Python backend instead:" " from ktav import pure (module ktav.pure)." ) + stub._ktav_stub = True # type: ignore[attr-defined] return stub @@ -103,16 +139,24 @@ def _load_core() -> bool: except ImportError: from ktav._pure import ( KtavDecodeError, + KtavEditError, KtavEncodeError, KtavError, + KtavInvalidPathError, + KtavNotFoundError, + KtavUnsupportedError, __spec_version__, __version__, ) globals().update( KtavDecodeError=KtavDecodeError, + KtavEditError=KtavEditError, KtavEncodeError=KtavEncodeError, KtavError=KtavError, + KtavInvalidPathError=KtavInvalidPathError, + KtavNotFoundError=KtavNotFoundError, + KtavUnsupportedError=KtavUnsupportedError, __spec_version__=__spec_version__, __version__=__version__, ) @@ -121,10 +165,24 @@ def _load_core() -> bool: _HAS_CORE = False else: for _name in _CORE_NAMES: - if _name not in _NATIVE_FUNCS: + if _name in _NATIVE_FUNCS: + continue + # A stale native binary predating the source-edit 0.9 symbols + # must degrade into a LOUD stub, never a silent pure fallback. + try: globals()[_name] = getattr(_core, _name) + except AttributeError: + if _name in _ERROR_CLASSES: + from ktav import _pure as _puremod + + globals()[_name] = getattr(_puremod, _name) + else: + globals()[_name] = _no_native(_name) for _name in _NATIVE_FUNCS: - globals()["_" + _name] = getattr(_core, _name) + try: + globals()["_" + _name] = getattr(_core, _name) + except AttributeError: + globals()["_" + _name] = _no_native(_name) _HAS_CORE = True _CORE_CHECKED = True return _HAS_CORE @@ -147,14 +205,20 @@ def __getattr__(name: str) -> Any: __all__ = [ "KtavDecodeError", + "KtavEditError", "KtavEncodeError", "KtavError", + "KtavInvalidPathError", + "KtavNotFoundError", + "KtavUnsupportedError", "__spec_version__", "__version__", "canonical_from_source", "dump", "dumps", "dumps_force_strings", + "edit_value", + "edit_value_segments", "emit_canonical", "format", "load", @@ -163,20 +227,31 @@ def __getattr__(name: str) -> Any: ] +def _as_text(s: str | bytes | bytearray | memoryview) -> str: + """UTF-8 byte boundary for every text entry point (§ 6.15). + + Raw bytes are validated (never lossily decoded) BEFORE any grammar + processing, so a malformed tail cannot lose to an earlier grammar + error and both backends raise the same structured ``InvalidUtf8`` + envelope with raw-byte offsets. + """ + if isinstance(s, (bytes, bytearray, memoryview)): + from ktav._pure._text import validate_utf8 + + return validate_utf8(bytes(s)) + return s + + def loads(s: str | bytes | bytearray) -> Any: """Parse a Ktav document from a string (or UTF-8 bytes).""" _load_core() - if isinstance(s, (bytes, bytearray)): - s = bytes(s).decode("utf-8") - return _loads(s) + return _loads(_as_text(s)) def loads_strict(s: str | bytes | bytearray) -> Any: """Parse a Ktav document with strict canonical-scalar validation.""" _load_core() - if isinstance(s, (bytes, bytearray)): - s = bytes(s).decode("utf-8") - return _loads_strict(s) + return _loads_strict(_as_text(s)) def dumps(obj: Any) -> str: @@ -221,9 +296,7 @@ def format(s: str | bytes | bytearray) -> str: Raises :class:`KtavDecodeError` on malformed input. """ _load_core() - if isinstance(s, (bytes, bytearray)): - s = bytes(s).decode("utf-8") - return _format(s) + return _format(_as_text(s)) def canonical_from_source(s: str | bytes | bytearray) -> str: @@ -241,9 +314,7 @@ def canonical_from_source(s: str | bytes | bytearray) -> str: Raises :class:`KtavDecodeError` on malformed input. """ _load_core() - if isinstance(s, (bytes, bytearray)): - s = bytes(s).decode("utf-8") - return _canonical_from_source(s) + return _canonical_from_source(_as_text(s)) def dumps_force_strings(obj: Any) -> str: @@ -266,6 +337,41 @@ def dumps_force_strings(obj: Any) -> str: return _dumps_force_strings(obj) +def edit_value(source: str | bytes | bytearray, path: str, replacement: Any) -> str: + """Replace the value at the textual ``path`` in ``source``. + + ``path`` is the dotted form (``key1.key2.0.key4``) where ``\\.`` + escapes a literal dot and ``\\\\`` a literal backslash; any other + escape raises :class:`KtavInvalidPathError` whose ``body`` is the + decimal UTF-8 byte offset of the problem. The source must parse; + only the bytes of the target value (plus inline wrappers the layout + rules force to unfold) change — comments, key spelling, whitespace, + BOM and CR/LF/CRLF survive. An equal replacement is a byte-identical + no-op. + + ``replacement`` is a native Python value. Raises + :class:`KtavNotFoundError` when the path addresses nothing and + :class:`KtavUnsupportedError` when no lossless splice exists. + """ + _load_core() + return _edit_value(_as_text(source), path, replacement) + + +def edit_value_segments(source: str | bytes | bytearray, segments: Any, replacement: Any) -> str: + """:func:`edit_value` with already-decoded literal path segments. + + ``segments`` is a sequence of exact key strings — no additional + unescaping. A numeric segment is a string key inside an Object and a + zero-based ASCII-digits index inside an Array (leading zeros + allowed, signs are not indexes). An empty sequence raises + :class:`KtavNotFoundError`. + """ + _load_core() + if not all(isinstance(segment, str) for segment in segments): + raise KtavEditError("edit path segments must be strings", error="Message") + return _edit_value_segments(_as_text(source), list(segments), replacement) + + def load(fp: IO[Any]) -> Any: """Parse a Ktav document read from a file-like object. diff --git a/python/ktav/_core.pyi b/python/ktav/_core.pyi index 491a4e8..a267088 100644 --- a/python/ktav/_core.pyi +++ b/python/ktav/_core.pyi @@ -92,6 +92,19 @@ def format(s: str) -> str: Raises :class:`KtavDecodeError` on malformed input. """ +def edit_value(source: str, path: str, replacement: Any) -> str: + """Replace the value at the textual ``path`` in ``source``. + + ``path`` is the dotted form (``key1.key2.0.key4``); ``\\.`` escapes + a literal dot, ``\\\\`` a literal backslash. Only the bytes of the + target (plus inline wrappers the layout rules force to unfold) + change. Raises ``KtavInvalidPathError`` (``body`` = decimal UTF-8 + byte offset), ``KtavNotFoundError``, or ``KtavUnsupportedError``. + """ + +def edit_value_segments(source: str, segments: list[str], replacement: Any) -> str: + """:func:`edit_value` with already-decoded literal path segments.""" + class KtavError(Exception): """Base class for every exception raised by this library. @@ -131,3 +144,22 @@ class KtavDecodeError(KtavError): class KtavEncodeError(KtavError): """Raised when serialising a Python value to Ktav fails.""" + +class KtavEditError(KtavError): + """Base class for source-edit failures. + + ``error`` is ``"EditInvalidPath"`` / ``"EditNotFound"`` / + ``"EditUnsupported"``; ``body`` is the decimal UTF-8 byte offset + into the path argument for ``EditInvalidPath`` and the limitation + reason for ``EditUnsupported``; every other envelope field stays + ``None``. + """ + +class KtavInvalidPathError(KtavEditError): + """The textual path is malformed (bad escape or empty segment).""" + +class KtavNotFoundError(KtavEditError): + """The path does not address an existing value.""" + +class KtavUnsupportedError(KtavEditError): + """No lossless single-range splice exists for this edit.""" diff --git a/python/ktav/_pure/__init__.py b/python/ktav/_pure/__init__.py index 3eed79a..005de7b 100644 --- a/python/ktav/_pure/__init__.py +++ b/python/ktav/_pure/__init__.py @@ -12,6 +12,9 @@ - :func:`format` — comment-preserving text-to-text formatter. - :func:`canonical_from_source` — ``emit_canonical(loads(text))`` without materialising the value tree. +- :func:`edit_value` / :func:`edit_value_segments` — lossless + source-value splice (source-edit 0.9; the version markers below keep + the released 0.8.1 values until the coordinated version bump). - :data:`__version__` / :data:`__spec_version__` — ``"0.8.1"``. Errors are the same :class:`KtavError` / @@ -25,7 +28,15 @@ from typing import Any from ._text import validate_utf8 -from .errors import KtavDecodeError, KtavEncodeError, KtavError +from .errors import ( + KtavDecodeError, + KtavEditError, + KtavEncodeError, + KtavError, + KtavInvalidPathError, + KtavNotFoundError, + KtavUnsupportedError, +) from .parser import parse_document from .writer import ( canonical_from_source, @@ -49,13 +60,19 @@ def _as_str(s: str | bytes | bytearray | memoryview) -> str: __all__ = [ "KtavDecodeError", + "KtavEditError", "KtavEncodeError", "KtavError", + "KtavInvalidPathError", + "KtavNotFoundError", + "KtavUnsupportedError", "__spec_version__", "__version__", "canonical_from_source", "dumps", "dumps_force_strings", + "edit_value", + "edit_value_segments", "emit_canonical", "format", "loads", @@ -69,7 +86,7 @@ def _as_str(s: str | bytes | bytearray | memoryview) -> str: def loads(s: str) -> Any: """Parse a Ktav document string into a native Python value. - Mapping (spec 0.8.1): ``null`` → ``None``, ``true``/``false`` → + Mapping (spec 0.9): ``null`` → ``None``, ``true``/``false`` → ``bool``, integer literal (§ 3.6) → ``int`` (i64 range; overflow → ``str``, with decimal→float fallback where applicable), float literal → ``float``, bare / ``::`` scalar → ``str``, ``[ … ]`` → ``list``, @@ -93,6 +110,60 @@ def loads_strict(s: str) -> Any: return parse_document(_as_str(s), strict=True) +def edit_value(source: str | bytes | bytearray | memoryview, path: str, replacement: Any) -> str: + """Replace the value at the textual ``path`` in ``source``. + + ``path`` is the dotted form (``key1.key2.0.key4``) where ``\\.`` + escapes a literal dot and ``\\\\`` a literal backslash; any other + escape raises :class:`KtavInvalidPathError` whose ``body`` carries + the UTF-8 byte offset of the problem. The source must parse; only + the bytes of the target (and, when the layout rules demand it, of + inline ancestor wrappers) are replaced — comments, key spelling, + whitespace, BOM, and CR/LF/CRLF all survive. A replacement equal to + the current value is a byte-identical no-op. + + ``replacement`` is a native Python value (``None`` / ``bool`` / + ``int`` / ``float`` / ``str`` / ``list`` / ``dict``). Raises + :class:`KtavNotFoundError` for a path that addresses nothing, + :class:`KtavUnsupportedError` when no lossless splice exists, and + :class:`KtavDecodeError` for a replacement that cannot be encoded. + """ + from ._edit import parse_path + from .parser import parse_document + + # Source first: validate + parse BEFORE the path transport decode so + # a malformed source reports the real 0.9 envelope, not InvalidPath. + parse_document(_as_str(source)) + return edit_value_segments(_as_str(source), parse_path(path), replacement) + + +def edit_value_segments( + source: str | bytes | bytearray | memoryview, + segments: Any, + replacement: Any, +) -> str: + """:func:`edit_value` with already-decoded literal segments. + + ``segments`` is a sequence of exact key strings — no additional + unescaping. A numeric segment is a string key inside an Object and + a zero-based ASCII-digits index inside an Array (leading zeros + allowed, signs are not indexes). An empty sequence raises + :class:`KtavNotFoundError`. + """ + from ._edit import apply_edit + from .parser import parse_document + from .writer import to_value + + # Source first (see :func:`edit_value`). + text = _as_str(source) + parse_document(text) + decoded = list(segments) + for segment in decoded: + if not isinstance(segment, str): + raise KtavEditError("edit path segments must be strings", error="Message") + return apply_edit(text, decoded, to_value(replacement)) + + def load(fp: Any) -> Any: """Parse a Ktav document read from a file-like object (text or. diff --git a/python/ktav/_pure/_edit.py b/python/ktav/_pure/_edit.py new file mode 100644 index 0000000..4e722ec --- /dev/null +++ b/python/ktav/_pure/_edit.py @@ -0,0 +1,37 @@ +"""Lossless source-value edit: the pure-Python splice engine. + +A port of the Rust core's positional-tree splice (``edit::tree`` / +``edit::splice`` / ``edit::splice::layout``) onto the pure-Python +parser's invariants. The document is never parse→mutate→serialize: +only the bytes of the target value (and of inline wrappers the layout +rules force to unfold) are replaced; comments, key spelling, blank +lines, BOM, CR/LF/CRLF, and the presence or absence of the final EOL +all survive. Every successful patch is re-parsed and semantically +verified, so a silent corruption is impossible. + +All internal offsets are *character* offsets into the source ``str`` +(slicing-safe); the only byte offsets this module hands out are the +``InvalidPath`` ones computed over the path argument itself, which the +contract pins as UTF-8 byte offsets. + +This module is the entry point only; the implementation lives in +focused private siblings: + +- :mod:`._edit_common` — error helpers, textual-path parsing, + category-exact value equality, line/EOL context helpers. +- :mod:`._edit_tree` — the positional tree (slots, dotted-key/reopen). +- :mod:`._edit_encode` — replacement-body planning (markers, inline + encoding, verbatim blocks, pretty compound rendering). +- :mod:`._edit_layout` — minimal inline unfolding. +- :mod:`._edit_splice` — the apply/verify loop and public entry ops. + +Import cycle note: the graph is a strict DAG (common ← tree/encode ← +layout ← splice ← this entry), so no circular imports exist. +""" + +from __future__ import annotations + +from ._edit_common import parse_path +from ._edit_splice import apply_edit, edit_value, edit_value_segments + +__all__ = ["apply_edit", "edit_value", "edit_value_segments", "parse_path"] diff --git a/python/ktav/_pure/_edit_common.py b/python/ktav/_pure/_edit_common.py new file mode 100644 index 0000000..25bd1cb --- /dev/null +++ b/python/ktav/_pure/_edit_common.py @@ -0,0 +1,294 @@ +"""Shared primitives for the source-edit engine. + +Error helpers, textual-path parsing (UTF-8 byte-offset errors), +category-exact value equality, target lookup/removal, line/EOL context +helpers, Rust-compatible line iteration, and top-level inline segment +splitting. Pure code motion from the former single-file engine; no +behavioural change. +""" + +from __future__ import annotations + +import math +from typing import Any + +from ._inline import _scan_body +from ._text import WS_CHARS, WS_INLINE_CHARS +from .errors import ( + KtavEditError, + KtavInvalidPathError, + KtavNotFoundError, + KtavUnsupportedError, +) + +__all__ = [ + "INDENT_UNIT", + "WS_CHARS", + "WS_INLINE_CHARS", + "_INDENT_UNIT", + "_inline_segments", + "_line_detect_eol", + "_line_indent", + "_line_start", + "_lookup_value", + "_not_found", + "_parse_index", + "_physical_line", + "_physical_line_span", + "_rust_lines", + "_same", + "_take_target", + "_unsupported", + "parse_path", +] + +_INDENT_UNIT = " " +INDENT_UNIT = _INDENT_UNIT + + +# --------------------------------------------------------------------------- +# Edit errors (internal raising helpers) +# --------------------------------------------------------------------------- + + +def _not_found() -> KtavEditError: + return KtavNotFoundError("edit path does not address an existing value", error="EditNotFound") + + +def _unsupported(reason: str) -> KtavEditError: + return KtavUnsupportedError( + f"no lossless source edit: {reason}", + error="EditUnsupported", + body=reason, + ) + + +# --------------------------------------------------------------------------- +# Textual path parsing (byte-offset errors) +# --------------------------------------------------------------------------- + + +def parse_path(path: str) -> list[str]: + """Split ``key1.key2\\.0`` on unescaped dots; ``\\.``/``\\\\`` decode. + + Other escapes, a trailing lone backslash, an empty path, and empty + segments raise :class:`KtavInvalidPathError` whose ``body`` is the + decimal UTF-8 byte offset of the offending position. + """ + if path == "": + raise KtavInvalidPathError( + "invalid edit path (bad escape or empty segment) at byte 0", + error="EditInvalidPath", + body="0", + ) + segments: list[str] = [] + cur: list[str] | None = None + seg_start = 0 # char offset + byte = 0 # byte offset of the current char + i = 0 + n = len(path) + while i < n: + c = path[i] + if c == ".": + if i == seg_start and cur is None: + raise KtavInvalidPathError( + f"invalid edit path (bad escape or empty segment) at byte {byte}", + error="EditInvalidPath", + body=str(byte), + ) + segments.append("".join(cur) if cur is not None else path[seg_start:i]) + cur = None + seg_start = i + 1 + byte += 1 + i += 1 + elif c == "\\": + if i + 1 < n and path[i + 1] in ".\\": + if cur is None: + cur = [path[seg_start:i]] + cur.append("." if path[i + 1] == "." else "\\") + i += 2 + byte += 2 + else: + raise KtavInvalidPathError( + f"invalid edit path (bad escape or empty segment) at byte {byte}", + error="EditInvalidPath", + body=str(byte), + ) + else: + i += 1 + byte += len(c.encode("utf-8")) + if seg_start == n and cur is None: + raise KtavInvalidPathError( + f"invalid edit path (bad escape or empty segment) at byte {byte}", + error="EditInvalidPath", + body=str(byte), + ) + segments.append("".join(cur) if cur is not None else path[seg_start:]) + return segments + + +def _parse_index(seg: str) -> int | None: + """Zero-based ASCII-digits array index; leading zeros OK, signs not.""" + if not seg or not all("0" <= ch <= "9" for ch in seg): + return None + return int(seg) + + +# --------------------------------------------------------------------------- +# Value equality (bool/int/float/str category-exact, order-free objects) +# --------------------------------------------------------------------------- + + +def _same(a: Any, b: Any) -> bool: + if a is None or b is None: + return a is None and b is None + if isinstance(a, bool) or isinstance(b, bool): + return isinstance(a, bool) and isinstance(b, bool) and a is b + if isinstance(a, str) or isinstance(b, str): + return isinstance(a, str) and isinstance(b, str) and a == b + if isinstance(a, int) or isinstance(b, int): + return isinstance(a, int) and isinstance(b, int) and a == b + if isinstance(a, float) or isinstance(b, float): + if not (isinstance(a, float) and isinstance(b, float)) or a != b: + return False + # Signed zero: -0.0 == 0.0 numerically, but the Rust core keeps + # Float as canonical text, where the lexemes differ — so a zero + # sign flip is a REAL edit, never a no-op. Non-zero floats skip + # this branch entirely (one comparison, no allocations). + if a == 0.0: + return math.copysign(1.0, a) == math.copysign(1.0, b) + return True + if isinstance(a, list) and isinstance(b, list): + return len(a) == len(b) and all(_same(x, y) for x, y in zip(a, b)) + if isinstance(a, dict) and isinstance(b, dict): + return set(a) == set(b) and all(_same(a[k], b[k]) for k in a) + return False + + +def _lookup_value(v: Any, segments: list[str]) -> Any: + cur = v + for seg in segments: + if isinstance(cur, dict): + if seg not in cur: + return None + cur = cur[seg] + elif isinstance(cur, list): + idx = _parse_index(seg) + if idx is None or idx >= len(cur): + return None + cur = cur[idx] + else: + return None + return cur + + +def _take_target(v: Any, segments: list[str]) -> Any: + last, parents = segments[-1], segments[:-1] + cur = v + for seg in parents: + if isinstance(cur, dict): + cur = cur[seg] + elif isinstance(cur, list): + cur = cur[_parse_index(seg) or 0] + else: + raise _not_found() + if isinstance(cur, dict): + if last not in cur: + raise _not_found() + old = cur[last] + cur[last] = None + return old + if isinstance(cur, list): + idx = _parse_index(last) + if idx is None or idx >= len(cur): + raise _not_found() + old = cur[idx] + cur[idx] = None + return old + raise _not_found() + + +# --------------------------------------------------------------------------- +# Line / EOL context helpers +# --------------------------------------------------------------------------- + + +def _line_indent(text: str, pos: int) -> str: + start = _line_start(text, pos) + end = start + n = len(text) + while end < n and text[end] in " \t": + end += 1 + return text[start:end] + + +def _line_start(text: str, pos: int) -> int: + idx = max(text.rfind("\n", 0, pos), text.rfind("\r", 0, pos)) + return idx + 1 + + +def _detect_eol(text: str, pos: int) -> str: + n = len(text) + i = pos + while i < n: + c = text[i] + if c == "\n": + return "\n" + if c == "\r": + return "\r\n" if i + 1 < n and text[i + 1] == "\n" else "\r" + i += 1 + before = text[:pos] + k = max(before.rfind("\n"), before.rfind("\r")) + if k >= 0: + if before[k] == "\r": + return "\r\n" if k + 1 < len(before) and before[k + 1] == "\n" else "\r" + return "\n" + + +def _physical_line_span(text: str, pos: int) -> tuple[int, int, str]: + start = _line_start(text, pos) + idx_n = text.find("\n", pos) + idx_r = text.find("\r", pos) + ends = [i for i in (idx_n, idx_r) if i != -1] + end = min(ends) if ends else len(text) + return start, end, text[start:end] + + +def _physical_line(text: str, pos: int) -> str: + return _physical_line_span(text, pos)[2] + + +# Kept under the historical name used across the engine modules. +_line_detect_eol = _detect_eol + + +def _rust_lines(s: str) -> list[str]: + """Rust ``str::lines``: split on LF, strip one trailing CR per line, + and drop the final empty piece after a trailing LF.""" + if not s: + return [] + lines = s.split("\n") + if lines and lines[-1] == "": + lines.pop() + return [ln[:-1] if ln.endswith("\r") else ln for ln in lines] + + +def _is_compound(body: str) -> bool: + return (body.startswith("{") and body.endswith("}")) or ( + body.startswith("[") and body.endswith("]") + ) + + +def _inline_segments(body: str) -> list[tuple[int, int]]: + """Top-level comma segments of a closed inline compound as spans.""" + open_c = body[0] + close_c = "}" if open_c == "{" else "]" + commas: list[int] = [] + _scan_body(body, 1, close_c, open_c == "{", False, commas, 0) + spans: list[tuple[int, int]] = [] + prev = 1 + for pos in commas: + spans.append((prev, pos)) + prev = pos + 1 + spans.append((prev, len(body) - 1)) + return spans diff --git a/python/ktav/_pure/_edit_encode.py b/python/ktav/_pure/_edit_encode.py new file mode 100644 index 0000000..00e4de9 --- /dev/null +++ b/python/ktav/_pure/_edit_encode.py @@ -0,0 +1,308 @@ +"""Replacement-body planning for the source-edit engine. + +Scalar/String/compound body rendering, inline § 3.7 encoding, marker +(raw `::`) decision rules, verbatim block framing, pretty compound +rendering with reindentation, and the Block-slot replacement planner. +Pure code motion from the former single-file engine. +""" + +from __future__ import annotations + +from typing import Any + +from ._inline import MAX_INLINE_DEPTH +from ._keys import is_pair_shape +from ._numbers import canonical_float +from ._text import is_ktav_whitespace +from ._edit_common import _INDENT_UNIT, _rust_lines, _unsupported +from .errors import KtavEditError +from .writer import ( + _Renderer, + _item_needs_raw_marker, + _needs_raw_marker, + push_key_segment, +) + +__all__ = [ + "_PlanCtx", + "_block_replacement", + "_encode_inline", + "_inline_body", + "_inline_compound_body", + "_plan_string", + "_plan_value", + "_reindent", + "_render_compound_body", + "_scalar_body", + "_verbatim_block_body", +] + + +def _scalar_body(v: Any) -> str: + if v is None: + return "null" + if v is True: + return "true" + if v is False: + return "false" + if isinstance(v, int): + return str(v) + if isinstance(v, float): + return canonical_float(v) + raise _unsupported("internal: scalar_body called on a compound value") + + +def _push_inline_string(s: str, force_string: bool, out: list[str]) -> None: + last = len(s) - 1 + for i, c in enumerate(s): + if force_string and i == 0: + out.append(f"\\u{ord(c):04X}") + else: + _encode_inline_char(out, c, i, last) + + +def _encode_inline_char(out: list[str], c: str, i: int, last: int) -> None: + if c == "\\": + out.append("\\\\") + elif c == ",": + out.append("\\,") + elif c == "}": + out.append("\\}") + elif c == "]": + out.append("\\]") + elif c == "{": + out.append("\\{") + elif c == "[": + out.append("\\[") + elif c == "\n": + out.append("\\n") + elif c == "\r": + out.append("\\r") + elif (i == 0 or i == last) and is_ktav_whitespace(c): + out.append(f"\\u{ord(c):04X}") + else: + out.append(c) + + +def _encode_inline(s: str, force_string: bool) -> str: + out: list[str] = [] + _push_inline_string(s, force_string, out) + return "".join(out) + + +def _push_inline_value(v: Any, depth: int, out: list[str]) -> None: + if depth > MAX_INLINE_DEPTH: + raise _unsupported("replacement nesting exceeds inline depth limit (128)") + if isinstance(v, dict): + out.append("{") + for index, (key, value) in enumerate(v.items()): + if index != 0: + out.append(", ") + parts: list[str] = [] + push_key_segment(key, False, parts) + out.extend(parts) + out.append(": ") + if isinstance(value, str): + _push_inline_string(value, _needs_raw_marker(value), out) + else: + _push_inline_value(value, depth + 1, out) + out.append("}") + elif isinstance(v, list): + out.append("[") + for index, value in enumerate(v): + if index != 0: + out.append(", ") + if isinstance(value, str) and value == "": + out.append("()") + elif isinstance(value, str): + _push_inline_string(value, _needs_raw_marker(value), out) + else: + _push_inline_value(value, depth + 1, out) + out.append("]") + elif isinstance(v, str): + _push_inline_string(v, _needs_raw_marker(v), out) + else: + out.append(_scalar_body(v)) + + +def _inline_compound_body(v: Any) -> str: + out: list[str] = [] + _push_inline_value(v, 0, out) + return "".join(out) + + +def _inline_body(replacement: Any, force_string: bool) -> str: + if isinstance(replacement, str): + return _encode_inline(replacement, force_string and _needs_raw_marker(replacement)) + if isinstance(replacement, (dict, list)): + return _inline_compound_body(replacement) + return _scalar_body(replacement) + + +class _PlanCtx: + __slots__ = ( + "block_compound", + "eol", + "existing_raw", + "indent", + "inline_compound", + "root_first", + "role_item_block", + "slot", + ) + + def __init__( + self, + slot: str, + role_item_block: bool, + existing_raw: bool, + root_first: bool, + indent: str, + eol: str, + inline_compound: bool, + block_compound: bool, + ) -> None: + self.slot = slot # "pair" | "item" | "block" + self.role_item_block = role_item_block + self.existing_raw = existing_raw + self.root_first = root_first + self.indent = indent + self.eol = eol + self.inline_compound = inline_compound + self.block_compound = block_compound + + +def _verbatim_block_body(s: str, indent: str, eol: str) -> str: + for line in _rust_lines(s): + if line.strip(" \t") == "))": + raise _unsupported( + "multiline string contains a line equal to '))'; no lossless verbatim form exists" + ) + out = ["((", eol] + first = True + for line in s.split("\n"): + if not first: + out.append(eol) + first = False + out.append(line) + out.append(eol) + out.append(indent) + out.append("))") + return "".join(out) + + +def _render_compound_body(v: Any) -> str: + out: list[str] = [] + renderer = _Renderer(False, False, out) + try: + if isinstance(v, dict): + if not v: + return "{}\n" + out.append("{\n") + renderer.render_object_body(v, 1) + out.append("}\n") + elif isinstance(v, list): + if not v: + return "[]\n" + out.append("[\n") + for item in v: + renderer.render_item(item, 1, False) + out.append("]\n") + else: + raise _unsupported("internal: render_compound_body on a scalar") + except KtavEditError: + raise + except Exception as exc: # KtavEncodeError from the § 5.9.0 walk + raise _unsupported(f"replacement is not representable: {exc}") from exc + return "".join(out) + + +def _reindent(doc: str, indent: str, eol: str) -> str: + out: list[str] = [] + in_verbatim = False + for i, line in enumerate(_rust_lines(doc)): + if line.startswith(_INDENT_UNIT): + line = line[len(_INDENT_UNIT) :] + verbatim_open = line.endswith("((") + verbatim_close = line.strip(" \t") == "))" and line != "" + if i > 0: + out.append(eol) + if not (in_verbatim or line == ""): + out.append(indent) + out.append(line) + if verbatim_open: + in_verbatim = True + elif verbatim_close: + in_verbatim = False + return "".join(out) + + +def _plan_value(replacement: Any, pc: "_PlanCtx") -> tuple[str, bool]: + if isinstance(replacement, str): + return _plan_string(replacement, pc) + if replacement is None or isinstance(replacement, (bool, int, float)): + return _scalar_body(replacement), False + if pc.inline_compound: + return _inline_compound_body(replacement), False + if isinstance(replacement, dict) and not replacement: + body = f"{{{pc.eol}{pc.indent}}}" if pc.block_compound else "{}" + return body, False + if isinstance(replacement, list) and not replacement: + body = f"[{pc.eol}{pc.indent}]" if pc.block_compound else "[]" + return body, False + return _reindent(_render_compound_body(replacement), pc.indent, pc.eol), False + + +def _plan_string(s: str, pc: "_PlanCtx") -> tuple[str, bool]: + if "\r" in s: + raise _unsupported("replacement string contains CR (U+000D)") + edge_ws = bool(s) and (is_ktav_whitespace(s[0]) or is_ktav_whitespace(s[-1])) + if "\n" in s or edge_ws: + return _verbatim_block_body(s, pc.indent, pc.eol), False + if pc.slot == "item": + desired_raw = _item_needs_raw_marker(s) or s == "" + else: + desired_raw = _needs_raw_marker(s) + if pc.slot == "item" and pc.root_first: + desired_raw = desired_raw or is_pair_shape(s) or s.startswith("") + if pc.existing_raw: + desired_raw = True + return s, desired_raw + + +def _block_replacement( + replacement: Any, pc: "_PlanCtx", a: int, b0: int, ws_pair: str +) -> tuple[int, str, str]: + """Returns (from, prefix, body).""" + if isinstance(replacement, str): + s = replacement + if "\r" in s: + raise _unsupported("replacement string contains CR (U+000D)") + multiline = "\n" in s or ( + bool(s) and (is_ktav_whitespace(s[0]) or is_ktav_whitespace(s[-1])) + ) + if multiline: + return b0, "", _verbatim_block_body(s, pc.indent, pc.eol) + if pc.role_item_block and s == "": + return b0, "", "()" + item_raw = pc.role_item_block and ( + _item_needs_raw_marker(s) or (pc.root_first and (is_pair_shape(s) or s.startswith(""))) + ) + pair_raw = (not pc.role_item_block) and _needs_raw_marker(s) + if item_raw: + return b0, ":: ", s + if pair_raw: + return a + 1, ":" + ws_pair, s + return b0, "", s + pc2 = _PlanCtx( + "block", + pc.role_item_block, + pc.existing_raw, + pc.root_first, + pc.indent, + pc.eol, + pc.inline_compound, + pc.block_compound, + ) + body, _ = _plan_value(replacement, pc2) + return b0, "", body diff --git a/python/ktav/_pure/_edit_layout.py b/python/ktav/_pure/_edit_layout.py new file mode 100644 index 0000000..ec5582c --- /dev/null +++ b/python/ktav/_pure/_edit_layout.py @@ -0,0 +1,278 @@ +"""Minimal inline unfolding for the source-edit engine. + +Port of ``edit::splice::layout``: locate the physical inline wrapper +owning the splice, decide whether readability was lost (multiline +replacement, character-width growth beyond max(100, the previous line), +or dense escaping), and expand only the touched ancestor chain while +untouched nested branches stay inline. Pure code motion from the +former single-file engine. +""" + +from __future__ import annotations + +from typing import Any + +from ._inline import MAX_INLINE_DEPTH, ScanError +from ._keys import find_unescaped_colon, process_escapes +from ._text import WS_CHARS, WS_INLINE_CHARS +from ._edit_common import ( + _INDENT_UNIT, + _detect_eol, + _inline_segments, + _is_compound, + _line_indent, + _not_found, + _parse_index, + _physical_line, + _physical_line_span, + _rust_lines, + _same, + _unsupported, +) +from ._edit_encode import _encode_inline, _plan_value, _PlanCtx +from .errors import KtavDecodeError +from .parser import parse_document + +__all__ = ["_layout_adjust", "_requires_block"] + + +def _layout_adjust( + source: str, + candidate: str, + root: Any, + path: list[str], + replacement: Any, + splice: tuple[int, int, int], +) -> str: + frm, to, inserted = splice + owner = _inline_owner(source, root, path, frm) + if owner is None: + return candidate + start, end = owner + if frm < start: + return candidate + new_end = end - (to - frm) + inserted + if not (0 <= start <= new_end <= len(candidate)): + raise _unsupported("invalid layout boundaries") + body = candidate[start:new_end] + if not _is_compound(body): + return candidate + old_width = len(_physical_line(source, frm)) + limit = max(old_width, 100) + force = _requires_block(replacement, 0) + if not force and len(_physical_line(candidate, frm)) <= limit: + return candidate + indent = _line_indent(source, start) + eol = _detect_eol(source, end) + expanded = _expand(body, indent, eol, limit, (frm - start, frm - start + inserted), force, 0) + if expanded is None: + return candidate + return candidate[:start] + expanded + candidate[new_end:] + + +def _slot_body(slot: tuple[Any, ...]) -> tuple[int, int]: + if slot[0] == "block": + return slot[2], slot[3] + return slot[2], slot[3] + + +def _inline_owner(source: str, root: Any, path: list[str], pos: int) -> tuple[int, int] | None: + node = root + if root.inline: + line_start, _line_end, raw_line = _physical_line_span(source, pos) + lead = len(raw_line) - len(raw_line.lstrip(WS_CHARS)) + stripped = raw_line[lead:].lstrip("") + start = line_start + lead + (len(raw_line) - lead - len(stripped)) + return start, start + len(stripped) + for segment in path: + if node.kind == "object": + nxt = node.child(segment) + elif node.kind == "array": + idx = _parse_index(segment) + if idx is None: + raise _not_found() + nxt = node.item(idx) + else: + nxt = None + if nxt is None: + raise _not_found() + node = nxt + if node.slot is not None: + start, end = _slot_body(node.slot) + if start <= len(source) and end <= len(source) and start <= end: + body = source[start:end] + if node.kind in ("object", "array") and _is_compound(body): + if "\n" not in body and "\r" not in body: + return start, end + if node.inline: + line_start, _line_end, raw_line = _physical_line_span(source, pos) + lead = len(raw_line) - len(raw_line.lstrip(WS_CHARS)) + trimmed_line = raw_line[lead:].lstrip("") + base_shift = lead + (len(raw_line) - lead - len(trimmed_line)) + body: str | None = None + if _is_compound(trimmed_line): + body = trimmed_line + else: + colon = find_unescaped_colon(trimmed_line) + if colon is not None: + after_c = trimmed_line[colon + 1 :] + body = after_c.lstrip(WS_INLINE_CHARS) + base_shift += colon + 1 + (len(after_c) - len(body)) + if body is not None and _is_compound(body): + base = line_start + base_shift + return base, base + len(body) + return None + + +def _requires_block(value: Any, depth: int) -> bool: + if depth > MAX_INLINE_DEPTH: + raise _unsupported("replacement nesting exceeds inline depth limit (128)") + if isinstance(value, str): + if "\n" in value or "\r" in value: + return True + encoded = _encode_inline(value, False) + escapes = 0 + i = 0 + while i < len(encoded): + if encoded[i] == "\\": + escapes += 1 + i += 2 + continue + i += 1 + plain = len(value) + overhead = max(0, len(encoded) - plain) + return escapes >= 4 and overhead >= plain // 4 + (1 if plain % 4 else 0) + if isinstance(value, dict): + return any(_requires_block(c, depth + 1) for c in value.values()) + if isinstance(value, list): + return any(_requires_block(c, depth + 1) for c in value) + return False + + +def _expand( + body: str, + indent: str, + eol: str, + limit: int, + changed: tuple[int, int], + force: bool, + depth: int, +) -> str | None: + if depth > MAX_INLINE_DEPTH: + raise _unsupported("layout nesting exceeds inline depth limit") + is_object = body.startswith("{") + try: + spans = _inline_segments(body) + except ScanError: + raise _unsupported("cannot split inline layout") from None + child_indent = indent + _INDENT_UNIT + out: list[str] = ["{" if is_object else "["] + for start, end in spans: + seg = body[start:end] + part = seg.strip(WS_INLINE_CHARS) + if not part: + continue + offset = start + (len(seg) - len(seg.lstrip(WS_INLINE_CHARS))) + touched = offset < changed[1] and offset + len(part) > changed[0] + if is_object: + colon = find_unescaped_colon(part) + if colon is None: + raise _unsupported("inline pair has no separator") + after = part[colon + 1 :] + raw = after.startswith(":") + value = (after[1:] if raw else after).lstrip(WS_INLINE_CHARS) + prefix = part[: len(part) - len(value)] + else: + prefix, value, raw = "", part, False + out.append(eol) + out.append(child_indent) + if ( + not raw + and _is_compound(value) + and touched + and (force or len(child_indent) + len(part) > limit) + ): + value_offset = offset + len(prefix) + local = ( + max(0, changed[0] - value_offset), + max(0, changed[1] - value_offset), + ) + out.append(prefix) + child = _expand(value, child_indent, eol, limit, local, force, depth + 1) + if child is None: + return None + out.append(child) + else: + moved = _standalone(part, prefix, value, raw, is_object, child_indent, eol) + if moved is None: + return None + out.append(moved) + out.append(eol) + out.append(indent) + out.append("}" if is_object else "]") + return "".join(out) + + +def _standalone( + part: str, + prefix: str, + body: str, + raw: bool, + is_object: bool, + indent: str, + eol: str, +) -> str | None: + inline = f"{{{part}}}" if is_object else f"[{part}]" + try: + expected = parse_document(inline) + except KtavDecodeError: + raise _unsupported("invalid inline layout") from None + line = part if is_object else f"[\n{part}\n]" + try: + standalone_parses = parse_document(line) + except KtavDecodeError: + standalone_parses = None + if standalone_parses is not None and _same(standalone_parses, expected): + return part + if raw: + try: + value: Any = process_escapes(body, 1, "", {"start": 0, "end": 0})[0] + except KtavDecodeError: + raise _unsupported("invalid inline string") from None + else: + wrapped = f"[{body}]" + try: + parsed = parse_document(wrapped) + except KtavDecodeError: + raise _unsupported("cannot decode moved inline value") from None + if not (isinstance(parsed, list) and len(parsed) == 1): + raise _unsupported("moved inline value is not a single value") + value = parsed[0] + if isinstance(value, str): + if "\r" in value or any(ln.strip(" \t") == "))" for ln in _rust_lines(value)): + return None + pc = _PlanCtx( + slot="pair" if is_object else "item", + role_item_block=False, + existing_raw=raw, + root_first=False, + indent=indent, + eol=eol, + inline_compound=True, + block_compound=False, + ) + new_body, desired_raw = _plan_value(value, pc) + out: list[str] = [] + if is_object: + colon = find_unescaped_colon(part) + if colon is None: + raise _unsupported("moved pair has no separator") + out.append(part[: colon + 1]) + if desired_raw: + out.append(":") + ws = prefix[colon + 2 :] if raw else prefix[colon + 1 :] + out.append(ws if ws else " ") + elif desired_raw: + out.append(":: ") + out.append(new_body) + return "".join(out) diff --git a/python/ktav/_pure/_edit_splice.py b/python/ktav/_pure/_edit_splice.py new file mode 100644 index 0000000..3fd3bba --- /dev/null +++ b/python/ktav/_pure/_edit_splice.py @@ -0,0 +1,202 @@ +"""Source-splice entry ops for the pure-Python edit engine. + +Port of ``edit::splice::apply``: pre-parse the source (0.9 source-first +priority — the parser envelope wins over any edit-level error), look up +and compare the target, build the positional tree, plan the marker-zone +splice, apply the minimal layout adjustment, and re-parse + semantically +verify the result. Pure code motion from the former single-file engine. +""" + +from __future__ import annotations + +from typing import Any + +from ._edit_common import ( + _detect_eol, + _line_indent, + _lookup_value, + _not_found, + _parse_index, + _same, + _take_target, + _unsupported, +) +from ._edit_encode import ( + _PlanCtx, + _block_replacement, + _inline_body, + _plan_value, +) +from ._edit_layout import _layout_adjust +from ._edit_tree import ( + _build_tree, + _collect_multiline_scalar_ranges, + _descend, + _has_comment_line, +) +from .errors import KtavDecodeError +from .parser import parse_document + +__all__ = ["apply_edit", "edit_value", "edit_value_segments"] + + +def apply_edit(source: str, segments: list[str], replacement: Any) -> str: + """Splice ``replacement`` over the value addressed by ``segments``. + + Source first: the source is validated and parsed BEFORE the path / + replacement transport decode, so a malformed source surfaces as the + real 0.9 parser envelope (line/line_text/span) — never as a + coordinate-less :class:`KtavUnsupportedError`. + """ + original = parse_document(source) + target = _lookup_value(original, segments) + if target is None and not _path_exists(original, segments): + raise _not_found() + if _same(target, replacement): + return source + root, root_explicit = _build_tree(source) + node, inline, root_first = _descend(root, segments, root_explicit) + slot = node.slot + if slot is None: + raise _unsupported( + "target is a noncontiguous value (dotted-key/reopen intermediate); " + "no lossless single-range splice exists" + ) + + slot_kind = slot[0] # "pair" | "item" | "block" + if slot_kind == "pair": + body = (slot[2], slot[3]) + anchor = slot[1] + block_item_role = False + elif slot_kind == "item": + body = (slot[2], slot[3]) + anchor = slot[1] + block_item_role = False + else: + body = (slot[2], slot[3]) + anchor = slot[1] if slot[1] is not None else slot[2] + block_item_role = slot[1] is None + b0, b1 = body + a = anchor + + if slot_kind == "block": + existing_raw = False + ws_raw = "" + ws_plain = "" + else: + inline_item = inline and slot_kind == "item" + if inline_item: + existing_raw = False + ws_raw = "" + ws_plain = "" + else: + existing_raw = a + 1 < len(source) and source[a + 1] == ":" + ws_raw = source[a + 2 : b0] if existing_raw else "" + ws_plain = source[a + 1 : b0] if slot_kind == "pair" else "" + + if node.kind in ("object", "array"): + ranges: list[tuple[int, int]] = [] + _collect_multiline_scalar_ranges(node, ranges) + if _has_comment_line(source, b0, b1, ranges): + raise _unsupported( + "target container contains comment lines; structural " + "replacement cannot preserve them losslessly" + ) + + indent = _line_indent(source, a) + eol = _detect_eol(source, b1) + pc = _PlanCtx( + slot=slot_kind, + role_item_block=slot_kind == "block" and block_item_role, + existing_raw=existing_raw, + root_first=root_first, + indent=indent, + eol=eol, + inline_compound=slot_kind != "block" and node.kind in ("object", "array"), + block_compound=slot_kind == "block" and node.kind in ("object", "array"), + ) + ws_pair_block = "" if slot_kind != "block" or block_item_role else source[a + 1 : b0] + + if inline: + if slot_kind == "item" and isinstance(replacement, str) and replacement == "": + plan = (b0, "", "()") + elif slot_kind == "item": + plan = (b0, "", _inline_body(replacement, True)) + elif slot_kind == "pair" and existing_raw and not isinstance(replacement, str): + plan = (a + 1, ws_raw, _inline_body(replacement, False)) + else: + plan = (b0, "", _inline_body(replacement, not existing_raw)) + elif slot_kind == "block": + plan = _block_replacement(replacement, pc, a, b0, ws_pair_block) + else: + new_body, desired_raw = _plan_value(replacement, pc) + if existing_raw == desired_raw: + prefix = " " if b0 == b1 and not ws_plain and not ws_raw else "" + plan = (b0, prefix, new_body) + elif existing_raw and not desired_raw and slot_kind == "pair": + ws = ws_raw if b0 != b1 or ws_raw else " " + plan = (a + 1, ws, new_body) + elif existing_raw and not desired_raw: + plan = (a, ws_raw, new_body) + elif desired_raw and slot_kind == "pair": + ws = ws_plain if b0 != b1 or ws_plain else " " + plan = (a + 1, ":" + ws, new_body) + else: + plan = (a, ":: ", new_body) + + frm, prefix, new_body = plan + result = source[:frm] + prefix + new_body + source[b1:] + result = _layout_adjust( + source, result, root, segments, replacement, (frm, b1, len(prefix) + len(new_body)) + ) + + try: + new_doc = parse_document(result) + except KtavDecodeError as exc: + raise _unsupported(f"patch produced an invalid document: {exc}") from None + actual = _take_target(new_doc, segments) + target_matches = _same(actual, replacement) + expected = original + _take_target(expected, segments) + if not target_matches or not _same(new_doc, expected): + raise _unsupported("semantic mismatch after splice; no lossless patch produced") + return result + + +def _path_exists(original: Any, segments: list[str]) -> bool: + cur = original + for seg in segments: + if isinstance(cur, dict): + if seg not in cur: + return False + cur = cur[seg] + elif isinstance(cur, list): + idx = _parse_index(seg) + if idx is None or idx >= len(cur): + return False + cur = cur[idx] + else: + return False + return True + + +def edit_value(source: str, path: str, replacement: Any) -> str: + """Replace the value at the textual ``path`` in ``source``.""" + from ._edit_common import parse_path + + segments = parse_path(path) + return _edit_segments_checked(source, segments, replacement) + + +def edit_value_segments(source: str, segments: list[str], replacement: Any) -> str: + """Replace the value at already-decoded literal ``segments``.""" + if not segments: + raise _not_found() + return _edit_segments_checked(source, list(segments), replacement) + + +def _edit_segments_checked(source: str, segments: list[str], replacement: Any) -> str: + from .writer import to_value + + value = to_value(replacement) + return apply_edit(source, segments, value) diff --git a/python/ktav/_pure/_edit_tree.py b/python/ktav/_pure/_edit_tree.py new file mode 100644 index 0000000..8f736ff --- /dev/null +++ b/python/ktav/_pure/_edit_tree.py @@ -0,0 +1,614 @@ +"""Positional tree for the source-edit engine (port of ``edit::tree``). + +One pass over the document's lines, mirroring the pure parser's line +dispatch decisions, but recording character-span slots (Pair / Item / +Block) instead of trivia. Dotted-key expansion and reopen follow the +parser's insertion semantics; reopen invalidates the slot of the +reopened node (noncontiguous). Inline compounds are positioned +recursively with the same child spans the parser scans. +""" + +from __future__ import annotations + +from typing import Any + +from ._inline import MAX_INLINE_DEPTH +from ._keys import ( + check_key, + decode_key_segment, + find_unescaped_colon, + split_key_path, +) +from ._text import WS_CHARS, WS_INLINE_CHARS, is_inline_whitespace +from ._edit_common import ( + _inline_segments, + _not_found, + _parse_index, + _unsupported, +) + +__all__ = [ + "_Node", + "_build_tree", + "_collect_multiline_scalar_ranges", + "_descend", + "_has_comment_line", +] + + +# --------------------------------------------------------------------------- +# Positional tree nodes +# --------------------------------------------------------------------------- + + +class _Node: + __slots__ = ("children", "inline", "kind", "slot") + + def __init__( + self, kind: str, children: Any, inline: bool, slot: tuple[Any, ...] | None + ) -> None: + self.kind = kind # "scalar" | "object" | "array" + self.children = children # dict[str, Node] | list[Node] | None + self.inline = inline + # slot tuples: + # ("pair", colon, b0, b1) + # ("item", start, b0, b1) + # ("block", anchor|None, start, end) + self.slot: tuple[Any, ...] | None = slot + + def child(self, key: str) -> "_Node | None": + if self.kind == "object": + return self.children.get(key) + return None + + def item(self, i: int) -> "_Node | None": + if self.kind == "array": + if 0 <= i < len(self.children): + return self.children[i] + return None + return None + + +# --------------------------------------------------------------------------- +# Inline positioning (char spans over one physical line) +# --------------------------------------------------------------------------- + + +def _position_inline( + body: str, + base: int, + line_num: int, + line_text: str, + span: dict[Any, Any], + is_object: bool, + depth: int, +) -> _Node: + if depth > MAX_INLINE_DEPTH: + raise _unsupported("replacement nesting exceeds inline depth limit (128)") + spans = _inline_segments(body) + if is_object: + table: dict[str, _Node] = {} + for start, end in spans: + seg = body[start:end] + trimmed = seg.strip(WS_INLINE_CHARS) + if not trimmed: + continue + seg_base = base + start + (len(seg) - len(seg.lstrip(WS_INLINE_CHARS))) + seg_end = seg_base + len(trimmed) + colon = find_unescaped_colon(trimmed) + if colon is None: + raise _unsupported("internal: inline pair without separator") + colon_abs = seg_base + colon + key = trimmed[:colon] + after = trimmed[colon + 1 :] + if after.startswith(":"): + rest_raw = after[1:] + v = rest_raw.lstrip(WS_INLINE_CHARS) + node = _Node("scalar", None, True, None) + node.slot = ( + "pair", + colon_abs, + seg_base + colon + 1 + (len(rest_raw) - len(v)), + seg_end, + ) + else: + v = after.lstrip(WS_INLINE_CHARS) + value_abs = seg_base + (len(after) - len(v)) + node = _inline_child(v, value_abs, line_num, line_text, span, depth) + node.slot = ("pair", colon_abs, value_abs, seg_end) + _insert_leaf_value(table, key, node, line_num, line_text, span) + return _Node("object", table, True, None) + items: list[_Node] = [] + for start, end in spans: + seg = body[start:end] + trimmed = seg.strip(WS_INLINE_CHARS) + if not trimmed: + continue + seg_base = base + start + (len(seg) - len(seg.lstrip(WS_INLINE_CHARS))) + seg_end = seg_base + len(trimmed) + value_abs = seg_base + node = _inline_child(trimmed, value_abs, line_num, line_text, span, depth) + node.slot = ("item", seg_base, value_abs, seg_end) + items.append(node) + return _Node("array", items, True, None) + + +def _inline_child( + v: str, v_abs: int, line_num: int, line_text: str, span: dict[Any, Any], depth: int +) -> _Node: + if v[:1] == "{": + return _position_inline(v, v_abs, line_num, line_text, span, True, depth + 1) + if v[:1] == "[": + return _position_inline(v, v_abs, line_num, line_text, span, False, depth + 1) + return _Node("scalar", None, True, None) + + +# --------------------------------------------------------------------------- +# Tree insertion with dotted-key / reopen semantics (parse already passed, +# so validation failures here are internal errors) +# --------------------------------------------------------------------------- + + +def _insert_leaf_value( + table: dict[str, _Node], + key_raw: str, + value: _Node, + line_num: int, + line_text: str, + span: dict[Any, Any], +) -> None: + """Full dotted-key insertion; the LAST segment inserts ``value``.""" + segs = [s.strip(WS_INLINE_CHARS) for s in split_key_path(key_raw)] + decoded: list[str] = [] + for seg in segs: + validity = check_key(seg) + if validity == "empty": + raise _unsupported("internal: empty key segment while positioning") + if validity == "invalid": + raise _unsupported("internal: invalid key segment while positioning") + decoded.append(decode_key_segment(seg, line_num, line_text, span)) + cur = table + for key in decoded[:-1]: + existing = cur.get(key) + if existing is None: + node = _Node("object", {}, False, None) + cur[key] = node + cur = node.children + elif existing.kind == "object": + existing.slot = None # reopen: noncontiguous from here on + cur = existing.children + else: + raise _unsupported("internal: dotted descent through a scalar") + last = decoded[-1] + if last in cur: + raise _unsupported("internal: duplicate key while positioning") + cur[last] = value + + +# --------------------------------------------------------------------------- +# Line shape classification (mirrors the Rust tree's classify_* helpers) +# --------------------------------------------------------------------------- + + +def _classify_value_shape(body: str) -> str: + trimmed = body.strip(WS_INLINE_CHARS) + if trimmed == "": + return "empty-string" + if trimmed == "{": + return "open-obj" + if trimmed == "[": + return "open-arr" + if trimmed.startswith("{"): + if trimmed.endswith("}") and trimmed[1:-1].strip(WS_INLINE_CHARS) == "": + return "empty-obj" + return "inline" + if trimmed.startswith("["): + if trimmed.endswith("]") and trimmed[1:-1].strip(WS_INLINE_CHARS) == "": + return "empty-arr" + return "inline" + if trimmed == "(": + return "open-ml" + if trimmed == "((": + return "open-mlv" + return "scalar" + + +def _classify_root_shape(trimmed: str) -> str: + if trimmed.startswith("{"): + return "inline-obj" if len(trimmed) > 1 else "explicit-obj" + if trimmed.startswith("["): + return "inline-arr" if len(trimmed) > 1 else "explicit-arr" + if trimmed in ("}", "]"): + return "bare-closer" + scan = find_unescaped_colon(trimmed) + if scan is not None: + prefix = trimmed[:scan].rstrip(WS_INLINE_CHARS) + after = trimmed[scan + 1 :] + if prefix and (after.startswith(":") or after == "" or is_inline_whitespace(after[0])): + return "obj" + return "arr" + + +# --------------------------------------------------------------------------- +# Positional tree builder +# --------------------------------------------------------------------------- + + +def _split_lines_char(text: str) -> list[tuple[str, int]]: + """(line_text, char_start) with LF / lone CR / CRLF terminators.""" + lines: list[tuple[str, int]] = [] + start = 0 + i = 0 + n = len(text) + while i < n: + c = text[i] + if c == "\n": + lines.append((text[start:i], start)) + i += 1 + start = i + elif c == "\r": + lines.append((text[start:i], start)) + i += 2 if i + 1 < n and text[i + 1] == "\n" else 1 + start = i + else: + i += 1 + lines.append((text[start:], start)) + return lines + + +class _Frame: + __slots__ = ("anchor", "kind", "pending_key", "range_start", "table") + + def __init__(self, kind: str, range_start: int) -> None: + self.kind = kind + self.table: dict[str, _Node] | list[_Node] = {} if kind == "obj" else [] + self.pending_key: str | None = None + self.anchor: int | None = None + self.range_start = range_start + + +class _Collecting: + __slots__ = ("anchor", "opener", "verbatim") + + def __init__(self, verbatim: bool, opener: int, anchor: int | None) -> None: + self.verbatim = verbatim + self.opener = opener + self.anchor = anchor + + def terminator(self) -> str: + return "))" if self.verbatim else ")" + + +def _build_tree(text: str) -> tuple[_Node, bool]: + skip = 1 if text.startswith("") else 0 + builder = _Builder() + line_num = 0 + for line_text, start in _split_lines_char(text[skip:]): + line_num += 1 + builder.handle_line(line_text, start + skip, line_num) + return builder.finish() + + +class _Builder: + def __init__(self) -> None: + self.stack: list[_Frame] = [] + self.collecting: _Collecting | None = None + self.root_inline: _Node | None = None + self.root_consumed = False + self.root_is_explicit = False + self.root_initialized = False + + # -- helpers ----------------------------------------------------------- + + @staticmethod + def _ts(line_text: str, line_start: int) -> tuple[int, int, str]: + trimmed = line_text.strip(WS_CHARS) + if not trimmed: + return line_start, line_start + len(line_text), "" + lead = len(line_text) - len(line_text.lstrip(WS_CHARS)) + start = line_start + lead + return start, start + len(trimmed), trimmed + + @staticmethod + def _line_ctx(line_num: int, line_text: str, start: int, end: int) -> dict[Any, Any]: + return {"start": start, "end": end} + + def _last_frame(self) -> _Frame: + return self.stack[-1] + + # -- dispatch ---------------------------------------------------------- + + def handle_line(self, line_text: str, line_start: int, line_num: int) -> None: + if self.collecting is not None: + trimmed = line_text.strip(WS_CHARS) + if trimmed == self.collecting.terminator(): + collecting = self.collecting + self.collecting = None + end = line_start + (len(line_text) - len(trimmed)) + len(trimmed) + node = _Node( + "scalar", None, False, ("block", collecting.anchor, collecting.opener, end) + ) + self._attach_scalar(node, line_num, line_start, line_text) + return + return + + start, end, trimmed = self._ts(line_text, line_start) + span = self._line_ctx(line_num, line_text, start, end) + if not trimmed or trimmed.startswith("##"): + return + if self.root_consumed: + raise _unsupported("internal: orphan line after top-level inline") + if not self.root_initialized: + self.root_initialized = True + if trimmed not in ("}", "]"): + shape = _classify_root_shape(trimmed) + if shape == "inline-obj": + self.root_consumed = True + self.root_inline = _position_inline( + trimmed, start, line_num, line_text, span, True, 0 + ) + return + if shape == "inline-arr": + self.root_consumed = True + self.root_inline = _position_inline( + trimmed, start, line_num, line_text, span, False, 0 + ) + return + if shape == "explicit-obj": + self.root_is_explicit = True + self.stack.append(_Frame("obj", start)) + return + if shape == "explicit-arr": + self.root_is_explicit = True + self.stack.append(_Frame("arr", start)) + return + if shape == "obj": + self.stack.append(_Frame("obj", 0)) + else: + self.stack.append(_Frame("arr", 0)) + + if trimmed == "}": + self._close_frame("obj", line_num, line_text, start, end) + return + if trimmed == "]": + self._close_frame("arr", line_num, line_text, start, end) + return + + if self._last_frame().kind == "arr": + self._array_item(trimmed, line_num, line_text, start, end) + else: + self._object_pair(trimmed, line_num, line_text, start, end) + + # -- object pairs -------------------------------------------------------- + + def _object_pair( + self, line: str, line_num: int, line_text: str, ts_start: int, ts_end: int + ) -> None: + colon = find_unescaped_colon(line) + if colon is None: + raise _unsupported("internal: pair without separator") + key = line[:colon].rstrip(WS_INLINE_CHARS) + colon_off = ts_start + colon + after = line[colon + 1 :] + raw = after.startswith(":") + rest = after[1:] if raw else after + if rest and not is_inline_whitespace(rest[0]): + raise _unsupported("internal: separator not followed by whitespace") + value = rest.strip(WS_INLINE_CHARS) + value_start = ts_start + colon + 1 + (len(after) - len(rest)) + (len(rest) - len(value)) + if raw: + node = _Node("scalar", None, False, ("pair", colon_off, value_start, ts_end)) + self._insert_pair(key, node, line_num, line_text) + return + shape = _classify_value_shape(after) + if shape in ("scalar", "empty-string"): + node = _Node("scalar", None, False, ("pair", colon_off, value_start, ts_end)) + self._insert_pair(key, node, line_num, line_text) + elif shape == "empty-obj": + node = _Node("object", {}, False, ("pair", colon_off, value_start, ts_end)) + self._insert_pair(key, node, line_num, line_text) + elif shape == "empty-arr": + node = _Node("array", [], False, ("pair", colon_off, value_start, ts_end)) + self._insert_pair(key, node, line_num, line_text) + elif shape in ("open-obj", "open-arr"): + self._set_pending(key, colon_off) + self.stack.append(_Frame("obj" if shape == "open-obj" else "arr", value_start)) + elif shape in ("open-ml", "open-mlv"): + self._set_pending(key, colon_off) + self.collecting = _Collecting(shape == "open-mlv", value_start, colon_off) + else: # inline + node = _position_inline( + value, + value_start, + line_num, + line_text, + {"start": ts_start, "end": ts_end}, + value.startswith("{"), + 0, + ) + node.inline = False + node.slot = ("pair", colon_off, value_start, ts_end) + self._insert_pair(key, node, line_num, line_text) + + # -- array items --------------------------------------------------------- + + def _array_item( + self, line: str, line_num: int, line_text: str, ts_start: int, ts_end: int + ) -> None: + if line.startswith("::"): + rest = line[2:] + if rest and not is_inline_whitespace(rest[0]): + raise _unsupported("internal: raw marker without separator space") + value = rest.lstrip(WS_INLINE_CHARS) + body_start = ts_start + 2 + (len(rest) - len(value)) + node = _Node("scalar", None, False, ("item", ts_start, body_start, ts_end)) + self._push_item(node) + return + shape = _classify_value_shape(line) + stripped = line.lstrip(WS_INLINE_CHARS) + body_start = ts_start + (len(line) - len(stripped)) + if shape in ("scalar", "empty-string"): + self._push_item(_Node("scalar", None, False, ("item", ts_start, body_start, ts_end))) + elif shape == "empty-obj": + self._push_item(_Node("object", {}, False, ("item", ts_start, body_start, ts_end))) + elif shape == "empty-arr": + self._push_item(_Node("array", [], False, ("item", ts_start, body_start, ts_end))) + elif shape in ("open-obj", "open-arr"): + self.stack.append(_Frame("obj" if shape == "open-obj" else "arr", body_start)) + elif shape in ("open-ml", "open-mlv"): + self.collecting = _Collecting(shape == "open-mlv", body_start, None) + else: # inline + node = _position_inline( + stripped, + body_start, + line_num, + line_text, + {"start": ts_start, "end": ts_end}, + stripped.startswith("{"), + 0, + ) + node.inline = False + node.slot = ("item", ts_start, body_start, ts_end) + self._push_item(node) + + # -- frames ---------------------------------------------------------------- + + def _set_pending(self, key: str, anchor: int | None) -> None: + frame = self._last_frame() + if frame.kind != "obj": + raise _unsupported("internal: pending key outside object frame") + frame.pending_key = key + frame.anchor = anchor + + def _insert_pair(self, key: str, node: _Node, line_num: int, line_text: str) -> None: + frame = self._last_frame() + if frame.kind != "obj": + raise _unsupported("internal: pair outside object frame") + _insert_leaf_value(frame.table, key, node, line_num, line_text, {"start": 0, "end": 0}) + + def _push_item(self, node: _Node) -> None: + frame = self._last_frame() + if frame.kind != "arr": + raise _unsupported("internal: item outside array frame") + frame.table.append(node) + + def _attach_scalar(self, node: _Node, line_num: int, line_start: int, line_text: str) -> None: + frame = self._last_frame() + if frame.kind == "obj": + key = frame.pending_key + if key is None: + raise _unsupported("internal: multiline closed without pending key") + frame.pending_key = None + anchor = frame.anchor + frame.anchor = None + if node.slot is not None and node.slot[0] == "block": + node.slot = ("block", anchor, node.slot[2], node.slot[3]) + _insert_leaf_value(frame.table, key, node, line_num, line_text, {"start": 0, "end": 0}) + else: + frame.table.append(node) + + def _close_frame( + self, expected: str, line_num: int, line_text: str, ts_start: int, ts_end: int + ) -> None: + if len(self.stack) == 1 and self.root_is_explicit: + self.root_consumed = True + return + if len(self.stack) <= 1: + raise _unsupported("internal: unbalanced bracket while positioning") + frame = self.stack.pop() + if frame.kind != expected: + raise _unsupported("internal: mismatched bracket while positioning") + anchor = None + if self.stack and self.stack[-1].kind == "obj": + anchor = self.stack[-1].anchor + self.stack[-1].anchor = None + node = _Node(frame.kind, frame.table, False, ("block", anchor, frame.range_start, ts_end)) + parent = self._last_frame() + if parent.kind == "obj": + key = parent.pending_key + if key is None: + raise _unsupported("internal: closed compound without pending key") + parent.pending_key = None + _insert_leaf_value(parent.table, key, node, line_num, line_text, {"start": 0, "end": 0}) + else: + parent.table.append(node) + + # -- finish ------------------------------------------------------------------- + + def finish(self) -> tuple[_Node, bool]: + if self.collecting is not None or len(self.stack) > 1: + raise _unsupported("internal: unclosed compound while positioning") + if self.root_is_explicit and not self.root_consumed: + raise _unsupported("internal: unclosed explicit root while positioning") + if self.root_inline is not None: + return self.root_inline, self.root_is_explicit + if not self.stack: + return _Node("object", {}, False, None), self.root_is_explicit + frame = self.stack.pop() + return _Node(frame.kind, frame.table, False, None), self.root_is_explicit + + +# --------------------------------------------------------------------------- +# Descent and comment scanning over the built tree +# --------------------------------------------------------------------------- + + +def _descend(root: _Node, segments: list[str], root_explicit: bool) -> tuple[_Node, bool, bool]: + container_is_root = True + cur = root + root_first = False + for seg in segments: + if cur.kind == "object": + root_first = False + nxt = cur.child(seg) + if nxt is None: + raise _not_found() + cur = nxt + elif cur.kind == "array": + idx = _parse_index(seg) + if idx is None: + raise _not_found() + root_first = container_is_root and not root_explicit and idx == 0 + nxt = cur.item(idx) + if nxt is None: + raise _not_found() + cur = nxt + else: + raise _not_found() + container_is_root = False + return cur, cur.inline, root_first + + +def _collect_multiline_scalar_ranges(node: _Node, out: list[tuple[int, int]]) -> None: + if node.kind == "scalar" and node.slot is not None and node.slot[0] == "block": + out.append((node.slot[2], node.slot[3])) + return + if node.kind == "object": + for child in node.children.values(): + _collect_multiline_scalar_ranges(child, out) + elif node.kind == "array": + for child in node.children: + _collect_multiline_scalar_ranges(child, out) + + +def _has_comment_line(text: str, frm: int, to: int, skip: list[tuple[int, int]]) -> bool: + skip = sorted(skip) + skip_i = 0 + i = frm + while i < to: + line_start = i + j = line_start + while j < to and text[j] not in "\n\r": + j += 1 + line = text[line_start:j] + trimmed = line.lstrip(WS_CHARS) + k = line_start + (len(line) - len(trimmed)) + while skip_i < len(skip) and skip[skip_i][1] <= k: + skip_i += 1 + in_data = skip_i < len(skip) and skip[skip_i][0] <= k < skip[skip_i][1] + if not in_data and trimmed.startswith("##"): + return True + i = j + while i < to and text[i] in "\n\r": + i += 1 + return False diff --git a/python/ktav/_pure/errors.py b/python/ktav/_pure/errors.py index 4c14dbc..4ef44fb 100644 --- a/python/ktav/_pure/errors.py +++ b/python/ktav/_pure/errors.py @@ -12,8 +12,12 @@ __all__ = [ "KtavDecodeError", + "KtavEditError", "KtavEncodeError", "KtavError", + "KtavInvalidPathError", + "KtavNotFoundError", + "KtavUnsupportedError", ] @@ -72,6 +76,31 @@ class KtavEncodeError(KtavError): """Raised when serialising a Python value to Ktav fails.""" +class KtavEditError(KtavError): + """Base class for source-edit failures (``edit_value`` family). + + Carries the same ten-field envelope; ``error`` is one of the + ``EditInvalidPath`` / ``EditNotFound`` / ``EditUnsupported`` codes, + ``body`` is the decimal UTF-8 byte offset into the path argument + for ``EditInvalidPath``, the human-readable limitation reason for + ``EditUnsupported``, and ``None`` for ``EditNotFound``. Every other + envelope field stays ``None`` — an edit error never invents a + source location. + """ + + +class KtavInvalidPathError(KtavEditError): + """The textual path is malformed (bad escape or empty segment).""" + + +class KtavNotFoundError(KtavEditError): + """The path does not address an existing value.""" + + +class KtavUnsupportedError(KtavEditError): + """No lossless single-range splice exists for this edit.""" + + # --------------------------------------------------------------------------- # Display strings — verbatim mirrors of the Rust core's `Display` impls, # so ``str(exc)`` (and ``exc.message``) is byte-identical across diff --git a/python/ktav/_pure/parser.py b/python/ktav/_pure/parser.py index 25be54a..6808eff 100644 --- a/python/ktav/_pure/parser.py +++ b/python/ktav/_pure/parser.py @@ -180,25 +180,51 @@ def _inline_gate( class _Frame: """One open Object or Array context (explicit parser stack).""" - __slots__ = ("explicit", "kind", "opener_byte", "pending_key", "value") + __slots__ = ( + "explicit", + "kind", + "opener_byte", + "opener_err_text", + "opener_line", + "pending_key", + "value", + ) - def __init__(self, kind: str, value: Any, explicit: bool, opener_byte: int) -> None: + def __init__( + self, + kind: str, + value: Any, + explicit: bool, + opener_byte: int, + opener_line: int | None = None, + opener_err_text: str | None = None, + ) -> None: self.kind = kind self.value = value self.explicit = explicit self.pending_key: str | None = None self.opener_byte = opener_byte + self.opener_line = opener_line + self.opener_err_text = opener_err_text class _Collecting: """An open multi-line string block (§ 5.6).""" - __slots__ = ("lines", "opener_byte", "verbatim") + __slots__ = ("opener_byte", "opener_err_text", "opener_line", "verbatim") - def __init__(self, verbatim: bool, opener_byte: int) -> None: + def __init__( + self, + verbatim: bool, + opener_byte: int, + opener_line: int | None = None, + opener_err_text: str | None = None, + ) -> None: self.verbatim = verbatim self.lines: list[str] = [] self.opener_byte = opener_byte + self.opener_line = opener_line + self.opener_err_text = opener_err_text def terminator(self) -> str: return "))" if self.verbatim else ")" @@ -338,10 +364,10 @@ def _handle_root_line( span: dict[Any, Any], ) -> None: if trimmed == "{": - self._open_root_frame("obj", line_num, line_start, explicit=True) + self._open_root_frame("obj", line_num, line_start, True, err_line) return if trimmed == "[": - self._open_root_frame("arr", line_num, line_start, explicit=True) + self._open_root_frame("arr", line_num, line_start, True, err_line) return if trimmed[0] in "{[": value = _inline_gate(trimmed, line_num, err_line, span, 0, self.strict) @@ -368,15 +394,17 @@ def _handle_root_line( after.startswith(":") or after == "" or is_inline_whitespace(after[0]) ) if pair: - self._open_root_frame("obj", line_num, line_start, False) + self._open_root_frame("obj", line_num, line_start, False, err_line) self._pair_line(trimmed, line_num, line_start, err_line, span) return - self._open_root_frame("arr", line_num, line_start, False) + self._open_root_frame("arr", line_num, line_start, False, err_line) self._array_item_line(trimmed, line_num, line_start, err_line, span) - def _open_root_frame(self, kind: str, line_num: int, line_start: int, explicit: bool) -> None: + def _open_root_frame( + self, kind: str, line_num: int, line_start: int, explicit: bool, err_line: str = "" + ) -> None: value: Any = {} if kind == "obj" else [] - frame = _Frame(kind, value, explicit, line_start) + frame = _Frame(kind, value, explicit, line_start, line_num, err_line) self.root = value self.stack.append(frame) self._hook_open(frame) @@ -446,14 +474,14 @@ def _dispatch_value( ) -> None: kind, payload = classify_value_start(body, line_num, err_line, span, self.strict) if kind == "open-obj": - self._push_frame("obj", line_num, line_start, False) + self._push_frame("obj", line_num, line_start, False, err_line) elif kind == "open-arr": - self._push_frame("arr", line_num, line_start, False) + self._push_frame("arr", line_num, line_start, False, err_line) elif kind == "open-ml": - self.collecting = _Collecting(False, line_start) + self.collecting = _Collecting(False, line_start, line_num, err_line) self.multiline_opener = line_start elif kind == "open-mlv": - self.collecting = _Collecting(True, line_start) + self.collecting = _Collecting(True, line_start, line_num, err_line) self.multiline_opener = line_start else: self._attach(payload, line_num, err_line, span) @@ -487,8 +515,10 @@ def _array_item_line( # -- frames and attachment -------------------------------------------------------- - def _push_frame(self, kind: str, line_num: int, line_start: int, explicit: bool) -> None: - frame = _Frame(kind, {} if kind == "obj" else [], explicit, line_start) + def _push_frame( + self, kind: str, line_num: int, line_start: int, explicit: bool, err_line: str = "" + ) -> None: + frame = _Frame(kind, {} if kind == "obj" else [], explicit, line_start, line_num, err_line) self.stack.append(frame) self._hook_open(frame) @@ -556,8 +586,8 @@ def finish(self, eof_byte: int) -> Any: start = self.multiline_opener or 0 raise_decode( "UnclosedCompound", - None, - None, + self.collecting.opener_line, + self.collecting.opener_err_text, {"start": start, "end": eof_byte}, "Unclosed multi-line string at end of input", ) @@ -567,8 +597,8 @@ def finish(self, eof_byte: int) -> Any: frame = self.stack[-1] raise_decode( "UnclosedCompound", - None, - None, + frame.opener_line, + frame.opener_err_text, {"start": frame.opener_byte, "end": eof_byte}, "Unclosed object at end of input" if frame.kind == "obj" diff --git a/python/ktav/pure.py b/python/ktav/pure.py index 99413f1..9197da4 100644 --- a/python/ktav/pure.py +++ b/python/ktav/pure.py @@ -18,14 +18,20 @@ from ._pure import ( KtavDecodeError, + KtavEditError, KtavEncodeError, KtavError, + KtavInvalidPathError, + KtavNotFoundError, + KtavUnsupportedError, __spec_version__, __version__, canonical_from_source, dump, dumps, dumps_force_strings, + edit_value, + edit_value_segments, emit_canonical, format, load, @@ -35,14 +41,20 @@ __all__ = [ "KtavDecodeError", + "KtavEditError", "KtavEncodeError", "KtavError", + "KtavInvalidPathError", + "KtavNotFoundError", + "KtavUnsupportedError", "__spec_version__", "__version__", "canonical_from_source", "dump", "dumps", "dumps_force_strings", + "edit_value", + "edit_value_segments", "emit_canonical", "format", "load", diff --git a/root-docs/CHANGELOG/unreleased-added/body-3.md b/root-docs/CHANGELOG/unreleased-added/body-3.md new file mode 100644 index 0000000..f92b7ce --- /dev/null +++ b/root-docs/CHANGELOG/unreleased-added/body-3.md @@ -0,0 +1,64 @@ +>>>>> lang=en +- `ktav.edit_value(source, path, replacement)` and + `ktav.edit_value_segments(source, segments, replacement)` — lossless + in-place source edits (source-edit 0.9). Only the bytes of the target + value (plus inline wrappers the layout rules force to unfold) change; + comments, key spelling, blank lines, BOM and CR/LF/CRLF survive, and + an equal replacement is a byte-identical no-op. New exception family + `KtavEditError` with `KtavInvalidPathError` (`body` = decimal UTF-8 + byte offset into the path), `KtavNotFoundError`, and + `KtavUnsupportedError`; the same ten-field envelope, `error` is one + of `EditInvalidPath` / `EditNotFound` / `EditUnsupported`. +- Spec 0.9 corpus: the conformance runner now executes the + `versions/0.9/tests` fixtures (162 invalid — 72 new `diagnostics09` + EOF/malformed-UTF-8 diagnostics), validates `expected_location` + metadata (line, raw-byte `start`, exact `end` or `minimum_end`), and + feeds every `raw_bytes`-flagged fixture through the shared byte + boundary, so invalid UTF-8 raises the structured `InvalidUtf8` + envelope in both backends. +- `UnclosedCompound` errors now pin the opener: `line` and `line_text` + name the line that opened the innermost unclosed compound, and + `span` runs from that opener to end of input. + +>>>>> lang=ru +- `ktav.edit_value(source, path, replacement)` и + `ktav.edit_value_segments(source, segments, replacement)` — + lossless-правка исходника по месту (source-edit 0.9). Меняются только + байты целевого значения (плюс inline-обёртки, которые требует + раскрыть layout-правило); комментарии, написание ключей, пустые + строки, BOM и CR/LF/CRLF сохраняются, а совпадающая замена — + byte-identical no-op. Новое семейство исключений `KtavEditError` с + `KtavInvalidPathError` (`body` = десятичный UTF-8 byte offset в + путь), `KtavNotFoundError` и `KtavUnsupportedError`; тот же + десятипольный конверт, `error` — один из `EditInvalidPath` / + `EditNotFound` / `EditUnsupported`. +- Корпус spec 0.9: conformance runner исполняет fixtures из + `versions/0.9/tests` (162 invalid — 72 новых диагностики + `diagnostics09` EOF/битый UTF-8), проверяет метаданные + `expected_location` (line, raw-byte `start`, точный `end` или + `minimum_end`) и прогоняет все `raw_bytes`-fixture через общий + byte-boundary, так что битый UTF-8 даёт структурированный конверт + `InvalidUtf8` в обоих backend-ах. +- Ошибки `UnclosedCompound` теперь указывают открыватель: `line` и + `line_text` называют строку, открывшую ближайший незакрытый + контейнер, а `span` тянется от него до конца ввода. + +>>>>> lang=zh +- `ktav.edit_value(source, path, replacement)` 与 + `ktav.edit_value_segments(source, segments, replacement)` —— + 无损的原地源文本编辑(source-edit 0.9)。只有目标值的字节会改变 + (加上布局规则要求展开的 inline 包装层);注释、键拼写、空行、 + BOM 与 CR/LF/CRLF 全部保留,相同替换是字节级 no-op。新增异常家族 + `KtavEditError`,含 `KtavInvalidPathError`(`body` = 路径参数的 + UTF-8 字节偏移十进制串)、`KtavNotFoundError` 与 + `KtavUnsupportedError`;信封仍是同样的十字段,`error` 取 + `EditInvalidPath` / `EditNotFound` / `EditUnsupported` 之一。 +- spec 0.9 语料:conformance 运行器现在执行 `versions/0.9/tests` + 的 fixture(162 个 invalid —— 其中 72 个是新的 `diagnostics09` + EOF/坏 UTF-8 诊断),校验 `expected_location` 元数据(line、 + 原始字节 `start`、精确 `end` 或 `minimum_end`),并让所有 + `raw_bytes` fixture 走共享的字节边界,因此两个 backend 中坏 + UTF-8 都会抛出结构化的 `InvalidUtf8` 信封。 +- `UnclosedCompound` 错误现在固定 opener:`line` 与 `line_text` + 指向打开最内层未闭合容器的行,`span` 从该 opener 延伸到输入末尾。 + diff --git a/root-docs/CHANGELOG/unreleased-added/meta.js b/root-docs/CHANGELOG/unreleased-added/meta.js index 7e0a98b..6f4ff20 100644 --- a/root-docs/CHANGELOG/unreleased-added/meta.js +++ b/root-docs/CHANGELOG/unreleased-added/meta.js @@ -3,5 +3,5 @@ export default { "number": null, "level": null, "title": null, - "bodyParts": 2 + "bodyParts": 3 } diff --git a/root-docs/README/manifest.js b/root-docs/README/manifest.js index 44c54c5..4e729c2 100644 --- a/root-docs/README/manifest.js +++ b/root-docs/README/manifest.js @@ -5,6 +5,8 @@ export default [ "quickstart-walk", "quickstart-build", "formatting", + "source-edit-status", + "source-edit", "canonical-and-entry-points", "type-mapping", "key-escaping", diff --git a/root-docs/README/source-edit-status/body-1.md b/root-docs/README/source-edit-status/body-1.md new file mode 100644 index 0000000..00dd212 --- /dev/null +++ b/root-docs/README/source-edit-status/body-1.md @@ -0,0 +1,9 @@ +>>>>> lang=en +> **Unreleased (0.9 preparation):** These editing APIs and updated diagnostics are not included in published 0.8.1 packages. Native backends require the pinned 0.9 core (C ABI 2 for C-FFI); pure backends remain standalone. + +>>>>> lang=ru +> **Ещё не выпущено (подготовка 0.9):** Эти API редактирования и обновлённая диагностика не входят в опубликованные пакеты 0.8.1. Native-backend требует закреплённое ядро 0.9 (C ABI 2 для C-FFI); pure-backend остаётся самостоятельным. + +>>>>> lang=zh +> **尚未发布(0.9 准备版本):** 这些编辑 API 和更新后的诊断不包含在已发布的 0.8.1 包中。原生后端需要固定的 0.9 核心(C-FFI 使用 C ABI 2);纯实现仍完全独立。 + diff --git a/root-docs/README/source-edit-status/meta.js b/root-docs/README/source-edit-status/meta.js new file mode 100644 index 0000000..eb86416 --- /dev/null +++ b/root-docs/README/source-edit-status/meta.js @@ -0,0 +1,7 @@ +export default { + "kind": "frontmatter", + "number": null, + "level": null, + "title": null, + "bodyParts": 1 +} diff --git a/root-docs/README/source-edit/body-1.md b/root-docs/README/source-edit/body-1.md new file mode 100644 index 0000000..702bcf1 --- /dev/null +++ b/root-docs/README/source-edit/body-1.md @@ -0,0 +1,103 @@ +>>>>> lang=en +### Source edits — `edit_value` / `edit_value_segments` + +```python +import ktav + +out = ktav.edit_value("port: 8080\nhost: db\n", "port", 9090) +# "port: 9090\nhost: db\n" + +out = ktav.edit_value_segments("a: {x: 1, y: 2}\n", ["a", "y"], {"n": [1]}) +# "a: {x: 1, y: {n: [1]}}\n" +``` + +`edit_value` takes a textual path (`key1.key2.0.key4`): inside a +segment `\.` is a literal dot and `\\` a literal backslash; any other +escape raises `KtavInvalidPathError` whose `body` is the decimal UTF-8 +byte offset of the problem. `edit_value_segments` takes +already-decoded literal segments with no extra unescaping — a numeric +segment is a string key inside an Object and a zero-based index inside +an Array. Only replacement of an existing value is supported; missing +keys, out-of-bounds indexes and descent through a scalar raise +`KtavNotFoundError`. + +The edit is a source splice, not a parse→mutate→serialize of the whole +document: comments, key spelling, order, whitespace, BOM and +CR/LF/CRLF outside the effective change are preserved byte-for-byte, +dotted/inline/block forms are kept, and an equal replacement is a +byte-identical no-op — even for noncanonical spelling. Every +successful patch is re-parsed and semantically verified. When no +lossless patch exists (noncontiguous dotted/reopened targets, internal +comments under structural replacement, a `))` line inside a multiline +replacement) the call raises `KtavUnsupportedError` instead of +guessing. Long or multiline replacements unfold only the touched +inline wrappers; untouched nested branches stay inline. Both the +native and the pure backend implement the identical semantics. + +>>>>> lang=ru +### Правка исходника — `edit_value` / `edit_value_segments` + +```python +import ktav + +out = ktav.edit_value("port: 8080\nhost: db\n", "port", 9090) +# "port: 9090\nhost: db\n" + +out = ktav.edit_value_segments("a: {x: 1, y: 2}\n", ["a", "y"], {"n": [1]}) +# "a: {x: 1, y: {n: [1]}}\n" +``` + +`edit_value` принимает текстовый путь (`key1.key2.0.key4`): внутри +сегмента `\.` — literal точка, `\\` — literal backslash; любой другой +escape бросает `KtavInvalidPathError`, у которого `body` — десятичный +UTF-8 byte offset проблемного места. `edit_value_segments` принимает +уже декодированные literal-сегменты без повторного unescape — числовой +сегмент это строковый ключ в Object и zero-based индекс в Array. +Поддерживается только замена существующего значения: отсутствующий +ключ, выход за границы массива и спуск сквозь скаляр дают +`KtavNotFoundError`. + +Правка — это splice исходника, а не parse→mutate→serialize всего +документа: комментарии, написание ключей, порядок, пробелы, BOM и +CR/LF/CRLF вне эффективного изменения сохраняются побайтово, +dotted/inline/block-формы держатся, а совпадающая замена — +byte-identical no-op, даже при неканоничном написании. Каждый +успешный патч перепарсится и семантически проверяется. Когда lossless +патча не существует (noncontiguous dotted/reopen-цели, внутренние +комментарии при структурной замене, строка `))` внутри multiline +замены), вызов бросает `KtavUnsupportedError`, а не угадывает. Длинные +и многострочные замены раскрывают только затронутые inline-обёртки; +нетронутые ветки остаются inline. Native и pure backend реализуют +идентичную семантику. + +>>>>> lang=zh +### 源文本编辑 —— `edit_value` / `edit_value_segments` + +```python +import ktav + +out = ktav.edit_value("port: 8080\nhost: db\n", "port", 9090) +# "port: 9090\nhost: db\n" + +out = ktav.edit_value_segments("a: {x: 1, y: 2}\n", ["a", "y"], {"n": [1]}) +# "a: {x: 1, y: {n: [1]}}\n" +``` + +`edit_value` 接受文本路径(`key1.key2.0.key4`):段内 `\.` 表示 +literal 点号,`\\` 表示 literal 反斜杠;任何其他转义都会抛出 +`KtavInvalidPathError`,其 `body` 是问题位置的 UTF-8 字节偏移十进制串。 +`edit_value_segments` 接受已解码的 literal 段,不再额外反转义 —— +数字段在 Object 中是字符串键,在 Array 中是零基索引。仅支持替换 +既有值:缺失键、越界索引与穿过标量的下降都会抛出 +`KtavNotFoundError`。 + +编辑是对源文本的 splice,而不是整个文档的 +parse→mutate→serialize:有效修改之外的注释、键拼写、顺序、空白、 +BOM 与 CR/LF/CRLF 按字节保留,dotted/inline/block 形式得以维持, +相同替换是字节级 no-op —— 即使拼写非规范。每个成功的补丁都会 +重新解析并做语义校验。当不存在无损补丁时(noncontiguous 的 +dotted/reopen 目标、结构替换下的内部注释、multiline 替换中的 +`))` 行),调用抛出 `KtavUnsupportedError` 而不是猜测。长或多行 +替换只展开被触及的 inline 包装层;未触及的嵌套分支保持 inline。 +native 与 pure backend 实现完全一致的语义。 + diff --git a/root-docs/README/source-edit/meta.js b/root-docs/README/source-edit/meta.js new file mode 100644 index 0000000..eb86416 --- /dev/null +++ b/root-docs/README/source-edit/meta.js @@ -0,0 +1,7 @@ +export default { + "kind": "frontmatter", + "number": null, + "level": null, + "title": null, + "bodyParts": 1 +} diff --git a/spec b/spec index 3889821..a452756 160000 --- a/spec +++ b/spec @@ -1 +1 @@ -Subproject commit 388982118c0d8f767bf283356ac9c9bb50fdc3f5 +Subproject commit a45275688557dc9ecc24a82afb491dd46e615d4d diff --git a/src/lib.rs b/src/lib.rs index 67d8bc6..4c8b2dd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,6 +37,73 @@ use rustc_hash::FxBuildHasher; create_exception!(_core, KtavError, pyo3::exceptions::PyException); create_exception!(_core, KtavDecodeError, KtavError); create_exception!(_core, KtavEncodeError, KtavError); +create_exception!(_core, KtavEditError, KtavError); +create_exception!(_core, KtavInvalidPathError, KtavEditError); +create_exception!(_core, KtavNotFoundError, KtavEditError); +create_exception!(_core, KtavUnsupportedError, KtavEditError); + +/// Shorthand for handing the edit exception *class object* for an +/// `EditError` variant. `EditError` is `#[non_exhaustive]` upstream, so +/// every match carries a fail-closed wildcard that degrades an unknown +/// future variant into the closest existing code (`Unsupported`) instead +/// of failing to compile (E0004) or panicking. +fn edit_error_class<'py>(py: Python<'py>, err: &ktav::edit::EditError) -> Bound<'py, PyAny> { + match err { + ktav::edit::EditError::InvalidPath { .. } => { + py.get_type::().into_any() + } + ktav::edit::EditError::NotFound => py.get_type::().into_any(), + ktav::edit::EditError::Unsupported { .. } => { + py.get_type::().into_any() + } + // Future variant: closest existing envelope; the message stays + // the upstream Display text, never a silent success. + _ => py.get_type::().into_any(), + } +} + +/// Raise the edit exception carrying the ten-field envelope: the +/// `error` code distinguishes `EditInvalidPath` / `EditNotFound` / +/// `EditUnsupported`; `body` is the decimal UTF-8 byte offset into the +/// path argument for `InvalidPath` and the limitation reason for +/// `Unsupported` (`None` for `NotFound`); every other envelope field +/// stays `None` — an edit error never invents a source location. +fn structured_edit_error(py: Python<'_>, err: &ktav::edit::EditError) -> PyErr { + let class = edit_error_class(py, err); + // Same non-exhaustive discipline: the wildcard maps an unknown + // variant onto EditUnsupported with the Display text as body. + let (code, body): (&str, Option) = match err { + ktav::edit::EditError::InvalidPath { offset } => { + ("EditInvalidPath", Some(offset.to_string())) + } + ktav::edit::EditError::NotFound => ("EditNotFound", None), + ktav::edit::EditError::Unsupported { reason } => ("EditUnsupported", Some(reason.clone())), + _ => ("EditUnsupported", Some(err.to_string())), + }; + let value = class + .call1((err.to_string(),)) + .expect("exception construction cannot fail"); + let attach = (|| -> PyResult<()> { + value.setattr("error", code)?; + value.setattr("reason", py.None())?; + value.setattr("line", py.None())?; + value.setattr("line_text", py.None())?; + value.setattr("span", py.None())?; + value.setattr("path", py.None())?; + match body { + Some(b) => value.setattr("body", b)?, + None => value.setattr("body", py.None())?, + } + value.setattr("canonical", py.None())?; + value.setattr("spec_section", py.None())?; + value.setattr("message", err.to_string())?; + Ok(()) + })(); + if let Err(e) = attach { + return e; + } + PyErr::from_value(value) +} /// Shorthand for handing the decode exception *class object* to /// `structured_error`. @@ -482,6 +549,56 @@ fn dumps_force_strings(py: Python<'_>, obj: &Bound<'_, PyAny>) -> PyResult, + source: &str, + path: &str, + replacement: &Bound<'_, PyAny>, +) -> PyResult { + // Source first: UTF-8 validation + parse happen BEFORE the path / + // replacement transport decode, so a malformed source surfaces as + // the real 0.9 parser envelope (line/line_text/span) — never as a + // coordinate-less EditUnsupported. Mirrors the C ABI ops. + ktav::parse(source).map_err(|e| structured_error(py, &decode_error_class(py), &e, source))?; + let value = py_to_value(replacement) + .map_err(|e| structured_error(py, &encode_error_class(py), &e, ""))?; + ktav::edit::edit_value(source, path, &value).map_err(|e| structured_edit_error(py, &e)) +} + +/// [`edit_value`] with pre-decoded literal path segments: no additional +/// unescaping. A numeric segment is a string key inside an Object and a +/// zero-based ASCII-digits index inside an Array. +#[pyfunction] +#[pyo3(text_signature = "(source, segments, replacement, /)")] +fn edit_value_segments( + py: Python<'_>, + source: &str, + segments: Vec, + replacement: &Bound<'_, PyAny>, +) -> PyResult { + // Source first (see `edit_value`): the parse envelope wins over any + // edit-level error. + ktav::parse(source).map_err(|e| structured_error(py, &decode_error_class(py), &e, source))?; + let value = py_to_value(replacement) + .map_err(|e| structured_error(py, &encode_error_class(py), &e, ""))?; + ktav::edit::edit_value_segments(source, &segments, &value) + .map_err(|e| structured_edit_error(py, &e)) +} + #[pymodule] fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add("__version__", env!("CARGO_PKG_VERSION"))?; @@ -494,11 +611,23 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(dumps, m)?)?; m.add_function(wrap_pyfunction!(emit_canonical, m)?)?; m.add_function(wrap_pyfunction!(dumps_force_strings, m)?)?; + m.add_function(wrap_pyfunction!(edit_value, m)?)?; + m.add_function(wrap_pyfunction!(edit_value_segments, m)?)?; let py = m.py(); m.add("KtavError", py.get_type::())?; m.add("KtavDecodeError", py.get_type::())?; m.add("KtavEncodeError", py.get_type::())?; + m.add("KtavEditError", py.get_type::())?; + m.add( + "KtavInvalidPathError", + py.get_type::(), + )?; + m.add("KtavNotFoundError", py.get_type::())?; + m.add( + "KtavUnsupportedError", + py.get_type::(), + )?; Ok(()) } diff --git a/tests/api/test_error_envelope.py b/tests/api/test_error_envelope.py index 3b1e05c..2707f36 100644 --- a/tests/api/test_error_envelope.py +++ b/tests/api/test_error_envelope.py @@ -162,12 +162,15 @@ def test_format_errors_carry_envelope_too() -> None: e = unclosed.value for field in FIELDS: assert hasattr(e, field) - # UnclosedCompound is an end-of-input class: span and line_text are - # populated, but there is no line number. + # UnclosedCompound is an end-of-input class: since the 0.9 source + # diagnostics it pins the OPENER line (line_text populated) and a + # span from the innermost opener to EOF. assert e.error == "UnclosedCompound" assert e.line_text == "x: [" - assert e.line is None + assert e.line == 1 assert isinstance(e.span, dict) + assert e.span["start"] == 0 + assert e.span["end"] == len("x: [") with pytest.raises(ktav.KtavDecodeError) as dup: ktav.format("a: 1\na: 2") diff --git a/tests/conformance_runner.py b/tests/conformance_runner.py index fd36953..56ce193 100644 --- a/tests/conformance_runner.py +++ b/tests/conformance_runner.py @@ -26,10 +26,10 @@ from ktav._pure.errors import SPEC_SECTIONS REPO = Path(__file__).resolve().parent.parent -SPEC_VERSION = "0.8" -# Spec 0.8.1 is published as the versions/0.8 directory at spec pin -# 388982118c0d8f767bf283356ac9c9bb50fdc3f5 — there is no separate -# versions/0.8.1 directory, so the tests run against versions/0.8. +SPEC_VERSION = "0.9" +# Spec 0.9 runs against the versions/0.9 directory at the pinned spec +# commit a45275688557dc9ecc24a82afb491dd46e615d4d (the submodule gitlink +# is updated by the release tooling, not by hand). SPEC_TESTS = REPO / "spec" / "versions" / SPEC_VERSION / "tests" VALID_DIR = SPEC_TESTS / "valid" @@ -46,7 +46,7 @@ # itself is present the two are cross-checked below. INVENTORY = { "valid": 228, - "invalid": 90, + "invalid": 162, "unrepresentable": 5, "parseable-unrepresentable": 4, "strict-lossy": 13, @@ -97,26 +97,25 @@ def _fixture_text(path: Path) -> str: def _validate_manifest_fixture_flags(manifest: Any) -> None: """Spec § 8.5: validate manifest['fixture_flags'] strictly. - Exactly the 9 known entries, all category 'invalid', every flag from - the closed {'raw_bytes'} set, and every flagged fixture present on - disk. An unknown category, flag, or count anywhere in the manifest - is a hard failure. + The closed set of ``raw_bytes`` fixtures is derived FROM DISK: every + .ktav under ``invalid_utf8/`` and every ``utf8_*`` fixture under + ``diagnostics09/`` is deliberately malformed UTF-8 (§ 6.15 / 0.9 + diagnostics09) and must be listed, and nothing else may be. An + unknown category, flag, or count anywhere in the manifest is a hard + failure. """ - # § 6.15 closed set; a manifest naming anything else must fail loudly. - expected_flags = [ - "invalid_utf8/lone_continuation_byte", - "invalid_utf8/multibyte_crlf_lone_continuation", - "invalid_utf8/bom_lone_cr_bad_byte", - "invalid_utf8/truncated_sequence_at_eof", - "invalid_utf8/interrupted_sequence", - "invalid_utf8/overlong_two_byte", - "invalid_utf8/surrogate_encoded", - "invalid_utf8/above_max_code_point", - "invalid_utf8/grammar_error_before_bad_byte", - ] + expected_flags: list[str] = [] + for directory, prefix in (("invalid_utf8", "invalid_utf8"), ("diagnostics09", "diagnostics09")): + base = INVALID_DIR / directory + if not base.exists(): + continue + for ktav_file in sorted(base.glob("*.ktav")): + stem = ktav_file.name[: -len(".ktav")] + if prefix == "diagnostics09" and not stem.startswith("utf8_"): + continue + expected_flags.append(f"{prefix}/{stem}") flags = manifest["fixture_flags"] assert isinstance(flags, list) - assert len(flags) == 9, f"expected exactly 9 fixture_flags entries, got {len(flags)}" seen = [] for entry in flags: assert set(entry) >= {"category", "fixture", "flags"}, f"malformed flag entry: {entry}" @@ -130,8 +129,8 @@ def _validate_manifest_fixture_flags(manifest: Any) -> None: ) fixture_path = INVALID_DIR / (entry["fixture"] + ".ktav") assert fixture_path.exists(), f"flagged fixture missing on disk: {fixture_path}" - # The observed set must be EXACTLY the 9-name set — no duplicates - # (checked per entry above) and none missing. + # The observed set must be EXACTLY the disk-derived set — no + # duplicates (checked per entry above) and none missing. assert set(seen) == set(expected_flags), ( f"fixture_flags incomplete: missing {sorted(set(expected_flags) - set(seen))}" ) @@ -270,26 +269,68 @@ def check_valid_fixture_matches_oracle(be: Backend, ktav_file: Path, json_file: def check_invalid_fixture_is_rejected(be: Backend, ktav_file: Path) -> None: skip_if_missing(INVALID_DIR) raw = ktav_file.read_bytes() - # § 6.15 InvalidUtf8 fixtures are rejected by the native wrapper's - # UTF-8 decode (UnicodeDecodeError) but by the pure backend's own - # pre-grammar validation (KtavDecodeError / InvalidUtf8). Both are - # rejections; the pure backend additionally gets the § 6.15 span - # checks below because it sees the raw bytes. - with pytest.raises((be.api.KtavDecodeError, UnicodeDecodeError)) as exc_info: + # § 6.15: since 0.9 the shared byte boundary validates raw bytes + # BEFORE any grammar processing in BOTH backends, so a malformed + # UTF-8 tail can never lose to an earlier grammar error and both + # raise the same structured InvalidUtf8 envelope with raw-byte + # offsets (line_text stays None for invalid UTF-8). + with pytest.raises(be.api.KtavDecodeError) as exc_info: be.api.loads(raw) - # When the fixture pins a category and the backend actually produced - # an envelope (pure always does; native not for the § 6.15 fixtures, - # which its wrapper rejects at the decode step), the category must - # match the spec's own expectation. error = exc_info.value - if isinstance(error, be.api.KtavDecodeError): - # invalid_cases() guarantees the oracle and its expected_error key. - json_file = ktav_file.with_suffix(".json") - oracle = json.loads(json_file.read_text(encoding="utf-8")) - assert error.error == oracle["expected_error"] - if be.kind == "pure": - assert error.span is not None - assert error.span["end"] > error.span["start"], "span must cover the malformed bytes" + # invalid_cases() guarantees the oracle and its expected_error key. + json_file = ktav_file.with_suffix(".json") + oracle = json.loads(json_file.read_text(encoding="utf-8")) + assert error.error == oracle["expected_error"] + assert error.span is not None + assert error.span["end"] > error.span["start"], "span must cover the malformed bytes" + if oracle["expected_error"] == "InvalidUtf8": + assert error.line_text is None, "invalid UTF-8 must not fabricate line_text" + + +def check_invalid_fixture_expected_location(be: Backend, ktav_file: Path) -> None: + """Spec 0.9 diagnostics: validate expected_location when pinned. + + ``expected_location`` carries ``line`` (>= 1), ``start`` (raw UTF-8 + byte offset, BOM included, EOL never normalised), and EXACTLY ONE + of ``end`` (exact span end) or ``minimum_end`` (span end >= bound; + e.g. UnclosedCompound spans opener..EOF where only the opener is + deterministic). Both the category and the envelope fields are + checked — optional metadata is never ignored. + """ + skip_if_missing(INVALID_DIR) + json_file = ktav_file.with_suffix(".json") + oracle = json.loads(json_file.read_text(encoding="utf-8")) + loc = oracle.get("expected_location") + if loc is None: + return + assert loc.get("line", 0) >= 1, f"expected_location.line must be >= 1: {json_file}" + assert "start" in loc, f"expected_location.start required: {json_file}" + has_end = "end" in loc + has_min = "minimum_end" in loc + assert has_end != has_min, ( + f"expected_location must pin exactly one of end/minimum_end: {json_file}" + ) + if has_end: + assert loc["end"] > loc["start"], f"empty span pinned: {json_file}" + else: + assert loc["minimum_end"] > loc["start"], f"empty minimum span pinned: {json_file}" + with pytest.raises(be.api.KtavDecodeError) as exc_info: + be.api.loads(ktav_file.read_bytes()) + error = exc_info.value + assert error.error == oracle["expected_error"] + assert error.line == loc["line"], f"{ktav_file}: line {error.line!r} != expected {loc['line']}" + assert error.span is not None, f"{ktav_file}: envelope must carry a span" + assert error.span["start"] == loc["start"], ( + f"{ktav_file}: span start {error.span['start']} != expected {loc['start']}" + ) + if has_end: + assert error.span["end"] == loc["end"], ( + f"{ktav_file}: span end {error.span['end']} != expected {loc['end']}" + ) + else: + assert error.span["end"] >= loc["minimum_end"], ( + f"{ktav_file}: span end {error.span['end']} < minimum_end {loc['minimum_end']}" + ) def check_valid_fixture_roundtrips_through_dump( diff --git a/tests/edit/__init__.py b/tests/edit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/edit/_backends.py b/tests/edit/_backends.py new file mode 100644 index 0000000..498713c --- /dev/null +++ b/tests/edit/_backends.py @@ -0,0 +1,21 @@ +"""Edit-backend parametrization: pure always, native when importable. + +Native and pure MUST agree byte-for-byte on every edit regression, so +both participate in the same oracle-driven cases whenever the compiled +extension exposes the source-edit 0.9 symbols. +""" + +from __future__ import annotations + +from typing import Any + +import ktav +import ktav.pure + + +def edit_apis() -> list[Any]: + apis: list[Any] = [ktav.pure] + native_edit = getattr(ktav, "_edit_value", None) + if ktav._HAS_CORE and native_edit is not None and not getattr(native_edit, "_ktav_stub", False): + apis.insert(0, ktav) + return apis diff --git a/tests/edit/test_edit_layout.py b/tests/edit/test_edit_layout.py new file mode 100644 index 0000000..8c8dd5e --- /dev/null +++ b/tests/edit/test_edit_layout.py @@ -0,0 +1,241 @@ +"""Layout regressions: exact output shape and minimal inline unfolding. + +Port of the Rust reference suite +``tests/suite/api/text/edit_value/layout.rs`` — all 22 layout +regressions, not just the happy path. Every case runs on the pure +backend and, when the extension exposes the 0.9 symbols, on the native +backend with the same literal-output oracle. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +import ktav.pure as pure +from tests.edit._backends import edit_apis + +APIS = edit_apis() + + +def check(api: Any, source: str, path: list[str], replacement: Any, expected: str) -> None: + """Literal-output oracle for both API forms of the same edit.""" + textual = ".".join(segment.replace("\\", "\\\\").replace(".", "\\.") for segment in path) + assert api.edit_value(source, textual, replacement) == expected, textual + assert api.edit_value_segments(source, path, replacement) == expected + + +@pytest.mark.parametrize("api", APIS) +def test_deep_leaf_preserves_each_source_form(api: Any) -> None: + for source in ( + "targets.storagehost.address: old\ntargets.storagehost.user: mcp-agent\n", + "targets: {storagehost: {address: old, user: mcp-agent}}\n", + "targets: {\n storagehost: {\n address: old\n user: mcp-agent\n }\n}\n", + ): + check( + api, + source, + ["targets", "storagehost", "address"], + "192.168.0.88:22", + source.replace("old", "192.168.0.88:22"), + ) + + +@pytest.mark.parametrize("api", APIS) +def test_deep_array_leaf_preserves_mixed_forms(api: Any) -> None: + for source in ( + "targets: [{storagehost: [old, keep]}, other]\n", + "targets: [\n {storagehost: [old, keep]}\n other\n]\n", + "targets: [\n {\n storagehost: [\n old\n keep\n ]\n }\n other\n]\n", + ): + check( + api, + source, + ["targets", "0", "storagehost", "0"], + "new", + source.replace("old", "new"), + ) + + +@pytest.mark.parametrize("api", APIS) +def test_whole_inline_containers_remain_inline(api: Any) -> None: + for source, value, expected in ( + ("a: {x: 1}\n", {"y": 2}, "a: {y: 2}\n"), + ("a: [1, 2]\n", [3, 4], "a: [3, 4]\n"), + ("a: {}\n", {"y": 2}, "a: {y: 2}\n"), + ("a: []\n", [3, 4], "a: [3, 4]\n"), + ("a: {}\n", [3, 4], "a: [3, 4]\n"), + ): + check(api, source, ["a"], value, expected) + + +@pytest.mark.parametrize("api", APIS) +def test_emptying_block_containers_does_not_collapse_them(api: Any) -> None: + check(api, "a: {\n x: 1\n}\n", ["a"], {}, "a: {\n}\n") + check(api, "a: [\n 1\n]\n", ["a"], [], "a: [\n]\n") + + +@pytest.mark.parametrize("api", APIS) +def test_newline_unfolds_only_inline_ancestor_chain(api: Any) -> None: + check( + api, + "## top\na: [{x: old, untouched: [1, 2]}, {keep: yes}]\nz: keep\n", + ["a", "0", "x"], + "one\ntwo", + "## top\na: [\n {\n x: ((\none\ntwo\n ))\n" + " untouched: [1, 2]\n }\n {keep: yes}\n]\nz: keep\n", + ) + + +@pytest.mark.parametrize("api", APIS) +def test_newline_under_dotted_key_does_not_expand_dotted_ancestors(api: Any) -> None: + check( + api, + "a.b: [old, keep]\na.c: other\n", + ["a", "b", "0"], + "one\ntwo", + "a.b: [\n ((\none\ntwo\n ))\n keep\n]\na.c: other\n", + ) + + +@pytest.mark.parametrize("api", APIS) +def test_dense_escapes_unfold_but_sparse_escapes_do_not(api: Any) -> None: + check(api, "a: [old, keep]\n", ["a", "0"], "a,b,c,d,e", "a: [\n a,b,c,d,e\n keep\n]\n") + check(api, "a: [old]\n", ["a", "0"], "a,b,c,d", "a: [a\\,b\\,c\\,d]\n") + sparse = "x" * 30 + "a,b,c,d,e" + check(api, "a: [old]\n", ["a", "0"], sparse, f"a: [{sparse.replace(',', '\\,')}]\n") + + +@pytest.mark.parametrize("api", APIS) +def test_width_boundary_is_in_characters_not_bytes(api: Any) -> None: + for size, expanded in ((95, False), (96, True)): + value = "я" * size + expected = f"a: [\n {value}\n]\n" if expanded else f"a: [{value}]\n" + check(api, "a: [old]\n", ["a", "0"], value, expected) + + +@pytest.mark.parametrize("api", APIS) +def test_already_long_line_only_unfolds_when_it_grows(api: Any) -> None: + source = f"a: [{'x' * 120}, keep]\n" + check(api, source, ["a", "0"], "y" * 120, source.replace("x", "y")) + check(api, source, ["a", "0"], "y" * 121, f"a: [\n {'y' * 121}\n keep\n]\n") + + +@pytest.mark.parametrize("api", APIS) +def test_width_expansion_stops_when_inner_line_fits(api: Any) -> None: + value = "x" * 91 + check(api, "a: [{x: old}]\n", ["a", "0", "x"], value, f"a: [\n {{x: {value}}}\n]\n") + + +@pytest.mark.parametrize("api", APIS) +def test_escape_overhead_boundary_is_exact(api: Any) -> None: + dense = "x" * 7 + "a,b,c,d,e" + sparse = "x" * 8 + "a,b,c,d,e" + check(api, "a: [old]\n", ["a", "0"], dense, f"a: [\n {dense}\n]\n") + check(api, "a: [old]\n", ["a", "0"], sparse, f"a: [{sparse.replace(',', '\\,')}]\n") + + +@pytest.mark.parametrize("api", APIS) +def test_long_changed_leaf_does_not_unfold_unrelated_nested_branches(api: Any) -> None: + value = "x" * 100 + check( + api, + "a: [old, {untouched: [1, 2]}]\n", + ["a", "0"], + value, + f"a: [\n {value}\n {{untouched: [1, 2]}}\n]\n", + ) + + +@pytest.mark.parametrize("api", APIS) +def test_promoted_siblings_keep_values_when_inline_escapes_lose_meaning(api: Any) -> None: + check( + api, + "a: [old, a\\,b, \\u00342, :: literal]\n", + ["a", "0"], + "x\ny", + "a: [\n ((\nx\ny\n ))\n a,b\n :: 42\n :: :: literal\n]\n", + ) + + +@pytest.mark.parametrize("api", APIS) +def test_promoted_pairs_keep_key_spelling_and_raw_markers(api: Any) -> None: + check( + api, + 'a: {"x.y": old, nested.key:: 42, n: 0007}\n', + ["a", "x.y"], + "x\ny", + 'a: {\n "x.y": ((\nx\ny\n ))\n nested.key:: 42\n n: 0007\n}\n', + ) + + +@pytest.mark.parametrize("api", APIS) +def test_root_inline_promotion_preserves_bom_and_eof(api: Any) -> None: + check(api, "[old, keep]", ["0"], "x\ny", "[\n ((\nx\ny\n ))\n keep\n]") + + +@pytest.mark.parametrize("api", APIS) +def test_promotion_preserves_eol_comments_and_no_final_terminator(api: Any) -> None: + for eol in ("\n", "\r", "\r\n"): + source = f"## header{eol}a: [old, keep]" + expected = f"## header{eol}a: [{eol} (({eol}x{eol}y{eol} )){eol} keep{eol}]" + check(api, source, ["a", "0"], "x\ny", expected) + + +@pytest.mark.parametrize("api", APIS) +def test_comments_outside_promoted_inline_value_remain_byte_exact(api: Any) -> None: + check( + api, + "list: [\n ## before\n {x: old}\n ## after\n]\n", + ["list", "0", "x"], + "x\ny", + "list: [\n ## before\n {\n x: ((\nx\ny\n ))\n }\n ## after\n]\n", + ) + + +@pytest.mark.parametrize("api", APIS) +def test_noop_ignores_all_readability_thresholds(api: Any) -> None: + source = f"a: [{'x' * 140}, a\\,b\\,c\\,d\\,e]\n" + check(api, source, ["a", "0"], "x" * 140, source) + check(api, source, ["a", "1"], "a,b,c,d,e", source) + + +@pytest.mark.parametrize("api", APIS) +def test_type_change_does_not_force_unrelated_layout_changes(api: Any) -> None: + check(api, "a: {x: [1], keep: [2]}\n", ["a", "x"], True, "a: {x: true, keep: [2]}\n") + check(api, "a: [1]\n", ["a"], "42", "a:: 42\n") + + +@pytest.mark.parametrize("api", APIS) +def test_reopened_inline_parent_promotes_only_its_physical_definition(api: Any) -> None: + check( + api, + "a: {x: old}\n## gap\na.y: keep\n", + ["a", "x"], + "x\ny", + "a: {\n x: ((\nx\ny\n ))\n}\n## gap\na.y: keep\n", + ) + + +@pytest.mark.parametrize("api", APIS) +def test_impossible_block_encoding_keeps_valid_inline_form(api: Any) -> None: + for value in ("a\rb", "a\n))\nb"): + encoded = value.replace("\r", "\\r").replace("\n", "\\n") + check(api, "a: [old]\n", ["a", "0"], value, f"a: [{encoded}]\n") + check(api, "a: [old, a\\rb]\n", ["a", "0"], "x\ny", "a: [x\\ny, a\\rb]\n") + + +@pytest.mark.parametrize("api", APIS) +def test_unsafe_comment_loss_still_fails_closed(api: Any) -> None: + for source in ("a: {\n ## keep\n x: 1\n}\n", "a: [\n ## keep\n 1\n]\n"): + with pytest.raises(api.KtavUnsupportedError): + api.edit_value_segments(source, ["a"], None) + + +@pytest.mark.parametrize("api", APIS) +def test_results_reparse_to_the_expected_tree(api: Any) -> None: + """Beyond literals: every expansion result parses to the edited tree.""" + source = "a: [old, keep]\n" + out = api.edit_value_segments(source, ["a", "0"], "one\ntwo") + assert pure.loads(out) == {"a": ["one\ntwo", "keep"]} diff --git a/tests/edit/test_edit_value.py b/tests/edit/test_edit_value.py new file mode 100644 index 0000000..538e63b --- /dev/null +++ b/tests/edit/test_edit_value.py @@ -0,0 +1,440 @@ +"""``edit_value`` / ``edit_value_segments``: lossless source-value edits. + +Port of the Rust reference suite ``tests/suite/api/text/edit_value.rs`` +(maintained here as literal-output oracles: a test proves the bytes +outside the target were not touched). Every case runs on the pure +backend and — when the extension exposes the 0.9 symbols — on the +native backend with identical oracles. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +import ktav.pure as pure +from tests.edit._backends import edit_apis + +APIS = edit_apis() + + +def _unwrap(exc: pytest.ExceptionInfo[Any]) -> Any: + return exc.value + + +@pytest.mark.parametrize("api", APIS) +def test_string_replacement_plain(api: Any) -> None: + assert api.edit_value("port: 8080\nhost: db\n", "port", "db.internal") == ( + "port: db.internal\nhost: db\n" + ) + + +@pytest.mark.parametrize("api", APIS) +def test_int_replacement_switches_raw_marker_off(api: Any) -> None: + # `:: 8080` was a String; an Integer replacement must switch it off. + assert api.edit_value("port:: 8080\n", "port", 9090) == "port: 9090\n" + + +@pytest.mark.parametrize("api", APIS) +def test_numeric_string_replacement_switches_raw_marker_on(api: Any) -> None: + assert api.edit_value("port: 8080\n", "port", "42") == "port:: 42\n" + + +@pytest.mark.parametrize("api", APIS) +def test_keyword_like_string_replacement_needs_raw(api: Any) -> None: + assert api.edit_value("mode: auto\n", "mode", "true") == "mode:: true\n" + + +@pytest.mark.parametrize("api", APIS) +def test_raw_marker_preserved_for_string(api: Any) -> None: + assert api.edit_value("k:: v1\n", "k", "v2") == "k:: v2\n" + + +@pytest.mark.parametrize("api", APIS) +def test_empty_string_replacement_keeps_raw(api: Any) -> None: + assert api.edit_value("k:: x\n", "k", "") == "k:: \n" + + +@pytest.mark.parametrize("api", APIS) +def test_null_and_bool_replacements(api: Any) -> None: + t = "a: x\nb: y\nc: z\n" + assert api.edit_value(t, "a", None) == "a: null\nb: y\nc: z\n" + assert api.edit_value(t, "b", False) == "a: x\nb: false\nc: z\n" + assert api.edit_value(t, "c", True) == "a: x\nb: y\nc: true\n" + + +@pytest.mark.parametrize("api", APIS) +def test_float_replacement(api: Any) -> None: + assert api.edit_value("ratio: 1\n", "ratio", 0.5) == "ratio: 0.5\n" + + +@pytest.mark.parametrize("api", APIS) +def test_nested_dotted_path(api: Any) -> None: + t = "srv: {\n host: a\n port: 1\n}\n" + assert api.edit_value(t, "srv.host", "b.example") == ( + "srv: {\n host: b.example\n port: 1\n}\n" + ) + + +@pytest.mark.parametrize("api", APIS) +def test_dotted_source_key_leaf(api: Any) -> None: + assert api.edit_value("a.b: 1\nc: 2\n", "a.b", 3) == "a.b: 3\nc: 2\n" + + +@pytest.mark.parametrize("api", APIS) +def test_array_index_replacement(api: Any) -> None: + t = "list: [\n one\n two\n]\n" + assert api.edit_value(t, "list.1", "TWO") == "list: [\n one\n TWO\n]\n" + + +@pytest.mark.parametrize("api", APIS) +def test_numeric_segment_is_string_key_in_object(api: Any) -> None: + t = "0: first\nlist: [\n x\n]\n" + assert api.edit_value(t, "0", "zero") == "0: zero\nlist: [\n x\n]\n" + assert api.edit_value(t, "list.0", "y") == "0: first\nlist: [\n y\n]\n" + + +@pytest.mark.parametrize("api", APIS) +def test_escaped_dot_in_path_vs_real_key(api: Any) -> None: + t = '"a.b": 1\na: {\n b: 2\n}\n' + assert api.edit_value(t, "a\\.b", 9) == '"a.b": 9\na: {\n b: 2\n}\n' + assert api.edit_value(t, "a.b", 9) == '"a.b": 1\na: {\n b: 9\n}\n' + + +@pytest.mark.parametrize("api", APIS) +def test_escaped_backslash_in_path(api: Any) -> None: + t = '"a\\\\b": 1\n' + assert api.edit_value(t, "a\\\\b", 2) == '"a\\\\b": 2\n' + + +@pytest.mark.parametrize("api", APIS) +def test_segments_api_equivalent(api: Any) -> None: + t = "a: {\n b: 1\n}\n" + assert api.edit_value_segments(t, ["a", "b"], 2) == "a: {\n b: 2\n}\n" + + +@pytest.mark.parametrize("api", APIS) +def test_inline_pair_byte_exact(api: Any) -> None: + assert api.edit_value("a: {x: 1, y: 2}\n", "a.y", 3) == "a: {x: 1, y: 3}\n" + + +@pytest.mark.parametrize("api", APIS) +def test_inline_raw_pair(api: Any) -> None: + assert api.edit_value("a: {k:: v, j: w}\n", "a.k", "z") == "a: {k:: z, j: w}\n" + + +@pytest.mark.parametrize("api", APIS) +def test_inline_nonempty_compound_replacement_inline_exact(api: Any) -> None: + assert api.edit_value("a: {x:1}\n", "a.x", {"n": 1}) == "a: {x:{n: 1}}\n" + + +@pytest.mark.parametrize("api", APIS) +def test_inline_nested_compound_replacement(api: Any) -> None: + assert api.edit_value("a: {x: 1}\n", "a.x", {"n": [{"m": 2}]}) == ("a: {x: {n: [{m: 2}]}}\n") + + +@pytest.mark.parametrize("api", APIS) +def test_scalar_to_compound(api: Any) -> None: + assert api.edit_value("a: 1\n", "a", {"x": 1}) == "a: {\n x: 1\n}\n" + + +@pytest.mark.parametrize("api", APIS) +def test_empty_compound_to_filled(api: Any) -> None: + assert api.edit_value("a: {}\nb: 2\n", "a", {"x": 1}) == "a: {x: 1}\nb: 2\n" + + +@pytest.mark.parametrize("api", APIS) +def test_compound_to_scalar(api: Any) -> None: + assert api.edit_value("a: {\n x: 1\n}\n", "a", 5) == "a: 5\n" + + +@pytest.mark.parametrize("api", APIS) +def test_reopen_leaf_editable_whole_target_unsupported(api: Any) -> None: + t = "a: {}\na.y: 2\n" + assert api.edit_value(t, "a.y", 3) == "a: {}\na.y: 3\n" + with pytest.raises(api.KtavUnsupportedError): + api.edit_value(t, "a", 9) + + +@pytest.mark.parametrize("api", APIS) +def test_multiline_string_replacement(api: Any) -> None: + assert api.edit_value("s: (\n hi\n)\n", "s", "a\nb") == "s: ((\na\nb\n))\n" + + +@pytest.mark.parametrize("api", APIS) +def test_multiline_string_with_terminator_line_unsupported(api: Any) -> None: + with pytest.raises(api.KtavUnsupportedError): + api.edit_value("s: (\n hi\n)\n", "s", "x\n))\ny") + + +@pytest.mark.parametrize("api", APIS) +def test_comments_blanks_and_trailing_bytes_preserved(api: Any) -> None: + t = "## top\n\na: 1\n\n## end\n" + assert api.edit_value(t, "a", 2) == "## top\n\na: 2\n\n## end\n" + + +@pytest.mark.parametrize("api", APIS) +def test_crlf_document_preserved(api: Any) -> None: + t = "a: 1\r\nb: two\r\n" + assert api.edit_value(t, "b", "three") == "a: 1\r\nb: three\r\n" + + +@pytest.mark.parametrize("api", APIS) +def test_cr_only_document_preserved(api: Any) -> None: + t = "a: 1\rb: two\r" + assert api.edit_value(t, "b", "three") == "a: 1\rb: three\r" + + +@pytest.mark.parametrize("api", APIS) +def test_bom_and_no_final_newline_preserved(api: Any) -> None: + t = "a: 1" + assert api.edit_value(t, "a", 2) == "a: 2" + + +@pytest.mark.parametrize("api", APIS) +def test_utf8_values_and_keys(api: Any) -> None: + assert api.edit_value("ключ: значение\n", "ключ", "число") == "ключ: число\n" + + +@pytest.mark.parametrize("api", APIS) +def test_not_found_cases(api: Any) -> None: + t = "a: {\n b: 1\n}\nlist: [\n x\n]\n" + with pytest.raises(api.KtavNotFoundError): + api.edit_value(t, "missing", "v") + with pytest.raises(api.KtavNotFoundError): + api.edit_value(t, "a.b.c", "v") + with pytest.raises(api.KtavNotFoundError): + api.edit_value(t, "list.5", "v") + with pytest.raises(api.KtavNotFoundError): + api.edit_value(t, "a.b.c.d", "v") + with pytest.raises(api.KtavNotFoundError): + api.edit_value("a: scalar\n", "a.b", "v") + with pytest.raises(api.KtavNotFoundError): + api.edit_value_segments(t, [], "v") + + +@pytest.mark.parametrize("api", APIS) +def test_invalid_path_cases(api: Any) -> None: + t = "a: 1\n" + for path, offset in ( + ("a\\nb", 1), + ("a\\", 1), + ("", 0), + ("a..b", 2), + (".a", 0), + ): + with pytest.raises(api.KtavInvalidPathError) as exc_info: + api.edit_value(t, path, "v") + assert exc_info.value.body == str(offset), path + assert exc_info.value.error == "EditInvalidPath" + assert exc_info.value.line is None + assert exc_info.value.span is None + + +@pytest.mark.parametrize("api", APIS) +def test_non_numeric_index_in_array_is_not_found(api: Any) -> None: + with pytest.raises(api.KtavNotFoundError): + api.edit_value_segments("list: [\n x\n]\n", ["list", "abc"], "v") + + +@pytest.mark.parametrize("api", APIS) +def test_leading_zero_index_addresses_the_same_item(api: Any) -> None: + assert api.edit_value("list: [\n one\n two\n]\n", "list.01", "TWO") == ( + "list: [\n one\n TWO\n]\n" + ) + + +@pytest.mark.parametrize("api", APIS) +def test_signed_index_is_not_found(api: Any) -> None: + with pytest.raises(api.KtavNotFoundError): + api.edit_value_segments("list: [\n x\n]\n", ["list", "-1"], "v") + + +@pytest.mark.parametrize("api", APIS) +def test_malformed_source_raises_the_real_source_error(api: Any) -> None: + """0.9 source-first priority: the parser envelope wins over EditUnsupported. + + Both edit entry points validate + parse the source BEFORE the path / + replacement transport decode, so a malformed source carries real + coordinates (line/line_text/span) — never a coordinate-less + edit error. + """ + broken = "a: {\n x: 1\n" + with pytest.raises(api.KtavDecodeError) as exc_info: + api.edit_value(broken, "a.x", "v") + e = exc_info.value + assert e.error == "UnclosedCompound" + assert e.line == 1 + assert isinstance(e.span, dict) + assert e.span["end"] > e.span["start"] + with pytest.raises(api.KtavDecodeError) as seg_exc: + api.edit_value_segments(broken, ["a", "x"], "v") + assert seg_exc.value.error == "UnclosedCompound" + + +@pytest.mark.parametrize("api", APIS) +def test_source_error_beats_path_and_replacement_errors(api: Any) -> None: + broken = "a: {" + # A bad path on an unparseable source still reports the source. + with pytest.raises(api.KtavDecodeError): + api.edit_value(broken, "a\\nb", "v") + # An unencodable replacement never masks the source error either. + with pytest.raises(api.KtavDecodeError): + api.edit_value(broken, "a", object()) + + +@pytest.mark.parametrize("api", APIS) +def test_patched_documents_reparse(api: Any) -> None: + out = api.edit_value("port: 8080\n", "port", 9090) + assert pure.loads(out) == {"port": 9090} + out = api.edit_value("list: [\n one\n]\n", "list.0", "two") + assert pure.loads(out) == {"list": ["two"]} + + +@pytest.mark.parametrize("api", APIS) +def test_pair_spacing_and_eol_preserved(api: Any) -> None: + t = " a:\t old \r\n" + assert api.edit_value(t, "a", "new") == " a:\t new \r\n" + + +@pytest.mark.parametrize("api", APIS) +def test_int_after_raw_drops_only_extra_colon(api: Any) -> None: + assert api.edit_value("k:: 8080\n", "k", 9090) == "k: 9090\n" + + +@pytest.mark.parametrize("api", APIS) +def test_same_value_no_op_is_byte_identical(api: Any) -> None: + t = "a: x\n" + assert api.edit_value(t, "a", "x") == t + + +@pytest.mark.parametrize("api", APIS) +def test_noop_distinguishes_int_from_bool(api: Any) -> None: + # True == 1 in Python; the edit engine must NOT treat them as equal. + t = "a: 1\n" + assert api.edit_value(t, "a", True) == "a:: true\n" + + +@pytest.mark.parametrize("api", APIS) +def test_signed_zero_flip_is_a_real_edit(api: Any) -> None: + """Float equality must preserve signed zero. + + The native core keeps Value::Float as canonical text, where `-0.0` + and `0.0` differ; the pure engine must agree, so a zero sign flip + performs a real splice in both directions instead of a no-op. + """ + assert api.edit_value("n: -0.0\n", "n", 0.0) == "n: 0.0\n" + assert api.edit_value("n: 0.0\n", "n", -0.0) == "n: -0.0\n" + assert api.edit_value_segments("n: -0.0\n", ["n"], 0.0) == "n: 0.0\n" + assert api.edit_value_segments("n: 0.0\n", ["n"], -0.0) == "n: -0.0\n" + + +@pytest.mark.parametrize("api", APIS) +def test_same_sign_zero_noop_keeps_noncanonical_spelling(api: Any) -> None: + for source in ("n: 0.00\n", "n: -0.00\n"): + replacement = -0.0 if source.startswith("n: -") else 0.0 + assert api.edit_value(source, "n", replacement) == source, source + assert api.edit_value_segments(source, ["n"], replacement) == source + + +@pytest.mark.parametrize("api", APIS) +def test_empty_body_pair_replacement(api: Any) -> None: + t = "k: \n" + assert api.edit_value(t, "k", "x") == "k: x \n" + assert api.edit_value(t, "k", "") == "k: \n" + + +@pytest.mark.parametrize("api", APIS) +def test_compound_with_internal_comments_structural_replacement_rejected(api: Any) -> None: + t = "a: {\n ## inner\n x: 1\n}\n" + with pytest.raises(api.KtavUnsupportedError): + api.edit_value(t, "a", {"y": 2}) + + +@pytest.mark.parametrize("api", APIS) +def test_leaf_inside_commented_container_edits_and_keeps_comments(api: Any) -> None: + t = "a: {\n ## inner\n x: 1\n}\n" + assert api.edit_value(t, "a.x", 2) == "a: {\n ## inner\n x: 2\n}\n" + + +@pytest.mark.parametrize("api", APIS) +def test_hash_hash_inside_multiline_string_is_data_not_comment(api: Any) -> None: + t = "a: {\n s: (\n ## not comment\n )\n}\n" + assert api.edit_value(t, "a", 7) == "a: 7\n" + + +@pytest.mark.parametrize("api", APIS) +def test_implicit_root_array_first_item_pair_shaped_gets_raw(api: Any) -> None: + t = ":: old\nsecond\n" + assert api.edit_value(t, "0", "a: b") == ":: a: b\nsecond\n" + + +@pytest.mark.parametrize("api", APIS) +def test_implicit_root_array_second_item_bare_ok(api: Any) -> None: + t = ":: old\nsecond\n" + assert api.edit_value(t, "1", "a: b") == ":: old\na: b\n" + + +@pytest.mark.parametrize("api", APIS) +def test_explicit_root_array_first_item_raw_preserved(api: Any) -> None: + t = "[\n :: old\n x\n]\n" + assert api.edit_value(t, "0", "k: v") == "[\n :: k: v\n x\n]\n" + + +@pytest.mark.parametrize("api", APIS) +def test_root_inline_object_child(api: Any) -> None: + assert api.edit_value("{x: 1}\n", "x", 2) == "{x: 2}\n" + + +@pytest.mark.parametrize("api", APIS) +def test_bigint_replacement_keeps_arbitrary_precision(api: Any) -> None: + big = 123456789012345678901234567890 + out = api.edit_value("n: 1\n", "n", big) + assert out == f"n: {big}\n" + # Beyond i64 the pure parser hands the literal back as a String — + # the point is that it is never coerced to float. + assert pure.loads(out)["n"] == str(big) + assert not isinstance(pure.loads(out)["n"], float) + + +@pytest.mark.parametrize("api", APIS) +def test_source_is_never_mutated(api: Any) -> None: + t = "a: 1\nb: 2\n" + api.edit_value(t, "a", 99) + assert t == "a: 1\nb: 2\n" + + +def test_edit_error_envelope_shape() -> None: + with pytest.raises(pure.KtavEditError) as unsupported: + pure.edit_value("a: {}\na.y: 2\n", "a", 9) + e = unsupported.value + for field in ( + "error", + "reason", + "line", + "line_text", + "span", + "path", + "body", + "canonical", + "spec_section", + "message", + ): + assert hasattr(e, field) + assert e.error == "EditUnsupported" + assert e.reason is None + assert e.line is None + assert e.span is None + assert isinstance(e.body, str) and e.body # the limitation reason + assert str(e).startswith("no lossless source edit: ") + with pytest.raises(pure.KtavNotFoundError) as not_found: + pure.edit_value("a: 1\n", "zzz", 1) + assert not_found.value.error == "EditNotFound" + assert not_found.value.body is None + + +def test_unsupported_replacement_type_is_a_message_envelope() -> None: + with pytest.raises(pure.KtavEncodeError) as exc_info: + pure.edit_value("a: 1\n", "a", object()) + assert exc_info.value.error == "Message" diff --git a/tests/edit/test_native_pin.py b/tests/edit/test_native_pin.py new file mode 100644 index 0000000..9ea5a55 --- /dev/null +++ b/tests/edit/test_native_pin.py @@ -0,0 +1,46 @@ +"""Pin guard: a native extension MUST expose the source-edit 0.9 symbols. + +The edit regression suites quietly restrict themselves to the pure +backend when the extension lacks the edit symbols (pre-pin bootstrap). +After the parent's immutable core pin that degradation would silently +halve coverage, so this guard fails loudly instead: whenever the +compiled extension is importable at all, its edit symbols must be real +— never the loud-stub fallback — and must actually work. + +Skipped ONLY in the pure-mode CI job, where ``ktav._core`` is absent +by construction (conftest enforces that scenario separately). +""" + +from __future__ import annotations + +import pytest + +import ktav + + +@pytest.mark.skipif(not ktav._HAS_CORE, reason="pure-mode job: extension absent") +def test_native_edit_symbols_are_real_after_the_pin() -> None: + for name in ("_edit_value", "_edit_value_segments"): + fn = getattr(ktav, name, None) + assert fn is not None, f"ktav.{name} missing from the module surface" + assert not getattr(fn, "_ktav_stub", False), ( + f"ktav._core does not export the source-edit 0.9 symbol behind" + f" ktav.{name}; the Cargo core pin (immutable rev) has drifted" + " — the edit coverage would silently run pure-only" + ) + + +@pytest.mark.skipif(not ktav._HAS_CORE, reason="pure-mode job: extension absent") +def test_native_edit_works_and_reports_source_errors() -> None: + if getattr(getattr(ktav, "_edit_value", None), "_ktav_stub", False): + pytest.fail( + "native edit symbols are stubs; see test_native_edit_symbols_are_real_after_the_pin" + ) + assert ktav.edit_value("a: 1\n", "a", 2) == "a: 2\n" + assert ktav.edit_value_segments("a: [x]\n", ["a", "0"], "y") == "a: [y]\n" + # 0.9 source-first priority: a malformed source raises the real + # parser envelope, never a coordinate-less edit error. + with pytest.raises(ktav.KtavDecodeError) as exc_info: + ktav.edit_value("a: {", "a", 2) + assert exc_info.value.error == "UnclosedCompound" + assert exc_info.value.line is not None diff --git a/tests/pure/_no_native_probe.py b/tests/pure/_no_native_probe.py index 4d74499..e5a4955 100644 --- a/tests/pure/_no_native_probe.py +++ b/tests/pure/_no_native_probe.py @@ -46,11 +46,18 @@ def find_module(self, fullname, path=None): # type: ignore[no-untyped-def] "dumps_force_strings", "format", "canonical_from_source", + "edit_value", + "edit_value_segments", ): func = getattr(ktav, name) parses_text = name in ("loads", "loads_strict", "format", "canonical_from_source") try: - func("x: 1\n" if parses_text else {}) + if name == "edit_value": + func("a: 1\n", "a", 2) + elif name == "edit_value_segments": + func("a: 1\n", ["a"], 2) + else: + func("x: 1\n" if parses_text else {}) except ImportError as exc: # Public contract (python/ktav/__init__.py _no_native stub): the # message must contain BOTH the import fix and the backend name. @@ -78,6 +85,8 @@ def find_module(self, fullname, path=None): # type: ignore[no-untyped-def] assert ( ktav.pure.canonical_from_source("a: {b: {c: 1}}") == "a: {\n b: {\n c: 1\n }\n}\n" ) +assert ktav.pure.edit_value("a: 1\n", "a", 2) == "a: 2\n" +assert ktav.pure.edit_value_segments("a: [x]\n", ["a", "0"], "y") == "a: [y]\n" with __import__("io").StringIO("a: 1\n") as handle: assert ktav.pure.load(handle) == {"a": 1} diff --git a/tests/pure/test_pure_api.py b/tests/pure/test_pure_api.py index bae56d5..8f5c1c2 100644 --- a/tests/pure/test_pure_api.py +++ b/tests/pure/test_pure_api.py @@ -57,9 +57,15 @@ def test_public_surface_mirrors_the_native_module() -> None: "dumps_force_strings", "format", "canonical_from_source", + "edit_value", + "edit_value_segments", "KtavError", "KtavDecodeError", "KtavEncodeError", + "KtavEditError", + "KtavInvalidPathError", + "KtavNotFoundError", + "KtavUnsupportedError", "__version__", "__spec_version__", ): @@ -99,8 +105,9 @@ def test_strict_lossy_envelope_payloads() -> None: def test_invalid_utf8_bytes_raise_invalidutf8_with_a_nonempty_span() -> None: """§ 6.15: the pure backend validates raw bytes BEFORE any grammar work. - The native wrapper decodes bytes itself (UnicodeDecodeError), so this - byte-level contract is pure-specific by construction. + Since 0.9 the shared byte boundary in ``ktav.__init__`` gives the + native backend the same structured envelope, so the byte-level + contract now holds for both backends. """ raw = b"a: \xff\xfe\n" with pytest.raises(pure.KtavDecodeError) as excinfo: diff --git a/tests/serialize/test_format.py b/tests/serialize/test_format.py index 3d3cd6d..b5c3579 100644 --- a/tests/serialize/test_format.py +++ b/tests/serialize/test_format.py @@ -50,9 +50,12 @@ def test_bytes_input_accepted() -> None: assert ktav.format(b"a: 1\n") == ktav.format("a: 1\n") -def test_invalid_utf8_bytes_raise_unicodedecodeerror() -> None: - with pytest.raises(UnicodeDecodeError): +def test_invalid_utf8_bytes_raise_structured_invalidutf8() -> None: + """0.9: raw bytes are validated before grammar work in every backend.""" + with pytest.raises(ktav.KtavDecodeError) as exc_info: ktav.format(b"\xff\xfe: 1\n") + assert exc_info.value.error == "InvalidUtf8" + assert exc_info.value.line_text is None def test_malformed_input_raises_decode_error() -> None: diff --git a/tests/test_conformance.py b/tests/test_conformance.py index 1e15420..8a93c0c 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -44,6 +44,12 @@ def test_invalid_fixture_is_rejected(be: cr.Backend, ktav_file: Path) -> None: cr.check_invalid_fixture_is_rejected(be, ktav_file) +@pytest.mark.parametrize("be", BACKENDS) +@pytest.mark.parametrize("ktav_file", list(cr.invalid_cases())) +def test_invalid_fixture_expected_location(be: cr.Backend, ktav_file: Path) -> None: + cr.check_invalid_fixture_expected_location(be, ktav_file) + + @pytest.mark.parametrize("be", BACKENDS) @pytest.mark.parametrize(("ktav_file", "json_file"), list(cr.valid_cases())) def test_valid_fixture_roundtrips_through_dump( From a86bfa44ea782149ae78d6e55c0756198bb13785 Mon Sep 17 00:00:00 2001 From: Marat K Date: Sun, 25 Oct 2026 18:42:09 +0100 Subject: [PATCH 2/7] =?UTF-8?q?fix(edit):=20=D0=B8=D1=81=D0=BF=D1=80=D0=B0?= =?UTF-8?q?=D0=B2=D0=B8=D1=82=D1=8C=20Python=20CI=20parser=20=D0=B8=20comp?= =?UTF-8?q?atibility?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Восстановить slot lines для multiline состояния, считать EOF от реального opener и согласовать frame/node kinds позиционного дерева. Исправить Python 3.9 f-string compatibility, backend exception identity, неверные bool/bigint оракулы и lint, включить edit suite в pure-only CI. Оракулы и проверки corpus не ослаблены. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/CI.yml | 5 +- python/ktav/__init__.py | 57 +++++++++++++++------ python/ktav/_core.pyi | 2 +- python/ktav/_pure/__init__.py | 2 +- python/ktav/_pure/_edit_common.py | 14 +++--- python/ktav/_pure/_edit_encode.py | 12 ++--- python/ktav/_pure/_edit_layout.py | 23 +++++---- python/ktav/_pure/_edit_splice.py | 8 +-- python/ktav/_pure/_edit_tree.py | 33 ++++++------ python/ktav/_pure/parser.py | 35 +++++++++---- tests/api/test_error_envelope.py | 5 +- tests/edit/_backends.py | 13 ++++- tests/edit/test_edit_layout.py | 9 ++-- tests/edit/test_edit_value.py | 31 +++++++----- tests/edit/test_native_pin.py | 3 +- tests/parse/test_unclosed_location.py | 72 +++++++++++++++++++++++++++ 16 files changed, 232 insertions(+), 92 deletions(-) create mode 100644 tests/parse/test_unclosed_location.py diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index fa06707..332362b 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -159,7 +159,8 @@ jobs: # Pure-Python backend, no native extension, on the support matrix # edges (oldest and newest CPython, both CI-relevant OSes). Installs # ONLY pytest — no maturin, no Rust toolchain — and runs the - # backend-parametrized conformance runner plus the tests/pure package + # backend-parametrized conformance runner plus the tests/pure and + # tests/edit packages # with ktav._core guaranteed absent (tests/conftest.py fails the job if # it is not). The native-only unit suites (tests/parse, tests/serialize, # tests/roundtrip, tests/api, tests/spec) call the native-default @@ -208,7 +209,7 @@ jobs: - name: Install ONLY pytest + build (no maturin, no Rust) run: python -m pip install --upgrade pip pytest build - name: Pytest (conformance both-backends + pure package, no _core) - run: python -m pytest tests/test_conformance.py tests/pure -v + run: python -m pytest tests/test_conformance.py tests/pure tests/edit -v # Packaged no-native scenario: STAGE only the package files # (python/ktav/**) plus a staged pyproject.toml whose package-dir diff --git a/python/ktav/__init__.py b/python/ktav/__init__.py index 6abb31a..b23db4e 100644 --- a/python/ktav/__init__.py +++ b/python/ktav/__init__.py @@ -169,20 +169,17 @@ def _load_core() -> bool: continue # A stale native binary predating the source-edit 0.9 symbols # must degrade into a LOUD stub, never a silent pure fallback. - try: - globals()[_name] = getattr(_core, _name) - except AttributeError: - if _name in _ERROR_CLASSES: - from ktav import _pure as _puremod - - globals()[_name] = getattr(_puremod, _name) - else: - globals()[_name] = _no_native(_name) + value = getattr(_core, _name, None) + if value is None and _name in _ERROR_CLASSES: + from ktav import _pure as _puremod + + value = getattr(_puremod, _name) + if value is None: + value = _no_native(_name) + globals()[_name] = value for _name in _NATIVE_FUNCS: - try: - globals()["_" + _name] = getattr(_core, _name) - except AttributeError: - globals()["_" + _name] = _no_native(_name) + value = getattr(_core, _name, None) + globals()["_" + _name] = value if value is not None else _no_native(_name) _HAS_CORE = True _CORE_CHECKED = True return _HAS_CORE @@ -227,6 +224,33 @@ def __getattr__(name: str) -> Any: ] +def _reraise_as_backend_error(exc: Any) -> Any: + """Re-raise a pure-backend decode error under the ACTIVE error class. + + The shared byte boundary runs before the native call, so its + :class:`KtavDecodeError` would otherwise be the pure class while the + backend advertised by this module is the native one — and + ``except ktav.KtavDecodeError`` would miss it. The ten-field + envelope is copied verbatim. + """ + cls = globals().get("KtavDecodeError", type(exc)) + err = cls(str(exc)) + for name in ( + "error", + "reason", + "line", + "line_text", + "span", + "path", + "body", + "canonical", + "spec_section", + "message", + ): + setattr(err, name, getattr(exc, name, None)) + raise err from exc + + def _as_text(s: str | bytes | bytearray | memoryview) -> str: """UTF-8 byte boundary for every text entry point (§ 6.15). @@ -238,7 +262,10 @@ def _as_text(s: str | bytes | bytearray | memoryview) -> str: if isinstance(s, (bytes, bytearray, memoryview)): from ktav._pure._text import validate_utf8 - return validate_utf8(bytes(s)) + try: + return validate_utf8(bytes(s)) + except Exception as exc: + _reraise_as_backend_error(exc) return s @@ -338,7 +365,7 @@ def dumps_force_strings(obj: Any) -> str: def edit_value(source: str | bytes | bytearray, path: str, replacement: Any) -> str: - """Replace the value at the textual ``path`` in ``source``. + r"""Replace the value at the textual ``path`` in ``source``. ``path`` is the dotted form (``key1.key2.0.key4``) where ``\\.`` escapes a literal dot and ``\\\\`` a literal backslash; any other diff --git a/python/ktav/_core.pyi b/python/ktav/_core.pyi index a267088..1a2115a 100644 --- a/python/ktav/_core.pyi +++ b/python/ktav/_core.pyi @@ -93,7 +93,7 @@ def format(s: str) -> str: """ def edit_value(source: str, path: str, replacement: Any) -> str: - """Replace the value at the textual ``path`` in ``source``. + r"""Replace the value at the textual ``path`` in ``source``. ``path`` is the dotted form (``key1.key2.0.key4``); ``\\.`` escapes a literal dot, ``\\\\`` a literal backslash. Only the bytes of the diff --git a/python/ktav/_pure/__init__.py b/python/ktav/_pure/__init__.py index 005de7b..9173e7b 100644 --- a/python/ktav/_pure/__init__.py +++ b/python/ktav/_pure/__init__.py @@ -111,7 +111,7 @@ def loads_strict(s: str) -> Any: def edit_value(source: str | bytes | bytearray | memoryview, path: str, replacement: Any) -> str: - """Replace the value at the textual ``path`` in ``source``. + r"""Replace the value at the textual ``path`` in ``source``. ``path`` is the dotted form (``key1.key2.0.key4``) where ``\\.`` escapes a literal dot and ``\\\\`` a literal backslash; any other diff --git a/python/ktav/_pure/_edit_common.py b/python/ktav/_pure/_edit_common.py index 25bd1cb..f6a242e 100644 --- a/python/ktav/_pure/_edit_common.py +++ b/python/ktav/_pure/_edit_common.py @@ -69,7 +69,7 @@ def _unsupported(reason: str) -> KtavEditError: def parse_path(path: str) -> list[str]: - """Split ``key1.key2\\.0`` on unescaped dots; ``\\.``/``\\\\`` decode. + r"""Split ``key1.key2\.0`` on unescaped dots; ``\.``/``\\`` decode. Other escapes, a trailing lone backslash, an empty path, and empty segments raise :class:`KtavInvalidPathError` whose ``body`` is the @@ -239,9 +239,8 @@ def _detect_eol(text: str, pos: int) -> str: i += 1 before = text[:pos] k = max(before.rfind("\n"), before.rfind("\r")) - if k >= 0: - if before[k] == "\r": - return "\r\n" if k + 1 < len(before) and before[k + 1] == "\n" else "\r" + if k >= 0 and before[k] == "\r": + return "\r\n" if k + 1 < len(before) and before[k + 1] == "\n" else "\r" return "\n" @@ -263,8 +262,11 @@ def _physical_line(text: str, pos: int) -> str: def _rust_lines(s: str) -> list[str]: - """Rust ``str::lines``: split on LF, strip one trailing CR per line, - and drop the final empty piece after a trailing LF.""" + """Rust ``str::lines`` semantics. + + Split on LF, strip one trailing CR per line, and drop the final + empty piece after a trailing LF. + """ if not s: return [] lines = s.split("\n") diff --git a/python/ktav/_pure/_edit_encode.py b/python/ktav/_pure/_edit_encode.py index 00e4de9..006730c 100644 --- a/python/ktav/_pure/_edit_encode.py +++ b/python/ktav/_pure/_edit_encode.py @@ -10,16 +10,16 @@ from typing import Any +from ._edit_common import _INDENT_UNIT, _rust_lines, _unsupported from ._inline import MAX_INLINE_DEPTH from ._keys import is_pair_shape from ._numbers import canonical_float from ._text import is_ktav_whitespace -from ._edit_common import _INDENT_UNIT, _rust_lines, _unsupported from .errors import KtavEditError from .writer import ( - _Renderer, _item_needs_raw_marker, _needs_raw_marker, + _Renderer, push_key_segment, ) @@ -146,8 +146,8 @@ class _PlanCtx: "existing_raw", "indent", "inline_compound", - "root_first", "role_item_block", + "root_first", "slot", ) @@ -237,7 +237,7 @@ def _reindent(doc: str, indent: str, eol: str) -> str: return "".join(out) -def _plan_value(replacement: Any, pc: "_PlanCtx") -> tuple[str, bool]: +def _plan_value(replacement: Any, pc: _PlanCtx) -> tuple[str, bool]: if isinstance(replacement, str): return _plan_string(replacement, pc) if replacement is None or isinstance(replacement, (bool, int, float)): @@ -253,7 +253,7 @@ def _plan_value(replacement: Any, pc: "_PlanCtx") -> tuple[str, bool]: return _reindent(_render_compound_body(replacement), pc.indent, pc.eol), False -def _plan_string(s: str, pc: "_PlanCtx") -> tuple[str, bool]: +def _plan_string(s: str, pc: _PlanCtx) -> tuple[str, bool]: if "\r" in s: raise _unsupported("replacement string contains CR (U+000D)") edge_ws = bool(s) and (is_ktav_whitespace(s[0]) or is_ktav_whitespace(s[-1])) @@ -271,7 +271,7 @@ def _plan_string(s: str, pc: "_PlanCtx") -> tuple[str, bool]: def _block_replacement( - replacement: Any, pc: "_PlanCtx", a: int, b0: int, ws_pair: str + replacement: Any, pc: _PlanCtx, a: int, b0: int, ws_pair: str ) -> tuple[int, str, str]: """Returns (from, prefix, body).""" if isinstance(replacement, str): diff --git a/python/ktav/_pure/_edit_layout.py b/python/ktav/_pure/_edit_layout.py index ec5582c..816cfe9 100644 --- a/python/ktav/_pure/_edit_layout.py +++ b/python/ktav/_pure/_edit_layout.py @@ -12,9 +12,6 @@ from typing import Any -from ._inline import MAX_INLINE_DEPTH, ScanError -from ._keys import find_unescaped_colon, process_escapes -from ._text import WS_CHARS, WS_INLINE_CHARS from ._edit_common import ( _INDENT_UNIT, _detect_eol, @@ -30,6 +27,9 @@ _unsupported, ) from ._edit_encode import _encode_inline, _plan_value, _PlanCtx +from ._inline import MAX_INLINE_DEPTH, ScanError +from ._keys import find_unescaped_colon, process_escapes +from ._text import WS_CHARS, WS_INLINE_CHARS from .errors import KtavDecodeError from .parser import parse_document @@ -101,9 +101,13 @@ def _inline_owner(source: str, root: Any, path: list[str], pos: int) -> tuple[in start, end = _slot_body(node.slot) if start <= len(source) and end <= len(source) and start <= end: body = source[start:end] - if node.kind in ("object", "array") and _is_compound(body): - if "\n" not in body and "\r" not in body: - return start, end + if ( + node.kind in ("object", "array") + and _is_compound(body) + and "\n" not in body + and "\r" not in body + ): + return start, end if node.inline: line_start, _line_end, raw_line = _physical_line_span(source, pos) lead = len(raw_line) - len(raw_line.lstrip(WS_CHARS)) @@ -248,9 +252,10 @@ def _standalone( if not (isinstance(parsed, list) and len(parsed) == 1): raise _unsupported("moved inline value is not a single value") value = parsed[0] - if isinstance(value, str): - if "\r" in value or any(ln.strip(" \t") == "))" for ln in _rust_lines(value)): - return None + if isinstance(value, str) and ( + "\r" in value or any(ln.strip(" \t") == "))" for ln in _rust_lines(value)) + ): + return None pc = _PlanCtx( slot="pair" if is_object else "item", role_item_block=False, diff --git a/python/ktav/_pure/_edit_splice.py b/python/ktav/_pure/_edit_splice.py index 3fd3bba..4b4b311 100644 --- a/python/ktav/_pure/_edit_splice.py +++ b/python/ktav/_pure/_edit_splice.py @@ -22,10 +22,10 @@ _unsupported, ) from ._edit_encode import ( - _PlanCtx, _block_replacement, _inline_body, _plan_value, + _PlanCtx, ) from ._edit_layout import _layout_adjust from ._edit_tree import ( @@ -64,11 +64,7 @@ def apply_edit(source: str, segments: list[str], replacement: Any) -> str: ) slot_kind = slot[0] # "pair" | "item" | "block" - if slot_kind == "pair": - body = (slot[2], slot[3]) - anchor = slot[1] - block_item_role = False - elif slot_kind == "item": + if slot_kind == "pair" or slot_kind == "item": body = (slot[2], slot[3]) anchor = slot[1] block_item_role = False diff --git a/python/ktav/_pure/_edit_tree.py b/python/ktav/_pure/_edit_tree.py index 8f736ff..5ce2627 100644 --- a/python/ktav/_pure/_edit_tree.py +++ b/python/ktav/_pure/_edit_tree.py @@ -12,6 +12,12 @@ from typing import Any +from ._edit_common import ( + _inline_segments, + _not_found, + _parse_index, + _unsupported, +) from ._inline import MAX_INLINE_DEPTH from ._keys import ( check_key, @@ -20,12 +26,6 @@ split_key_path, ) from ._text import WS_CHARS, WS_INLINE_CHARS, is_inline_whitespace -from ._edit_common import ( - _inline_segments, - _not_found, - _parse_index, - _unsupported, -) __all__ = [ "_Node", @@ -56,12 +56,12 @@ def __init__( # ("block", anchor|None, start, end) self.slot: tuple[Any, ...] | None = slot - def child(self, key: str) -> "_Node | None": + def child(self, key: str) -> _Node | None: if self.kind == "object": return self.children.get(key) return None - def item(self, i: int) -> "_Node | None": + def item(self, i: int) -> _Node | None: if self.kind == "array": if 0 <= i < len(self.children): return self.children[i] @@ -280,11 +280,9 @@ def terminator(self) -> str: def _build_tree(text: str) -> tuple[_Node, bool]: - skip = 1 if text.startswith("") else 0 + skip = 1 if text.startswith("\ufeff") else 0 builder = _Builder() - line_num = 0 - for line_text, start in _split_lines_char(text[skip:]): - line_num += 1 + for line_num, (line_text, start) in enumerate(_split_lines_char(text[skip:]), start=1): builder.handle_line(line_text, start + skip, line_num) return builder.finish() @@ -522,7 +520,12 @@ def _close_frame( if self.stack and self.stack[-1].kind == "obj": anchor = self.stack[-1].anchor self.stack[-1].anchor = None - node = _Node(frame.kind, frame.table, False, ("block", anchor, frame.range_start, ts_end)) + node = _Node( + "object" if frame.kind == "obj" else "array", + frame.table, + False, + ("block", anchor, frame.range_start, ts_end), + ) parent = self._last_frame() if parent.kind == "obj": key = parent.pending_key @@ -545,7 +548,9 @@ def finish(self) -> tuple[_Node, bool]: if not self.stack: return _Node("object", {}, False, None), self.root_is_explicit frame = self.stack.pop() - return _Node(frame.kind, frame.table, False, None), self.root_is_explicit + # Frame kinds are "obj"/"arr"; node kinds are "object"/"array". + kind = "object" if frame.kind == "obj" else "array" + return _Node(kind, frame.table, False, None), self.root_is_explicit # --------------------------------------------------------------------------- diff --git a/python/ktav/_pure/parser.py b/python/ktav/_pure/parser.py index 6808eff..6a0d1c7 100644 --- a/python/ktav/_pure/parser.py +++ b/python/ktav/_pure/parser.py @@ -211,7 +211,7 @@ def __init__( class _Collecting: """An open multi-line string block (§ 5.6).""" - __slots__ = ("opener_byte", "opener_err_text", "opener_line", "verbatim") + __slots__ = ("lines", "opener_byte", "opener_err_text", "opener_line", "verbatim") def __init__( self, @@ -364,10 +364,12 @@ def _handle_root_line( span: dict[Any, Any], ) -> None: if trimmed == "{": - self._open_root_frame("obj", line_num, line_start, True, err_line) + # The root opener sits at the trimmed-span start, after any + # indent — never at the raw line start. + self._open_root_frame("obj", line_num, span["start"], True, err_line) return if trimmed == "[": - self._open_root_frame("arr", line_num, line_start, True, err_line) + self._open_root_frame("arr", line_num, span["start"], True, err_line) return if trimmed[0] in "{[": value = _inline_gate(trimmed, line_num, err_line, span, 0, self.strict) @@ -462,7 +464,15 @@ def _pair_line( self._attach(value, line_num, err_line, span) return self._hook_pending_item() - self._dispatch_value(after.lstrip(_WS_INLINE), line_num, line_start, err_line, span) + # The opener byte of a nested compound is the first byte of the + # VALUE ('{' / '[' / '('), not the line start — the 0.9 EOF + # diagnostics pin opener..EOF. `span["start"]` is the trimmed + # line's byte start (after any indent). + prefix_len = idx + 1 + (len(after) - len(after.lstrip(_WS_INLINE))) + value_start = span["start"] + byte_len(trimmed[:prefix_len]) + self._dispatch_value( + after.lstrip(_WS_INLINE), line_num, line_start, err_line, span, value_start + ) def _dispatch_value( self, @@ -471,18 +481,19 @@ def _dispatch_value( line_start: int, err_line: str, span: dict[Any, Any], + value_start: int, ) -> None: kind, payload = classify_value_start(body, line_num, err_line, span, self.strict) if kind == "open-obj": - self._push_frame("obj", line_num, line_start, False, err_line) + self._push_frame("obj", line_num, value_start, False, err_line) elif kind == "open-arr": - self._push_frame("arr", line_num, line_start, False, err_line) + self._push_frame("arr", line_num, value_start, False, err_line) elif kind == "open-ml": - self.collecting = _Collecting(False, line_start, line_num, err_line) - self.multiline_opener = line_start + self.collecting = _Collecting(False, value_start, line_num, err_line) + self.multiline_opener = value_start elif kind == "open-mlv": - self.collecting = _Collecting(True, line_start, line_num, err_line) - self.multiline_opener = line_start + self.collecting = _Collecting(True, value_start, line_num, err_line) + self.multiline_opener = value_start else: self._attach(payload, line_num, err_line, span) @@ -511,7 +522,9 @@ def _array_item_line( value, _ = process_escapes(after.strip(_WS_INLINE), line_num, err_line, span) self._attach(value, line_num, err_line, span) return - self._dispatch_value(trimmed, line_num, line_start, err_line, span) + # Array-item lines are already trimmed: the value starts at the + # trimmed-span start (`span["start"]`), not the raw line start. + self._dispatch_value(trimmed, line_num, line_start, err_line, span, span["start"]) # -- frames and attachment -------------------------------------------------------- diff --git a/tests/api/test_error_envelope.py b/tests/api/test_error_envelope.py index 2707f36..f043e23 100644 --- a/tests/api/test_error_envelope.py +++ b/tests/api/test_error_envelope.py @@ -164,12 +164,13 @@ def test_format_errors_carry_envelope_too() -> None: assert hasattr(e, field) # UnclosedCompound is an end-of-input class: since the 0.9 source # diagnostics it pins the OPENER line (line_text populated) and a - # span from the innermost opener to EOF. + # span from the innermost opener to EOF. The opener is the '[' + # itself (byte 3), not the line start. assert e.error == "UnclosedCompound" assert e.line_text == "x: [" assert e.line == 1 assert isinstance(e.span, dict) - assert e.span["start"] == 0 + assert e.span["start"] == 3 assert e.span["end"] == len("x: [") with pytest.raises(ktav.KtavDecodeError) as dup: diff --git a/tests/edit/_backends.py b/tests/edit/_backends.py index 498713c..bc6d544 100644 --- a/tests/edit/_backends.py +++ b/tests/edit/_backends.py @@ -2,11 +2,16 @@ Native and pure MUST agree byte-for-byte on every edit regression, so both participate in the same oracle-driven cases whenever the compiled -extension exposes the source-edit 0.9 symbols. +extension exposes the source-edit 0.9 symbols. The restriction to the +pure backend is legitimate ONLY in the explicitly pure-mode CI job +(``KTAV_REQUIRE_PURE=1``, extension absent by construction); in a +native job (``KTAV_REQUIRE_NATIVE=1``) a missing/stubbed symbol is a +loud failure here, never a silent skip. """ from __future__ import annotations +import os from typing import Any import ktav @@ -18,4 +23,10 @@ def edit_apis() -> list[Any]: native_edit = getattr(ktav, "_edit_value", None) if ktav._HAS_CORE and native_edit is not None and not getattr(native_edit, "_ktav_stub", False): apis.insert(0, ktav) + if os.environ.get("KTAV_REQUIRE_NATIVE") == "1" and len(apis) == 1: + raise RuntimeError( + "KTAV_REQUIRE_NATIVE job: the compiled extension does not expose" + " the source-edit 0.9 edit symbols; native edit coverage would" + " be silently skipped (see tests/edit/test_native_pin.py)" + ) return apis diff --git a/tests/edit/test_edit_layout.py b/tests/edit/test_edit_layout.py index 8c8dd5e..69d01f1 100644 --- a/tests/edit/test_edit_layout.py +++ b/tests/edit/test_edit_layout.py @@ -11,9 +11,8 @@ from typing import Any -import pytest - import ktav.pure as pure +import pytest from tests.edit._backends import edit_apis APIS = edit_apis() @@ -104,7 +103,8 @@ def test_dense_escapes_unfold_but_sparse_escapes_do_not(api: Any) -> None: check(api, "a: [old, keep]\n", ["a", "0"], "a,b,c,d,e", "a: [\n a,b,c,d,e\n keep\n]\n") check(api, "a: [old]\n", ["a", "0"], "a,b,c,d", "a: [a\\,b\\,c\\,d]\n") sparse = "x" * 30 + "a,b,c,d,e" - check(api, "a: [old]\n", ["a", "0"], sparse, f"a: [{sparse.replace(',', '\\,')}]\n") + escaped = sparse.replace(",", "\\,") + check(api, "a: [old]\n", ["a", "0"], sparse, f"a: [{escaped}]\n") @pytest.mark.parametrize("api", APIS) @@ -133,7 +133,8 @@ def test_escape_overhead_boundary_is_exact(api: Any) -> None: dense = "x" * 7 + "a,b,c,d,e" sparse = "x" * 8 + "a,b,c,d,e" check(api, "a: [old]\n", ["a", "0"], dense, f"a: [\n {dense}\n]\n") - check(api, "a: [old]\n", ["a", "0"], sparse, f"a: [{sparse.replace(',', '\\,')}]\n") + escaped = sparse.replace(",", "\\,") + check(api, "a: [old]\n", ["a", "0"], sparse, f"a: [{escaped}]\n") @pytest.mark.parametrize("api", APIS) diff --git a/tests/edit/test_edit_value.py b/tests/edit/test_edit_value.py index 538e63b..5156534 100644 --- a/tests/edit/test_edit_value.py +++ b/tests/edit/test_edit_value.py @@ -11,9 +11,8 @@ from typing import Any -import pytest - import ktav.pure as pure +import pytest from tests.edit._backends import edit_apis APIS = edit_apis() @@ -311,9 +310,13 @@ def test_same_value_no_op_is_byte_identical(api: Any) -> None: @pytest.mark.parametrize("api", APIS) def test_noop_distinguishes_int_from_bool(api: Any) -> None: - # True == 1 in Python; the edit engine must NOT treat them as equal. - t = "a: 1\n" - assert api.edit_value(t, "a", True) == "a:: true\n" + """True == 1 in Python; the edit engine must NOT treat them as equal. + + Oracle per the Rust reference suite: a Bool replacement stays a + plain bare scalar (`a: true`) — the raw `::` marker is reserved for + String provenance, and a raw marker here would re-parse as String. + """ + assert api.edit_value("a: 1\n", "a", True) == "a: true\n" @pytest.mark.parametrize("api", APIS) @@ -388,14 +391,17 @@ def test_root_inline_object_child(api: Any) -> None: @pytest.mark.parametrize("api", APIS) -def test_bigint_replacement_keeps_arbitrary_precision(api: Any) -> None: - big = 123456789012345678901234567890 +def test_integer_precision_is_never_coerced_to_float(api: Any) -> None: + # 2**53 + 1 is exactly representable in i64 but NOT in f64: the edit + # must keep it an exact integer literal, never a float spelling. + big = 9007199254740993 # 2**53 + 1 out = api.edit_value("n: 1\n", "n", big) assert out == f"n: {big}\n" - # Beyond i64 the pure parser hands the literal back as a String — - # the point is that it is never coerced to float. - assert pure.loads(out)["n"] == str(big) - assert not isinstance(pure.loads(out)["n"], float) + parsed = pure.loads(out)["n"] + assert parsed == big + assert isinstance(parsed, int) and not isinstance(parsed, bool) + # The float spelling would round-trip to a different value. + assert f"{big}.0" not in out and "e" not in out.lower() @pytest.mark.parametrize("api", APIS) @@ -426,7 +432,8 @@ def test_edit_error_envelope_shape() -> None: assert e.reason is None assert e.line is None assert e.span is None - assert isinstance(e.body, str) and e.body # the limitation reason + assert isinstance(e.body, str) # the limitation reason + assert e.body # nonempty assert str(e).startswith("no lossless source edit: ") with pytest.raises(pure.KtavNotFoundError) as not_found: pure.edit_value("a: 1\n", "zzz", 1) diff --git a/tests/edit/test_native_pin.py b/tests/edit/test_native_pin.py index 9ea5a55..938010f 100644 --- a/tests/edit/test_native_pin.py +++ b/tests/edit/test_native_pin.py @@ -13,9 +13,8 @@ from __future__ import annotations -import pytest - import ktav +import pytest @pytest.mark.skipif(not ktav._HAS_CORE, reason="pure-mode job: extension absent") diff --git a/tests/parse/test_unclosed_location.py b/tests/parse/test_unclosed_location.py new file mode 100644 index 0000000..f744f60 --- /dev/null +++ b/tests/parse/test_unclosed_location.py @@ -0,0 +1,72 @@ +"""UnclosedCompound span start must be the REAL opener byte (0.9). + +Regression for the diagnostics09 `eof_nested` / `eof_*` class: the span +starts at the innermost opener (`{` / `[` / `(` / `((`), after any +indent and after the `key:` separator — never at the pair/line start. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +import ktav +import ktav.pure as pure + + +def _apis() -> list[Any]: + """pure always; native only when its symbols really exist. + + In the explicitly pure-mode CI job the compiled extension is absent + by construction, so only the pure backend is parametrized there. + """ + apis: list[Any] = [pure] + native_loads = getattr(ktav, "_loads", None) + if ( + ktav._HAS_CORE + and native_loads is not None + and not getattr(native_loads, "_ktav_stub", False) + ): + apis.insert(0, ktav) + return apis + + +APIS = _apis() + + +@pytest.mark.parametrize("api", APIS) +def test_nested_block_opener_offset(api: Any) -> None: + source = "a: {\n b: {\n" + with pytest.raises(api.KtavDecodeError) as exc_info: + api.loads(source) + e = exc_info.value + assert e.error == "UnclosedCompound" + assert e.line == 2, e.line + assert e.line_text == " b: {" + assert e.span is not None + assert e.span["start"] == 10, e.span # the inner '{', after ' b: ' + assert e.span["end"] == len(source.encode("utf-8")) + + +@pytest.mark.parametrize("api", APIS) +def test_multiline_opener_offset(api: Any) -> None: + source = "k: (\n hi\n" + with pytest.raises(api.KtavDecodeError) as exc_info: + api.loads(source) + e = exc_info.value + assert e.error == "UnclosedCompound" + assert e.line == 1 + assert e.span is not None + assert e.span["start"] == 3, e.span # the '(', right after 'k: ' + + +@pytest.mark.parametrize("api", APIS) +def test_explicit_root_opener_offset_with_indent(api: Any) -> None: + source = " {\n" + with pytest.raises(api.KtavDecodeError) as exc_info: + api.loads(source) + e = exc_info.value + assert e.error == "UnclosedCompound" + assert e.span is not None + assert e.span["start"] == 2, e.span From cf679b227ce0865ae172457e8530660254314886 Mon Sep 17 00:00:00 2001 From: Marat K Date: Sun, 25 Oct 2026 19:55:04 +0100 Subject: [PATCH 3/7] =?UTF-8?q?fix(edit):=20=D0=B8=D1=81=D0=BF=D1=80=D0=B0?= =?UTF-8?q?=D0=B2=D0=B8=D1=82=D1=8C=20=D0=BF=D1=83=D1=82=D0=B8=20=D0=B8=20?= =?UTF-8?q?inline=20spans=20=D0=B2=20Python?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Сохранить хвост escaped segment и считать inline value offsets после separator. Исправить compound rendering, пустой segments path и CRLF при раскрытии. Не пропускать внутренние lookup errors через semantic guard. Co-Authored-By: Claude Opus 5.5 --- python/ktav/_pure/__init__.py | 6 ++++++ python/ktav/_pure/_edit_common.py | 15 +++++++++++++-- python/ktav/_pure/_edit_encode.py | 2 +- python/ktav/_pure/_edit_splice.py | 14 ++++++++++---- python/ktav/_pure/_edit_tree.py | 19 ++++++++----------- tests/edit/test_edit_layout.py | 1 + tests/edit/test_edit_value.py | 25 +++++++++++++++++++++++-- tests/parse/test_unclosed_location.py | 3 +-- 8 files changed, 63 insertions(+), 22 deletions(-) diff --git a/python/ktav/_pure/__init__.py b/python/ktav/_pure/__init__.py index 9173e7b..679a615 100644 --- a/python/ktav/_pure/__init__.py +++ b/python/ktav/_pure/__init__.py @@ -161,6 +161,12 @@ def edit_value_segments( for segment in decoded: if not isinstance(segment, str): raise KtavEditError("edit path segments must be strings", error="Message") + if not decoded: + # An empty segments list addresses nothing — NotFound, matching + # the segments API contract (never a noncontiguous Unsupported). + raise KtavNotFoundError( + "edit path does not address an existing value", error="EditNotFound" + ) return apply_edit(text, decoded, to_value(replacement)) diff --git a/python/ktav/_pure/_edit_common.py b/python/ktav/_pure/_edit_common.py index f6a242e..ff6e1ae 100644 --- a/python/ktav/_pure/_edit_common.py +++ b/python/ktav/_pure/_edit_common.py @@ -115,6 +115,12 @@ def parse_path(path: str) -> list[str]: body=str(byte), ) else: + # Once an escape switched the segment into accumulate mode, + # every following plain char must be appended too (mirrors + # the Rust `cur.push_str` in the default branch) — otherwise + # `a\.b` would decode to "a." and lose the trailing text. + if cur is not None: + cur.append(c) i += 1 byte += len(c.encode("utf-8")) if seg_start == n and cur is None: @@ -239,8 +245,13 @@ def _detect_eol(text: str, pos: int) -> str: i += 1 before = text[:pos] k = max(before.rfind("\n"), before.rfind("\r")) - if k >= 0 and before[k] == "\r": - return "\r\n" if k + 1 < len(before) and before[k + 1] == "\n" else "\r" + if k >= 0: + if before[k] == "\r": + return "\r\n" if k + 1 < len(before) and before[k + 1] == "\n" else "\r" + # A trailing LF belongs to a CRLF pair when a CR precedes it + # (Rust: `i > 0 && bytes[i - 1] == b'\r'`). + if k > 0 and before[k - 1] == "\r": + return "\r\n" return "\n" diff --git a/python/ktav/_pure/_edit_encode.py b/python/ktav/_pure/_edit_encode.py index 006730c..14c64ca 100644 --- a/python/ktav/_pure/_edit_encode.py +++ b/python/ktav/_pure/_edit_encode.py @@ -199,7 +199,7 @@ def _render_compound_body(v: Any) -> str: if not v: return "{}\n" out.append("{\n") - renderer.render_object_body(v, 1) + renderer.render_object_body(v, 1, False) out.append("}\n") elif isinstance(v, list): if not v: diff --git a/python/ktav/_pure/_edit_splice.py b/python/ktav/_pure/_edit_splice.py index 4b4b311..bec65ae 100644 --- a/python/ktav/_pure/_edit_splice.py +++ b/python/ktav/_pure/_edit_splice.py @@ -150,10 +150,16 @@ def apply_edit(source: str, segments: list[str], replacement: Any) -> str: new_doc = parse_document(result) except KtavDecodeError as exc: raise _unsupported(f"patch produced an invalid document: {exc}") from None - actual = _take_target(new_doc, segments) - target_matches = _same(actual, replacement) - expected = original - _take_target(expected, segments) + try: + actual = _take_target(new_doc, segments) + target_matches = _same(actual, replacement) + expected = original + _take_target(expected, segments) + except (KeyError, IndexError, TypeError): + # The patch re-parsed but no longer carries the target path — a + # semantic mismatch, reported as the typed edit error, never as + # an internal KeyError. + raise _unsupported("semantic mismatch after splice; no lossless patch produced") from None if not target_matches or not _same(new_doc, expected): raise _unsupported("semantic mismatch after splice; no lossless patch produced") return result diff --git a/python/ktav/_pure/_edit_tree.py b/python/ktav/_pure/_edit_tree.py index 5ce2627..dd32010 100644 --- a/python/ktav/_pure/_edit_tree.py +++ b/python/ktav/_pure/_edit_tree.py @@ -102,20 +102,17 @@ def _position_inline( key = trimmed[:colon] after = trimmed[colon + 1 :] if after.startswith(":"): - rest_raw = after[1:] - v = rest_raw.lstrip(WS_INLINE_CHARS) + # Raw `::`: body = second colon + whitespace run + value; + # the raw body is literal String data, never a compound. + rest = after[1:] + v = rest.lstrip(WS_INLINE_CHARS) + body_abs = seg_base + colon + 2 + (len(rest) - len(v)) node = _Node("scalar", None, True, None) - node.slot = ( - "pair", - colon_abs, - seg_base + colon + 1 + (len(rest_raw) - len(v)), - seg_end, - ) else: v = after.lstrip(WS_INLINE_CHARS) - value_abs = seg_base + (len(after) - len(v)) - node = _inline_child(v, value_abs, line_num, line_text, span, depth) - node.slot = ("pair", colon_abs, value_abs, seg_end) + body_abs = seg_base + colon + 1 + (len(after) - len(v)) + node = _inline_child(v, body_abs, line_num, line_text, span, depth) + node.slot = ("pair", colon_abs, body_abs, seg_end) _insert_leaf_value(table, key, node, line_num, line_text, span) return _Node("object", table, True, None) items: list[_Node] = [] diff --git a/tests/edit/test_edit_layout.py b/tests/edit/test_edit_layout.py index 69d01f1..9e6e453 100644 --- a/tests/edit/test_edit_layout.py +++ b/tests/edit/test_edit_layout.py @@ -13,6 +13,7 @@ import ktav.pure as pure import pytest + from tests.edit._backends import edit_apis APIS = edit_apis() diff --git a/tests/edit/test_edit_value.py b/tests/edit/test_edit_value.py index 5156534..84134de 100644 --- a/tests/edit/test_edit_value.py +++ b/tests/edit/test_edit_value.py @@ -13,6 +13,7 @@ import ktav.pure as pure import pytest + from tests.edit._backends import edit_apis APIS = edit_apis() @@ -399,9 +400,11 @@ def test_integer_precision_is_never_coerced_to_float(api: Any) -> None: assert out == f"n: {big}\n" parsed = pure.loads(out)["n"] assert parsed == big - assert isinstance(parsed, int) and not isinstance(parsed, bool) + assert isinstance(parsed, int) + assert not isinstance(parsed, bool) # The float spelling would round-trip to a different value. - assert f"{big}.0" not in out and "e" not in out.lower() + assert f"{big}.0" not in out + assert "e" not in out.lower() @pytest.mark.parametrize("api", APIS) @@ -445,3 +448,21 @@ def test_unsupported_replacement_type_is_a_message_envelope() -> None: with pytest.raises(pure.KtavEncodeError) as exc_info: pure.edit_value("a: 1\n", "a", object()) assert exc_info.value.error == "Message" + + +def test_parse_path_keeps_plain_text_after_escapes() -> None: + """Regression: plain chars following an escape must survive decoding. + + `a\\.b` once decoded to "a." because plain characters were never + appended to the accumulator once an escape switched the segment + into accumulate mode. + """ + from ktav._pure._edit import parse_path + + assert parse_path("a") == ["a"] + assert parse_path("a.b") == ["a", "b"] + assert parse_path("a\\.b") == ["a.b"] + assert parse_path("a\\.b.c") == ["a.b", "c"] + assert parse_path("x\\.y\\.z") == ["x.y.z"] + assert parse_path("a\\\\b") == ["a\\b"] + assert parse_path("pre\\\\post\\.tail") == ["pre\\post.tail"] diff --git a/tests/parse/test_unclosed_location.py b/tests/parse/test_unclosed_location.py index f744f60..e90dac3 100644 --- a/tests/parse/test_unclosed_location.py +++ b/tests/parse/test_unclosed_location.py @@ -9,10 +9,9 @@ from typing import Any -import pytest - import ktav import ktav.pure as pure +import pytest def _apis() -> list[Any]: From 3189323c1c6355a9fe039c094bd6ce06221a3d7e Mon Sep 17 00:00:00 2001 From: Marat K Date: Sun, 25 Oct 2026 20:14:41 +0100 Subject: [PATCH 4/7] =?UTF-8?q?fix(edit):=20=D0=B2=D1=8B=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=BD=D1=8F=D1=82=D1=8C=20Python=20rendering=20=D0=B8=20error?= =?UTF-8?q?=20boundary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Сохранить уровень отступа compound replacement и явно сузить типы editor nodes. Строить Message под активным классом без несуществующих keyword constructors; сохранять source-first и однократное чтение segments iterator. Co-Authored-By: Claude Opus 5.5 --- python/ktav/__init__.py | 35 ++++++++++-- python/ktav/_pure/_edit_encode.py | 21 +++---- python/ktav/_pure/_edit_layout.py | 12 ++-- python/ktav/_pure/_edit_tree.py | 20 ++++--- tests/edit/test_public_segments_api.py | 78 ++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 31 deletions(-) create mode 100644 tests/edit/test_public_segments_api.py diff --git a/python/ktav/__init__.py b/python/ktav/__init__.py index b23db4e..362a4d6 100644 --- a/python/ktav/__init__.py +++ b/python/ktav/__init__.py @@ -21,7 +21,7 @@ from __future__ import annotations -from typing import IO, TYPE_CHECKING, Any +from typing import IO, TYPE_CHECKING, Any, NoReturn if TYPE_CHECKING: from ktav._core import ( @@ -224,7 +224,7 @@ def __getattr__(name: str) -> Any: ] -def _reraise_as_backend_error(exc: Any) -> Any: +def _reraise_as_backend_error(exc: Any) -> NoReturn: """Re-raise a pure-backend decode error under the ACTIVE error class. The shared byte boundary runs before the native call, so its @@ -384,6 +384,25 @@ def edit_value(source: str | bytes | bytearray, path: str, replacement: Any) -> return _edit_value(_as_text(source), path, replacement) +def _backend_message_error(message: str) -> Any: + """Build the ACTIVE backend's Message exception: one message arg, + then the ten envelope fields set explicitly (native create_exception! + classes have no custom __init__ — kwargs would be a TypeError).""" + cls = globals().get("KtavEditError", KtavEditError) + err = cls(message) + err.error = "Message" + err.reason = None + err.line = None + err.line_text = None + err.span = None + err.path = None + err.body = None + err.canonical = None + err.spec_section = None + err.message = message + return err + + def edit_value_segments(source: str | bytes | bytearray, segments: Any, replacement: Any) -> str: """:func:`edit_value` with already-decoded literal path segments. @@ -394,9 +413,15 @@ def edit_value_segments(source: str | bytes | bytearray, segments: Any, replacem :class:`KtavNotFoundError`. """ _load_core() - if not all(isinstance(segment, str) for segment in segments): - raise KtavEditError("edit path segments must be strings", error="Message") - return _edit_value_segments(_as_text(source), list(segments), replacement) + decoded = list(segments) # materialize once — segments may be a generator + text = _as_text(source) + if any(not isinstance(segment, str) for segment in decoded): + # Source first: on a broken source the real parser envelope wins + # over the transport error; the whole parse runs only on this + # rejected path (the entry op parses on the accepted path). + loads(text) + raise _backend_message_error("edit path segments must be strings") + return _edit_value_segments(text, decoded, replacement) def load(fp: IO[Any]) -> Any: diff --git a/python/ktav/_pure/_edit_encode.py b/python/ktav/_pure/_edit_encode.py index 14c64ca..7156b69 100644 --- a/python/ktav/_pure/_edit_encode.py +++ b/python/ktav/_pure/_edit_encode.py @@ -192,22 +192,19 @@ def _verbatim_block_body(s: str, indent: str, eol: str) -> str: def _render_compound_body(v: Any) -> str: + # Mirror Rust `render_compound_item`: render the container as an + # array ITEM at indent level 1 — braces and closer at 4 spaces, the + # body at 8 — so `_reindent` strips exactly one level before + # applying the target line indent. out: list[str] = [] renderer = _Renderer(False, False, out) try: - if isinstance(v, dict): - if not v: - return "{}\n" - out.append("{\n") - renderer.render_object_body(v, 1, False) - out.append("}\n") + if isinstance(v, (dict, list)) and v: + renderer.render_item(v, 1, False) + elif isinstance(v, dict): + out.append("{}\n") elif isinstance(v, list): - if not v: - return "[]\n" - out.append("[\n") - for item in v: - renderer.render_item(item, 1, False) - out.append("]\n") + out.append("[]\n") else: raise _unsupported("internal: render_compound_body on a scalar") except KtavEditError: diff --git a/python/ktav/_pure/_edit_layout.py b/python/ktav/_pure/_edit_layout.py index 816cfe9..5546f63 100644 --- a/python/ktav/_pure/_edit_layout.py +++ b/python/ktav/_pure/_edit_layout.py @@ -113,18 +113,18 @@ def _inline_owner(source: str, root: Any, path: list[str], pos: int) -> tuple[in lead = len(raw_line) - len(raw_line.lstrip(WS_CHARS)) trimmed_line = raw_line[lead:].lstrip("") base_shift = lead + (len(raw_line) - lead - len(trimmed_line)) - body: str | None = None + tail_body: str | None = None if _is_compound(trimmed_line): - body = trimmed_line + tail_body = trimmed_line else: colon = find_unescaped_colon(trimmed_line) if colon is not None: after_c = trimmed_line[colon + 1 :] - body = after_c.lstrip(WS_INLINE_CHARS) - base_shift += colon + 1 + (len(after_c) - len(body)) - if body is not None and _is_compound(body): + tail_body = after_c.lstrip(WS_INLINE_CHARS) + base_shift += colon + 1 + (len(after_c) - len(tail_body)) + if tail_body is not None and _is_compound(tail_body): base = line_start + base_shift - return base, base + len(body) + return base, base + len(tail_body) return None diff --git a/python/ktav/_pure/_edit_tree.py b/python/ktav/_pure/_edit_tree.py index dd32010..2ff026e 100644 --- a/python/ktav/_pure/_edit_tree.py +++ b/python/ktav/_pure/_edit_tree.py @@ -57,12 +57,12 @@ def __init__( self.slot: tuple[Any, ...] | None = slot def child(self, key: str) -> _Node | None: - if self.kind == "object": + if self.kind == "object" and isinstance(self.children, dict): return self.children.get(key) return None def item(self, i: int) -> _Node | None: - if self.kind == "array": + if self.kind == "array" and isinstance(self.children, list): if 0 <= i < len(self.children): return self.children[i] return None @@ -477,19 +477,19 @@ def _set_pending(self, key: str, anchor: int | None) -> None: def _insert_pair(self, key: str, node: _Node, line_num: int, line_text: str) -> None: frame = self._last_frame() - if frame.kind != "obj": + if frame.kind != "obj" or not isinstance(frame.table, dict): raise _unsupported("internal: pair outside object frame") _insert_leaf_value(frame.table, key, node, line_num, line_text, {"start": 0, "end": 0}) def _push_item(self, node: _Node) -> None: frame = self._last_frame() - if frame.kind != "arr": + if frame.kind != "arr" or not isinstance(frame.table, list): raise _unsupported("internal: item outside array frame") frame.table.append(node) def _attach_scalar(self, node: _Node, line_num: int, line_start: int, line_text: str) -> None: frame = self._last_frame() - if frame.kind == "obj": + if frame.kind == "obj" and isinstance(frame.table, dict): key = frame.pending_key if key is None: raise _unsupported("internal: multiline closed without pending key") @@ -499,8 +499,10 @@ def _attach_scalar(self, node: _Node, line_num: int, line_start: int, line_text: if node.slot is not None and node.slot[0] == "block": node.slot = ("block", anchor, node.slot[2], node.slot[3]) _insert_leaf_value(frame.table, key, node, line_num, line_text, {"start": 0, "end": 0}) - else: + elif isinstance(frame.table, list): frame.table.append(node) + else: + raise _unsupported("internal: multiline attach outside a frame") def _close_frame( self, expected: str, line_num: int, line_text: str, ts_start: int, ts_end: int @@ -524,14 +526,16 @@ def _close_frame( ("block", anchor, frame.range_start, ts_end), ) parent = self._last_frame() - if parent.kind == "obj": + if parent.kind == "obj" and isinstance(parent.table, dict): key = parent.pending_key if key is None: raise _unsupported("internal: closed compound without pending key") parent.pending_key = None _insert_leaf_value(parent.table, key, node, line_num, line_text, {"start": 0, "end": 0}) - else: + elif isinstance(parent.table, list): parent.table.append(node) + else: + raise _unsupported("internal: close without a parent frame") # -- finish ------------------------------------------------------------------- diff --git a/tests/edit/test_public_segments_api.py b/tests/edit/test_public_segments_api.py new file mode 100644 index 0000000..bdf051d --- /dev/null +++ b/tests/edit/test_public_segments_api.py @@ -0,0 +1,78 @@ +"""Public root-API `edit_value_segments` contract (both backends). + +Covers the Python-level wrapper semantics that the pure and native +entry ops do not see themselves: + +- source-first: a broken source wins over a bad segments transport; +- non-string segments on a VALID source are the active backend's + typed Message error with all-null positions; +- generator segments are materialized once (never drained twice). +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from tests.edit._backends import edit_apis + +APIS = edit_apis() + +FIELDS = ( + "error", + "reason", + "line", + "line_text", + "span", + "path", + "body", + "canonical", + "spec_section", + "message", +) + + +@pytest.mark.parametrize("api", APIS) +def test_valid_source_nonstr_segments_are_a_typed_message(api: Any) -> None: + with pytest.raises(api.KtavEditError) as exc_info: + segments: Any = ["a", 0] + api.edit_value_segments("a: 1\n", segments, 2) + e = exc_info.value + for field in FIELDS: + assert hasattr(e, field) + assert e.error == "Message" + assert e.message == "edit path segments must be strings" + assert str(e) == e.message + # A transport error never invents source positions. + assert e.reason is None + assert e.line is None + assert e.line_text is None + assert e.span is None + assert e.path is None + assert e.body is None + assert e.canonical is None + assert e.spec_section is None + # Identity: raised under the ACTIVE backend's class. + assert isinstance(e, api.KtavEditError) + + +@pytest.mark.parametrize("api", APIS) +def test_broken_source_beats_nonstr_segments(api: Any) -> None: + with pytest.raises(api.KtavDecodeError) as exc_info: + segments: Any = ["a", 0] + api.edit_value_segments("a: {", segments, 2) + e = exc_info.value + assert e.error == "UnclosedCompound" + assert e.line == 1 + assert e.span is not None + assert e.span["end"] > e.span["start"] + + +@pytest.mark.parametrize("api", APIS) +def test_generator_segments_are_materialized_once(api: Any) -> None: + gen = iter(["a", "0"]) + out = api.edit_value_segments("a: [old]\n", gen, "new") + assert out == "a: [new]\n" + # The generator was consumed exactly once by the wrapper. + assert list(gen) == [] From c95721a6138e06e6bf3f37e576b306c3838b23b0 Mon Sep 17 00:00:00 2001 From: Marat K Date: Sun, 25 Oct 2026 20:19:38 +0100 Subject: [PATCH 5/7] =?UTF-8?q?fix(test):=20=D0=B8=D1=81=D0=BF=D1=80=D0=B0?= =?UTF-8?q?=D0=B2=D0=B8=D1=82=D1=8C=20Ruff=20=D0=B7=D0=B0=D0=BC=D0=B5?= =?UTF-8?q?=D1=87=D0=B0=D0=BD=D0=B8=D1=8F=20=D0=BA=20segments=20regression?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Разместить setup вне pytest.raises и сократить docstring до одной строки без отключения lint rules. Co-Authored-By: Claude Opus 5.5 --- python/ktav/__init__.py | 4 +--- tests/edit/test_public_segments_api.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/python/ktav/__init__.py b/python/ktav/__init__.py index 362a4d6..5513d15 100644 --- a/python/ktav/__init__.py +++ b/python/ktav/__init__.py @@ -385,9 +385,7 @@ def edit_value(source: str | bytes | bytearray, path: str, replacement: Any) -> def _backend_message_error(message: str) -> Any: - """Build the ACTIVE backend's Message exception: one message arg, - then the ten envelope fields set explicitly (native create_exception! - classes have no custom __init__ — kwargs would be a TypeError).""" + """Создать Message-конверт под активным классом backend.""" cls = globals().get("KtavEditError", KtavEditError) err = cls(message) err.error = "Message" diff --git a/tests/edit/test_public_segments_api.py b/tests/edit/test_public_segments_api.py index bdf051d..36b5eaf 100644 --- a/tests/edit/test_public_segments_api.py +++ b/tests/edit/test_public_segments_api.py @@ -35,8 +35,8 @@ @pytest.mark.parametrize("api", APIS) def test_valid_source_nonstr_segments_are_a_typed_message(api: Any) -> None: + segments: Any = ["a", 0] with pytest.raises(api.KtavEditError) as exc_info: - segments: Any = ["a", 0] api.edit_value_segments("a: 1\n", segments, 2) e = exc_info.value for field in FIELDS: @@ -59,8 +59,8 @@ def test_valid_source_nonstr_segments_are_a_typed_message(api: Any) -> None: @pytest.mark.parametrize("api", APIS) def test_broken_source_beats_nonstr_segments(api: Any) -> None: + segments: Any = ["a", 0] with pytest.raises(api.KtavDecodeError) as exc_info: - segments: Any = ["a", 0] api.edit_value_segments("a: {", segments, 2) e = exc_info.value assert e.error == "UnclosedCompound" From ce599f431353e35fdbf342898a36fd4a7af90129 Mon Sep 17 00:00:00 2001 From: Marat K Date: Sun, 25 Oct 2026 20:24:32 +0100 Subject: [PATCH 6/7] =?UTF-8?q?fix(types):=20=D1=83=D0=BA=D0=B0=D0=B7?= =?UTF-8?q?=D0=B0=D1=82=D1=8C=20=D1=82=D0=BE=D1=87=D0=BD=D1=8B=D0=B9=20?= =?UTF-8?q?=D1=82=D0=B8=D0=BF=20children=20editor=20node?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Убрать Any на входе Node, чтобы list lookup возвращал Node, а не скрывал тип через cast или suppression. Co-Authored-By: Claude Opus 5.5 --- python/ktav/_pure/_edit_tree.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/ktav/_pure/_edit_tree.py b/python/ktav/_pure/_edit_tree.py index 2ff026e..a51c683 100644 --- a/python/ktav/_pure/_edit_tree.py +++ b/python/ktav/_pure/_edit_tree.py @@ -45,7 +45,11 @@ class _Node: __slots__ = ("children", "inline", "kind", "slot") def __init__( - self, kind: str, children: Any, inline: bool, slot: tuple[Any, ...] | None + self, + kind: str, + children: dict[str, _Node] | list[_Node] | None, + inline: bool, + slot: tuple[Any, ...] | None, ) -> None: self.kind = kind # "scalar" | "object" | "array" self.children = children # dict[str, Node] | list[Node] | None From 62ec55b865af4af208e6f3c914ceffecfcc9881e Mon Sep 17 00:00:00 2001 From: Marat K Date: Sun, 25 Oct 2026 20:30:20 +0100 Subject: [PATCH 7/7] =?UTF-8?q?fix(types):=20=D1=81=D1=83=D0=B7=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20children=20=D0=B2=20editor=20traversals?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Явно различать dict/list перед обходом и сохранять тип созданного object child table. Co-Authored-By: Claude Opus 5.5 --- python/ktav/_pure/_edit_tree.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/ktav/_pure/_edit_tree.py b/python/ktav/_pure/_edit_tree.py index a51c683..a35be26 100644 --- a/python/ktav/_pure/_edit_tree.py +++ b/python/ktav/_pure/_edit_tree.py @@ -172,10 +172,11 @@ def _insert_leaf_value( for key in decoded[:-1]: existing = cur.get(key) if existing is None: - node = _Node("object", {}, False, None) + children: dict[str, _Node] = {} + node = _Node("object", children, False, None) cur[key] = node - cur = node.children - elif existing.kind == "object": + cur = children + elif existing.kind == "object" and isinstance(existing.children, dict): existing.slot = None # reopen: noncontiguous from here on cur = existing.children else: @@ -593,10 +594,10 @@ def _collect_multiline_scalar_ranges(node: _Node, out: list[tuple[int, int]]) -> if node.kind == "scalar" and node.slot is not None and node.slot[0] == "block": out.append((node.slot[2], node.slot[3])) return - if node.kind == "object": + if node.kind == "object" and isinstance(node.children, dict): for child in node.children.values(): _collect_multiline_scalar_ranges(child, out) - elif node.kind == "array": + elif node.kind == "array" and isinstance(node.children, list): for child in node.children: _collect_multiline_scalar_ranges(child, out)