Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
3 changes: 3 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,6 @@ versions/*/tests/valid/multiline/verbatim_whitespace_only_line_in_middle.canonic
# condition; two of them contain CR bytes that `* text=auto` would
# normalize away at check-in. `-text` keeps git from ever rewriting them.
versions/*/tests/invalid/invalid_utf8/*.ktav -text

# Exact diagnostic byte offsets require original BOM and line endings.
versions/0.9/tests/invalid/diagnostics09/*.ktav -text
3 changes: 3 additions & 0 deletions .github/workflows/corpus-validate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ jobs:
python scripts/validate_corpus.py versions/0.7/tests --require-unrepresentable --require-boundary --require-manifest --boundary-manifest-lock scripts/locks/boundary-fixtures.0.7.lock.json --corpus-inventory-lock scripts/locks/corpus-inventory.0.7.lock.json
python scripts/validate_corpus.py versions/0.8/tests --require-unrepresentable --require-boundary --require-manifest --boundary-manifest-lock scripts/locks/boundary-fixtures.0.8.lock.json --corpus-inventory-lock scripts/locks/corpus-inventory.0.8.lock.json

- name: Validate draft 0.9 diagnostics corpus
run: python scripts/validate_corpus.py versions/0.9/tests --require-unrepresentable --require-boundary --require-manifest

- name: Run validator unit tests
run: python scripts/internal/corpus/test_validate_corpus.py

Expand Down
4 changes: 4 additions & 0 deletions scripts/internal/builder/build_spec/registry.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ export const DOCS_REGISTRY = {
'versions/*/content/README.source.md',
'docs/checkpoints/',
'docs/reviews/',
// Draft diagnostic contracts, not generated release specifications.
'versions/0.9/README.md',
'versions/0.9/README.ru.md',
'versions/0.9/README.zh.md',
],
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,36 @@


class CategoriesAndUtf8Tests:
def test_diagnostic_location_schema_and_raw_line(self):
tests = self.build_minimal()
stem = "tests/invalid/diagnostics09/unclosed"
self.write(stem + ".ktav", "## comment\r\n{\r\n")
oracle = {"expected_error": "UnclosedCompound", "note": "opener line",
"expected_location": {"line": 2, "start": 12, "end": 15}}
self.write(stem + ".json", json.dumps(oracle))
code, out = self.run_main(tests)
self.assertEqual(code, 0, out)
for location in ({"line": 1, "start": 12, "end": 15},
{"line": 2, "start": 12, "end": 12},
{"line": 2, "start": 12, "end": 16},
{"line": True, "start": 12, "end": 15},
{"line": 2, "start": 12, "minimum_end": 15, "end": 15}):
with self.subTest(location=location):
oracle["expected_location"] = location
self.write(stem + ".json", json.dumps(oracle))
code, out = self.run_main(tests)
self.assertEqual(code, 1, out)
self.assertIn("location", out)

def test_diagnostic_location_cannot_be_removed(self):
tests = self.build_minimal()
stem = "tests/invalid/diagnostics09/unclosed"
self.write(stem + ".ktav", "{\n")
self.write(stem + ".json", json.dumps({"expected_error": "UnclosedCompound", "note": "missing location"}))
code, out = self.run_main(tests)
self.assertEqual(code, 1, out)
self.assertIn("expected_location", out)

def write_invalid_utf8_fixture(self, root, name="bad_utf8"):
"""A well-formed invalid/invalid_utf8/ fixture: genuinely invalid
UTF-8 bytes paired with expected_error 'InvalidUtf8', satisfying the
Expand Down
33 changes: 33 additions & 0 deletions scripts/internal/corpus/validate_corpus/checks_corpus.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,37 @@ def select_error_categories(tests_dir):
return DEFAULT_ERROR_CATEGORIES


def check_error_location(obj, input_path, rpath, results, category):
location = obj.get("expected_location")
required = "/diagnostics09/" in "/" + rpath.replace("\\", "/")
if location is None and not required:
return
if not isinstance(location, dict) or set(location) not in (
{"line", "start", "end"}, {"line", "start", "minimum_end"}):
results.fail(category, "%s: invalid expected_location schema" % rpath)
return
if any(type(value) is not int or value < 0 for value in location.values()):
results.fail(category, "%s: location fields must be nonnegative integers" % rpath)
return
try:
with open(input_path, "rb") as source:
raw = source.read()
except OSError as error:
results.fail(category, "%s: cannot read location input: %s" % (rpath, error))
return
start = location["start"]
end = location.get("end", location.get("minimum_end"))
if not (location["line"] > 0 and 0 <= start < end <= len(raw)):
results.fail(category, "%s: invalid location bounds" % rpath)
return
prefix = raw[:start].replace(b"\r\n", b"\n").replace(b"\r", b"\n")
line = 1 + prefix.count(b"\n")
if start and raw[start - 1:start + 1] == b"\r\n":
line -= 1
if location["line"] != line:
results.fail(category, "%s: location line disagrees with original bytes" % rpath)


def check_invalid(tests_dir, results, parsed, error_categories):
"""Check 3: .ktav/.json pairs under invalid/ with expected_error strings."""
category = "invalid/ pairs"
Expand Down Expand Up @@ -215,6 +246,8 @@ def check_invalid(tests_dir, results, parsed, error_categories):
results.fail(category, "%s: expected_error check skipped: "
"not a JSON object" % rpath)
continue
check_error_location(obj, os.path.join(root, name + ".ktav"),
rpath, results, category)
err = obj.get("expected_error")
if not isinstance(err, str) or err == "":
results.fail(category, "%s: 'expected_error' must be a "
Expand Down
1 change: 1 addition & 0 deletions scripts/internal/corpus/validate_corpus/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
"0.6": ERROR_CATEGORIES_V0_6,
"0.7": ERROR_CATEGORIES_V0_7,
"0.8": ERROR_CATEGORIES_V0_8,
"0.9": ERROR_CATEGORIES_V0_8,
}
DEFAULT_ERROR_CATEGORIES = ERROR_CATEGORIES_V0_8 # == union of all versions (0.6/0.7 are subsets)

Expand Down
29 changes: 29 additions & 0 deletions versions/0.9/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Ktav 0.9 — draft diagnostics corpus

This directory prepares version 0.9.0; it is not a published release. The language and existing fixture categories inherit version 0.8. New diagnostic fixtures live only in this version. Version 0.8 remains unchanged.

## Source locations

Source parse errors carry a 1-based line and a half-open byte span in the original input. BOM bytes and LF, lone CR and CRLF terminators are never removed or normalized before calculating offsets. CRLF counts as one terminator. A position inside a terminator belongs to the line that terminator closes.

For `UnclosedCompound`, the diagnostic anchors the innermost unclosed opener: the line is the opener's line and the span runs from the opener to EOF. For multiline values, the opener starts at `(`, not at the pair's key.

For `InvalidUtf8`, validation precedes grammar processing. The span starts at the first invalid sequence, covers at least one original byte and stays within the input. Implementations may cover different widths of the malformed subsequence. Raw-byte fixtures must never pass through lossy decoding. I/O failures and writer rejections do not receive invented source locations.

## Fixture schema

The existing `invalid/<name>.ktav` / `<name>.json` pair remains. An oracle may additionally contain `expected_location`, with exactly one of these shapes:

```json
{"line": 2, "start": 12, "end": 27}
```

```json
{"line": 3, "start": 33, "minimum_end": 34}
```

`line` is positive; the remaining values are nonnegative byte offsets. `end` requires an exact endpoint; `minimum_end` requires a covering endpoint at least that large and no larger than the input. The two endpoint fields are mutually exclusive. A runner must check every declared location, not merely the error category. All fixtures under `invalid/diagnostics09/` require this location oracle.

The manifest counts every new fixture and flags malformed UTF-8 inputs as `raw_bytes`. The Rust owned parser, strict parser, thin event parser and formatter exercise valid-UTF-8 fixtures; `from_file` exercises malformed byte inputs. Direct error accessors and public envelope conversion must agree. Native C ABI source operations use the same location contract.

The human-readable message is not a cross-language coordinate oracle. The existing ten-field error-envelope schema is unchanged; this migration makes its source coordinates complete and consistent with the direct Rust API.
29 changes: 29 additions & 0 deletions versions/0.9/README.ru.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Ktav 0.9 — черновой corpus диагностики

Этот каталог готовит версию 0.9.0; это не опубликованный релиз. Язык и существующие категории фикстур наследуют версию 0.8. Новые диагностические фикстуры находятся только в новой версии. Версия 0.8 не меняется.

## Координаты исходного текста

Ошибка разбора исходника несёт 1-based номер строки и полуинтервальный байтовый span в исходных данных. BOM и терминаторы LF, одиночный CR и CRLF не удаляются и не нормализуются перед вычислением смещений. CRLF считается одним терминатором. Позиция внутри терминатора принадлежит завершаемой им строке.

Для `UnclosedCompound` диагностика привязана к самому внутреннему незакрытому opener: строка соответствует opener, span идёт от opener до EOF. Для multiline-значений opener начинается с `(`, не с ключа пары.

Для `InvalidUtf8` проверка кодировки предшествует разбору грамматики. Span начинается с первой ошибочной последовательности, покрывает хотя бы один исходный байт и не выходит за входные данные. Реализации могут покрывать разную ширину ошибочной последовательности. Raw-byte фикстуры запрещено декодировать с заменой ошибочных байтов. Ошибкам I/O и отказам writer не приписываются выдуманные координаты исходника.

## Схема фикстур

Сохраняется пара `invalid/<name>.ktav` / `<name>.json`. Oracle может дополнительно содержать `expected_location` ровно одной из форм:

```json
{"line": 2, "start": 12, "end": 27}
```

```json
{"line": 3, "start": 33, "minimum_end": 34}
```

`line` положителен; остальные значения — неотрицательные байтовые смещения. `end` задаёт точную конечную позицию; `minimum_end` требует покрытия как минимум до этой позиции, но не за концом исходника. Эти два поля взаимоисключающие. Runner обязан проверять каждую объявленную позицию, а не только категорию ошибки. Все фикстуры `invalid/diagnostics09/` требуют такого oracle.

Manifest учитывает каждую новую фикстуру и помечает malformed UTF-8 как `raw_bytes`. Rust owned parser, strict parser, thin event parser и formatter проверяют UTF-8-фикстуры; `from_file` проверяет ошибочные байтовые данные. Прямые accessor ошибок и публичное преобразование в envelope обязаны совпадать. Native C ABI source operations используют тот же контракт координат.

Человекочитаемое сообщение не служит межъязыковым oracle координат. Существующая схема error-envelope из десяти полей не меняется; миграция делает координаты исходника полными и согласованными с прямым Rust API.
29 changes: 29 additions & 0 deletions versions/0.9/README.zh.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Ktav 0.9 — 诊断语料草案

本目录用于准备 0.9.0,并非已发布版本。语言及原有测试分类继承 0.8。新增诊断用例仅放在新版本中;0.8 保持不变。

## 源位置

源解析错误携带从 1 开始的行号及原始输入中的半开字节区间。计算偏移前不得删除 BOM,也不得规范化 LF、单独 CR 或 CRLF。CRLF 算一个换行符;换行符内部的位置属于该换行符结束的行。

`UnclosedCompound` 指向最内层未闭合的起始标记:行号为起始标记所在行,区间从起始标记延伸到 EOF。多行值的起始位置为 `(`,不是键名。

`InvalidUtf8` 检查优先于语法解析。区间从第一个错误序列开始,至少覆盖一个原始字节,且不得越过输入末尾。不同实现可以覆盖不同宽度的错误序列。原始字节用例不得经过有损解码。I/O 错误及 writer 拒绝不得携带虚构的源位置。

## 用例格式

保留 `invalid/<name>.ktav` / `<name>.json` 文件对。Oracle 可以增加 `expected_location`,且只能采用以下一种结构:

```json
{"line": 2, "start": 12, "end": 27}
```

```json
{"line": 3, "start": 33, "minimum_end": 34}
```

`line` 为正整数,其余字段为非负字节偏移。`end` 要求精确终点;`minimum_end` 要求终点不小于该值且不大于输入长度。两种终点字段互斥。Runner 必须检查每个声明的位置,不能只检查错误类别。`invalid/diagnostics09/` 中的所有用例都必须声明位置 oracle。

Manifest 记录全部新增用例,并将无效 UTF-8 标记为 `raw_bytes`。Rust owned parser、strict parser、thin event parser 及 formatter 检查有效 UTF-8 输入;`from_file` 检查无效字节输入。直接错误访问器和公开 envelope 转换必须一致;native C ABI 源操作遵循相同位置契约。

面向人的错误消息不是跨语言位置 oracle。现有十字段 error-envelope 格式不变;此次迁移补全位置,并使其与直接 Rust API 一致。
16 changes: 16 additions & 0 deletions versions/0.9/tests/boundary-fixtures.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"$comment": "Leaf-level list of valid/ fixture Values known to probe a numeric-domain boundary named in Sec 5 / Sec 8.1 / Sec 8.2. This file lives outside valid/ so a runner enumerating valid/**/*.json as fixtures never mistakes it for one. Each entry names exactly one leaf -- the Object field at 'path' (a JSON Pointer, RFC 6901, relative to the fixture's own root Value) inside fixture '<fixture>.json' / '<fixture>.canonical.ktav' -- NOT the whole fixture: a fixture may mix boundary-dependent leaves with ordinary ones (e.g. big_overflow_to_string's 'tiny' field is an ordinary Integer(1) in every conforming domain, while its 'big'/'bigger' fields are not), and only the listed leaf is exempt. 'boundary_class' names which numeric-domain axis the leaf probes: 'integer_range' (exceeds the mandatory i64 range), 'float_range' (overflows to non-finite on binary64), 'float_underflow' (underflows to zero on binary64), or 'float_precision' (binary64 rounds or shortens the value; a higher-precision domain would not). An implementation is exempt from matching a listed leaf's value/byte-form ONLY if it genuinely supports a domain wider than the minimum ALONG THAT SPECIFIC boundary_class for that specific leaf -- e.g. a BigInt-but-plain-binary64 implementation is exempt on 'integer_range' leaves but not on any 'float_*' leaf, and vice versa. Every field of a fixture NOT listed here, and every fixture not appearing here at all, carries no exemption for any implementation of any domain -- it MUST match its <name>.json / <name>.canonical.ktav exactly, the same as any other valid/ fixture. This file does not define what a wider-domain implementation's output must be for an exempt leaf -- only that the shared corpus does not pin it; correctness there is governed directly by Sec 5 (which value and kind the implementation holds) and Sec 5.9 (how it canonically writes whatever it holds).",
"boundary_dependent_leaves": [
{"fixture": "numbers/integer/i64_overflow_to_string", "path": "/overflow", "boundary_class": "integer_range"},
{"fixture": "numbers/integer/big_overflow_to_string", "path": "/big", "boundary_class": "integer_range"},
{"fixture": "numbers/integer/big_overflow_to_string", "path": "/bigger", "boundary_class": "integer_range"},
{"fixture": "numbers/float/positive_overflow_to_string", "path": "/overflow", "boundary_class": "float_range"},
{"fixture": "numbers/float/negative_overflow_to_string", "path": "/overflow", "boundary_class": "float_range"},
{"fixture": "numbers/float/just_above_max_finite_to_string", "path": "/overflow", "boundary_class": "float_range"},
{"fixture": "numbers/float/underflow_to_zero", "path": "/underflow", "boundary_class": "float_underflow"},
{"fixture": "numbers/float/negative_underflow_to_negative_zero", "path": "/underflow", "boundary_class": "float_underflow"},
{"fixture": "numbers/float/half_min_subnormal_underflow_to_zero", "path": "/underflow", "boundary_class": "float_underflow"},
{"fixture": "numbers/float/decimal_rounding_tie", "path": "/tie", "boundary_class": "float_precision"},
{"fixture": "numbers/float/min_positive_subnormal", "path": "/min_subnormal", "boundary_class": "float_precision"}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"expected_error": "BadEscapeSequence", "note": "\\t is not a named escape in any version -- tab has no dedicated named form. As of 0.7.0 a literal tab can be spelled `\\u0009`; a bare `\\t` remains BadEscapeSequence."}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
cfg: {a: foo\tbar}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"expected_error": "BadEscapeSequence", "note": "A high surrogate followed by an escape that is not a valid low surrogate is a lone surrogate."}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
k: {a: \uD800\u0041}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"expected_error": "BadEscapeSequence", "note": "A high surrogate (U+D800) with no following low-surrogate \\uXXXX escape is a lone surrogate."}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
k: {a: \uD800}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"expected_error": "BadEscapeSequence", "note": "A low surrogate (U+DC00) with no preceding high surrogate is a lone surrogate."}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
k: {a: \uDC00}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"expected_error": "BadEscapeSequence", "note": "\\u followed by fewer than four hex digits before a non-hex byte is malformed."}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
k: {a: \u12}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"expected_error": "BadEscapeSequence", "note": "\\q is not one of the recognised escape sequences."}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
cfg: {a: hello\q world}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"expected_error": "BadEscapeSequence", "note": "An invalid escape before a matching inline closer takes precedence over UnterminatedInlineCompound."}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
cfg: {a: \q
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"expected_error": "UnclosedCompound",
"note": "0.9 source diagnostics: original byte offsets, BOM and LF/CR/CRLF preserved; direct API and envelope agree.",
"expected_location": {
"line": 2,
"start": 15,
"end": 25
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
## שלום[ value
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"expected_error": "UnclosedCompound",
"note": "0.9 source diagnostics: original byte offsets, BOM and LF/CR/CRLF preserved; direct API and envelope agree.",
"expected_location": {
"line": 2,
"start": 12,
"end": 22
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
## שלום[ value
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"expected_error": "UnclosedCompound",
"note": "0.9 source diagnostics: original byte offsets, BOM and LF/CR/CRLF preserved; direct API and envelope agree.",
"expected_location": {
"line": 2,
"start": 16,
"end": 28
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
## שלום
[
value
Loading
Loading