From 9ef93496e53ad002104ed98c108d67a5023606d1 Mon Sep 17 00:00:00 2001 From: Ignacio Perez Date: Wed, 28 Oct 2026 18:49:25 -0300 Subject: [PATCH] feat(std): add String::toScalars and deprecate the byte-wise toChars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit String::toChars (and toVectorChars, and str's toChars extension) copy one char per byte without decoding, so "Ć©" gives two chars, which the name hides; the regex parser tripped over it. String::toScalars decodes the UTF-8 into Unicode scalars (a malformed byte becomes U+FFFD and advances one byte), and the byte-wise helpers are marked @deprecated, pointing to toScalars or toBytes. The compiler's two callers move off it: the failed-test output limit counts bytes, as the substring it cuts with does, and the language server's leadingChars takes scalars. --- ignis/build/test_runner.ign | 4 +- ignis/lsp/completion_context.ign | 2 +- std/string/mod.ign | 54 +++++++++++++++---- std/string/tests.ign | 30 +++++++++++ .../doc/__doc_baselines__/fixture_adds.json | 14 +++-- .../__doc_baselines__/fixture_counter.json | 14 +++-- .../__doc_baselines__/fixture_described.json | 14 +++-- .../doc/__doc_baselines__/fixture_helper.json | 14 +++-- .../doc/__doc_baselines__/fixture_math.json | 14 +++-- .../__doc_baselines__/fixture_outcome.json | 14 +++-- .../__doc_baselines__/fixture_written.json | 14 +++-- .../__doc_baselines__/output__std_io_mod.json | 14 +++-- 12 files changed, 159 insertions(+), 43 deletions(-) diff --git a/ignis/build/test_runner.ign b/ignis/build/test_runner.ign index c68474ae..fa678926 100644 --- a/ignis/build/test_runner.ign +++ b/ignis/build/test_runner.ign @@ -991,7 +991,9 @@ export namespace TestRunner { } let display: String = testRunnerTrimEnd(output); - let characters: u64 = display.toChars().length(); + + // `substring` cuts at a byte offset, so the limit counts bytes. + let characters: u64 = display.length(); let mut shown: String = display.clone(); let mut truncated: boolean = false; diff --git a/ignis/lsp/completion_context.ign b/ignis/lsp/completion_context.ign index 4c55d717..af83f3cd 100644 --- a/ignis/lsp/completion_context.ign +++ b/ignis/lsp/completion_context.ign @@ -316,7 +316,7 @@ export function matchesPrefix(name: &String, prefix: &String): boolean { /// The first `count` characters of `text`. function leadingChars(text: &String, count: u64): String { - let chars: Vector = text.toChars(); + let chars: Vector = text.toScalars(); let mut result: String = String::new(); let mut index: u64 = 0; diff --git a/std/string/mod.ign b/std/string/mod.ign index 8b920725..8953ab8a 100644 --- a/std/string/mod.ign +++ b/std/string/mod.ign @@ -68,7 +68,8 @@ //! | Method | Returns | Description | //! |-----------|-----------------|------------------------------------| //! | `toBytes` | `Vector` | Copy bytes into a byte vector | -//! | `toChars` | `Vector` | Copy bytes into a char vector | +//! | `toScalars` | `Vector` | Decode into Unicode scalars | +//! | `toChars` | `Vector` | Deprecated: one `char` per byte | //! //! ## Numeric Conversions //! @@ -1123,11 +1124,43 @@ export record String { return @sliceFromParts(self.data, self.len); } - /// Copies the underlying bytes into a new `Vector`. + /// Decodes the string into its Unicode scalars, in order. A byte that does + /// not start a well-formed UTF-8 sequence decodes as U+FFFD and advances + /// one byte, as `charAt` does, so the result never splits a scalar. /// - /// This is a byte-oriented compatibility helper: each raw byte is cast to - /// `char` directly without UTF-8 decoding. Use `charAt()` with byte offsets - /// when you need Unicode scalar traversal. + /// # Example + /// + /// ```ignis + /// let text: String = String::create("aĆ±šŸ˜€"); + /// let scalars: Vector = text.toScalars(); // 3 scalars, 7 bytes + /// ``` + public toScalars(&self): Vector { + let mut result: Vector = Vector::new(self.len); + let mut offset: u64 = 0; + + while (offset < self.len) { + let mut end: u64 = offset; + + if (let Option::SOME(scalar) = Utf8::decodeAt(self, offset, &mut end)) { + result.push(scalar); + } + + if (end <= offset) { + end = offset + 1; + } + + offset = end; + } + + return result; + } + + /// Copies the underlying bytes into a new `Vector`, one `char` per + /// byte, without UTF-8 decoding: `"Ć©"` gives two `char`s. + /// + /// Deprecated because the name suggests characters. Use `toScalars()` for + /// Unicode scalars, or `toBytes()` / `asBytes()` for bytes. + @deprecated("byte-wise despite its name; use toScalars() for Unicode scalars or toBytes() for bytes") public toChars(&self): Vector { let mut result: Vector = Vector::new(self.len); @@ -1140,7 +1173,8 @@ export record String { return result; } - /// Compatibility alias for `toChars()`. + /// Compatibility alias for `toChars()`, byte-wise like it. + @deprecated("byte-wise despite its name; use toScalars() for Unicode scalars or toBytes() for bytes") public toVectorChars(&self): Vector { return self.toChars(); } @@ -1945,10 +1979,12 @@ function asBytes(value: str): u8[] { return @sliceFromParts(data, len); } -/// Copies a NUL-terminated `str` into an owned `Vector`. +/// Copies a NUL-terminated `str` into an owned `Vector`, one `char` +/// per byte, without UTF-8 decoding; the scan stops at the first NUL. /// -/// This remains byte-oriented: each byte is cast to `char` directly and the -/// scan stops at the first terminating NUL. +/// Deprecated because the name suggests characters. Use +/// `String::create(value).toScalars()` for Unicode scalars. +@deprecated("byte-wise despite its name; use String::create(value).toScalars() for Unicode scalars") @extension(str) function toChars(value: str): Vector { let data: *mut u8 = value as *mut u8; diff --git a/std/string/tests.ign b/std/string/tests.ign index f082e7fb..f93d6e63 100644 --- a/std/string/tests.ign +++ b/std/string/tests.ign @@ -1027,3 +1027,33 @@ function fromUtf8AcceptsEmptyBytes(): void { Test::assertEq(text.length(), 0); } + +@test +function toScalarsDecodesUtf8(): void { + let text: String = String::create("aĆ±šŸ˜€"); + let scalars: Vector = text.toScalars(); + + Test::assertEq(text.length(), 7); + Test::assertEq(scalars.length(), 3); + Test::assertEq(*scalars.get(0).unwrap(), 'a'); + Test::assertEq(*scalars.get(1).unwrap(), 'Ʊ'); + Test::assertEq(*scalars.get(2).unwrap(), 'šŸ˜€'); +} + +@test +function toScalarsReplacesMalformedBytes(): void { + let mut text: String = String::create("a"); + text.pushByte(255); + text.pushStr("b"); + let scalars: Vector = text.toScalars(); + + Test::assertEq(scalars.length(), 3); + Test::assertEq(*scalars.get(1).unwrap(), 65533 as char); + Test::assertEq(*scalars.get(2).unwrap(), 'b'); +} + +@test +function toScalarsOfEmptyIsEmpty(): void { + let text: String = String::new(); + Test::assertEq(text.toScalars().length(), 0); +} diff --git a/test_cases/doc/__doc_baselines__/fixture_adds.json b/test_cases/doc/__doc_baselines__/fixture_adds.json index b8de643a..81f1bcaa 100644 --- a/test_cases/doc/__doc_baselines__/fixture_adds.json +++ b/test_cases/doc/__doc_baselines__/fixture_adds.json @@ -102,7 +102,7 @@ }, { "name": "std::string", - "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toChars` | `Vector` | Copy bytes into a char vector | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" + "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toScalars` | `Vector` | Decode into Unicode scalars | \n| `toChars` | `Vector` | Deprecated: one `char` per byte | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" }, { "name": "std::types", @@ -6364,7 +6364,7 @@ "name": "toChars", "kind": "method", "signature": "toChars(&self): Vector", - "doc": "Copies the underlying bytes into a new `Vector`. \n \nThis is a byte-oriented compatibility helper: each raw byte is cast to \n`char` directly without UTF-8 decoding. Use `charAt()` with byte offsets \nwhen you need Unicode scalar traversal." + "doc": "Copies the underlying bytes into a new `Vector`, one `char` per \nbyte, without UTF-8 decoding: `\"Ć©\"` gives two `char`s. \n \nDeprecated because the name suggests characters. Use `toScalars()` for \nUnicode scalars, or `toBytes()` / `asBytes()` for bytes." }, { "name": "toLowerCase", @@ -6372,6 +6372,12 @@ "signature": "toLowerCase(&self): String", "doc": "Returns a new string with all ASCII uppercase bytes converted to \nlowercase. \n \nConversion is byte-oriented: every byte above 127 becomes U+FFFD, so this \nis only safe on ASCII content. \n \n# Example \n \n```ignis \n \nlet s: String = String::create(\"Hello\"); \nlet lower: String = s.toLowerCase(); // \"hello\" \n```" }, + { + "name": "toScalars", + "kind": "method", + "signature": "toScalars(&self): Vector", + "doc": "Decodes the string into its Unicode scalars, in order. A byte that does \nnot start a well-formed UTF-8 sequence decodes as U+FFFD and advances \none byte, as `charAt` does, so the result never splits a scalar. \n \n# Example \n \n```ignis \nlet text: String = String::create(\"aĆ±šŸ˜€\"); \nlet scalars: Vector = text.toScalars(); // 3 scalars, 7 bytes \n```" + }, { "name": "toStr", "kind": "method", @@ -6388,7 +6394,7 @@ "name": "toVectorChars", "kind": "method", "signature": "toVectorChars(&self): Vector", - "doc": "Compatibility alias for `toChars()`." + "doc": "Compatibility alias for `toChars()`, byte-wise like it." }, { "name": "trimAsciiWhitespace", @@ -6543,7 +6549,7 @@ "kind": "function", "visibility": "private", "signature": "function toChars(value: str): Vector", - "doc": "Copies a NUL-terminated `str` into an owned `Vector`. \n \nThis remains byte-oriented: each byte is cast to `char` directly and the \nscan stops at the first terminating NUL." + "doc": "Copies a NUL-terminated `str` into an owned `Vector`, one `char` \nper byte, without UTF-8 decoding; the scan stops at the first NUL. \n \nDeprecated because the name suggests characters. Use \n`String::create(value).toScalars()` for Unicode scalars." }, { "module": "std::string", diff --git a/test_cases/doc/__doc_baselines__/fixture_counter.json b/test_cases/doc/__doc_baselines__/fixture_counter.json index 74f18277..fa50ba8c 100644 --- a/test_cases/doc/__doc_baselines__/fixture_counter.json +++ b/test_cases/doc/__doc_baselines__/fixture_counter.json @@ -102,7 +102,7 @@ }, { "name": "std::string", - "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toChars` | `Vector` | Copy bytes into a char vector | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" + "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toScalars` | `Vector` | Decode into Unicode scalars | \n| `toChars` | `Vector` | Deprecated: one `char` per byte | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" }, { "name": "std::types", @@ -6382,7 +6382,7 @@ "name": "toChars", "kind": "method", "signature": "toChars(&self): Vector", - "doc": "Copies the underlying bytes into a new `Vector`. \n \nThis is a byte-oriented compatibility helper: each raw byte is cast to \n`char` directly without UTF-8 decoding. Use `charAt()` with byte offsets \nwhen you need Unicode scalar traversal." + "doc": "Copies the underlying bytes into a new `Vector`, one `char` per \nbyte, without UTF-8 decoding: `\"Ć©\"` gives two `char`s. \n \nDeprecated because the name suggests characters. Use `toScalars()` for \nUnicode scalars, or `toBytes()` / `asBytes()` for bytes." }, { "name": "toLowerCase", @@ -6390,6 +6390,12 @@ "signature": "toLowerCase(&self): String", "doc": "Returns a new string with all ASCII uppercase bytes converted to \nlowercase. \n \nConversion is byte-oriented: every byte above 127 becomes U+FFFD, so this \nis only safe on ASCII content. \n \n# Example \n \n```ignis \n \nlet s: String = String::create(\"Hello\"); \nlet lower: String = s.toLowerCase(); // \"hello\" \n```" }, + { + "name": "toScalars", + "kind": "method", + "signature": "toScalars(&self): Vector", + "doc": "Decodes the string into its Unicode scalars, in order. A byte that does \nnot start a well-formed UTF-8 sequence decodes as U+FFFD and advances \none byte, as `charAt` does, so the result never splits a scalar. \n \n# Example \n \n```ignis \nlet text: String = String::create(\"aĆ±šŸ˜€\"); \nlet scalars: Vector = text.toScalars(); // 3 scalars, 7 bytes \n```" + }, { "name": "toStr", "kind": "method", @@ -6406,7 +6412,7 @@ "name": "toVectorChars", "kind": "method", "signature": "toVectorChars(&self): Vector", - "doc": "Compatibility alias for `toChars()`." + "doc": "Compatibility alias for `toChars()`, byte-wise like it." }, { "name": "trimAsciiWhitespace", @@ -6561,7 +6567,7 @@ "kind": "function", "visibility": "private", "signature": "function toChars(value: str): Vector", - "doc": "Copies a NUL-terminated `str` into an owned `Vector`. \n \nThis remains byte-oriented: each byte is cast to `char` directly and the \nscan stops at the first terminating NUL." + "doc": "Copies a NUL-terminated `str` into an owned `Vector`, one `char` \nper byte, without UTF-8 decoding; the scan stops at the first NUL. \n \nDeprecated because the name suggests characters. Use \n`String::create(value).toScalars()` for Unicode scalars." }, { "module": "std::string", diff --git a/test_cases/doc/__doc_baselines__/fixture_described.json b/test_cases/doc/__doc_baselines__/fixture_described.json index b72ec550..1a66323b 100644 --- a/test_cases/doc/__doc_baselines__/fixture_described.json +++ b/test_cases/doc/__doc_baselines__/fixture_described.json @@ -103,7 +103,7 @@ }, { "name": "std::string", - "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toChars` | `Vector` | Copy bytes into a char vector | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" + "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toScalars` | `Vector` | Decode into Unicode scalars | \n| `toChars` | `Vector` | Deprecated: one `char` per byte | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" }, { "name": "std::types", @@ -6365,7 +6365,7 @@ "name": "toChars", "kind": "method", "signature": "toChars(&self): Vector", - "doc": "Copies the underlying bytes into a new `Vector`. \n \nThis is a byte-oriented compatibility helper: each raw byte is cast to \n`char` directly without UTF-8 decoding. Use `charAt()` with byte offsets \nwhen you need Unicode scalar traversal." + "doc": "Copies the underlying bytes into a new `Vector`, one `char` per \nbyte, without UTF-8 decoding: `\"Ć©\"` gives two `char`s. \n \nDeprecated because the name suggests characters. Use `toScalars()` for \nUnicode scalars, or `toBytes()` / `asBytes()` for bytes." }, { "name": "toLowerCase", @@ -6373,6 +6373,12 @@ "signature": "toLowerCase(&self): String", "doc": "Returns a new string with all ASCII uppercase bytes converted to \nlowercase. \n \nConversion is byte-oriented: every byte above 127 becomes U+FFFD, so this \nis only safe on ASCII content. \n \n# Example \n \n```ignis \n \nlet s: String = String::create(\"Hello\"); \nlet lower: String = s.toLowerCase(); // \"hello\" \n```" }, + { + "name": "toScalars", + "kind": "method", + "signature": "toScalars(&self): Vector", + "doc": "Decodes the string into its Unicode scalars, in order. A byte that does \nnot start a well-formed UTF-8 sequence decodes as U+FFFD and advances \none byte, as `charAt` does, so the result never splits a scalar. \n \n# Example \n \n```ignis \nlet text: String = String::create(\"aĆ±šŸ˜€\"); \nlet scalars: Vector = text.toScalars(); // 3 scalars, 7 bytes \n```" + }, { "name": "toStr", "kind": "method", @@ -6389,7 +6395,7 @@ "name": "toVectorChars", "kind": "method", "signature": "toVectorChars(&self): Vector", - "doc": "Compatibility alias for `toChars()`." + "doc": "Compatibility alias for `toChars()`, byte-wise like it." }, { "name": "trimAsciiWhitespace", @@ -6544,7 +6550,7 @@ "kind": "function", "visibility": "private", "signature": "function toChars(value: str): Vector", - "doc": "Copies a NUL-terminated `str` into an owned `Vector`. \n \nThis remains byte-oriented: each byte is cast to `char` directly and the \nscan stops at the first terminating NUL." + "doc": "Copies a NUL-terminated `str` into an owned `Vector`, one `char` \nper byte, without UTF-8 decoding; the scan stops at the first NUL. \n \nDeprecated because the name suggests characters. Use \n`String::create(value).toScalars()` for Unicode scalars." }, { "module": "std::string", diff --git a/test_cases/doc/__doc_baselines__/fixture_helper.json b/test_cases/doc/__doc_baselines__/fixture_helper.json index 58eb8ecd..ef97e59a 100644 --- a/test_cases/doc/__doc_baselines__/fixture_helper.json +++ b/test_cases/doc/__doc_baselines__/fixture_helper.json @@ -102,7 +102,7 @@ }, { "name": "std::string", - "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toChars` | `Vector` | Copy bytes into a char vector | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" + "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toScalars` | `Vector` | Decode into Unicode scalars | \n| `toChars` | `Vector` | Deprecated: one `char` per byte | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" }, { "name": "std::types", @@ -6364,7 +6364,7 @@ "name": "toChars", "kind": "method", "signature": "toChars(&self): Vector", - "doc": "Copies the underlying bytes into a new `Vector`. \n \nThis is a byte-oriented compatibility helper: each raw byte is cast to \n`char` directly without UTF-8 decoding. Use `charAt()` with byte offsets \nwhen you need Unicode scalar traversal." + "doc": "Copies the underlying bytes into a new `Vector`, one `char` per \nbyte, without UTF-8 decoding: `\"Ć©\"` gives two `char`s. \n \nDeprecated because the name suggests characters. Use `toScalars()` for \nUnicode scalars, or `toBytes()` / `asBytes()` for bytes." }, { "name": "toLowerCase", @@ -6372,6 +6372,12 @@ "signature": "toLowerCase(&self): String", "doc": "Returns a new string with all ASCII uppercase bytes converted to \nlowercase. \n \nConversion is byte-oriented: every byte above 127 becomes U+FFFD, so this \nis only safe on ASCII content. \n \n# Example \n \n```ignis \n \nlet s: String = String::create(\"Hello\"); \nlet lower: String = s.toLowerCase(); // \"hello\" \n```" }, + { + "name": "toScalars", + "kind": "method", + "signature": "toScalars(&self): Vector", + "doc": "Decodes the string into its Unicode scalars, in order. A byte that does \nnot start a well-formed UTF-8 sequence decodes as U+FFFD and advances \none byte, as `charAt` does, so the result never splits a scalar. \n \n# Example \n \n```ignis \nlet text: String = String::create(\"aĆ±šŸ˜€\"); \nlet scalars: Vector = text.toScalars(); // 3 scalars, 7 bytes \n```" + }, { "name": "toStr", "kind": "method", @@ -6388,7 +6394,7 @@ "name": "toVectorChars", "kind": "method", "signature": "toVectorChars(&self): Vector", - "doc": "Compatibility alias for `toChars()`." + "doc": "Compatibility alias for `toChars()`, byte-wise like it." }, { "name": "trimAsciiWhitespace", @@ -6543,7 +6549,7 @@ "kind": "function", "visibility": "private", "signature": "function toChars(value: str): Vector", - "doc": "Copies a NUL-terminated `str` into an owned `Vector`. \n \nThis remains byte-oriented: each byte is cast to `char` directly and the \nscan stops at the first terminating NUL." + "doc": "Copies a NUL-terminated `str` into an owned `Vector`, one `char` \nper byte, without UTF-8 decoding; the scan stops at the first NUL. \n \nDeprecated because the name suggests characters. Use \n`String::create(value).toScalars()` for Unicode scalars." }, { "module": "std::string", diff --git a/test_cases/doc/__doc_baselines__/fixture_math.json b/test_cases/doc/__doc_baselines__/fixture_math.json index a32be980..7eb03d64 100644 --- a/test_cases/doc/__doc_baselines__/fixture_math.json +++ b/test_cases/doc/__doc_baselines__/fixture_math.json @@ -102,7 +102,7 @@ }, { "name": "std::string", - "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toChars` | `Vector` | Copy bytes into a char vector | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" + "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toScalars` | `Vector` | Decode into Unicode scalars | \n| `toChars` | `Vector` | Deprecated: one `char` per byte | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" }, { "name": "std::types", @@ -6372,7 +6372,7 @@ "name": "toChars", "kind": "method", "signature": "toChars(&self): Vector", - "doc": "Copies the underlying bytes into a new `Vector`. \n \nThis is a byte-oriented compatibility helper: each raw byte is cast to \n`char` directly without UTF-8 decoding. Use `charAt()` with byte offsets \nwhen you need Unicode scalar traversal." + "doc": "Copies the underlying bytes into a new `Vector`, one `char` per \nbyte, without UTF-8 decoding: `\"Ć©\"` gives two `char`s. \n \nDeprecated because the name suggests characters. Use `toScalars()` for \nUnicode scalars, or `toBytes()` / `asBytes()` for bytes." }, { "name": "toLowerCase", @@ -6380,6 +6380,12 @@ "signature": "toLowerCase(&self): String", "doc": "Returns a new string with all ASCII uppercase bytes converted to \nlowercase. \n \nConversion is byte-oriented: every byte above 127 becomes U+FFFD, so this \nis only safe on ASCII content. \n \n# Example \n \n```ignis \n \nlet s: String = String::create(\"Hello\"); \nlet lower: String = s.toLowerCase(); // \"hello\" \n```" }, + { + "name": "toScalars", + "kind": "method", + "signature": "toScalars(&self): Vector", + "doc": "Decodes the string into its Unicode scalars, in order. A byte that does \nnot start a well-formed UTF-8 sequence decodes as U+FFFD and advances \none byte, as `charAt` does, so the result never splits a scalar. \n \n# Example \n \n```ignis \nlet text: String = String::create(\"aĆ±šŸ˜€\"); \nlet scalars: Vector = text.toScalars(); // 3 scalars, 7 bytes \n```" + }, { "name": "toStr", "kind": "method", @@ -6396,7 +6402,7 @@ "name": "toVectorChars", "kind": "method", "signature": "toVectorChars(&self): Vector", - "doc": "Compatibility alias for `toChars()`." + "doc": "Compatibility alias for `toChars()`, byte-wise like it." }, { "name": "trimAsciiWhitespace", @@ -6551,7 +6557,7 @@ "kind": "function", "visibility": "private", "signature": "function toChars(value: str): Vector", - "doc": "Copies a NUL-terminated `str` into an owned `Vector`. \n \nThis remains byte-oriented: each byte is cast to `char` directly and the \nscan stops at the first terminating NUL." + "doc": "Copies a NUL-terminated `str` into an owned `Vector`, one `char` \nper byte, without UTF-8 decoding; the scan stops at the first NUL. \n \nDeprecated because the name suggests characters. Use \n`String::create(value).toScalars()` for Unicode scalars." }, { "module": "std::string", diff --git a/test_cases/doc/__doc_baselines__/fixture_outcome.json b/test_cases/doc/__doc_baselines__/fixture_outcome.json index 3550e1e2..288522b9 100644 --- a/test_cases/doc/__doc_baselines__/fixture_outcome.json +++ b/test_cases/doc/__doc_baselines__/fixture_outcome.json @@ -102,7 +102,7 @@ }, { "name": "std::string", - "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toChars` | `Vector` | Copy bytes into a char vector | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" + "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toScalars` | `Vector` | Decode into Unicode scalars | \n| `toChars` | `Vector` | Deprecated: one `char` per byte | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" }, { "name": "std::types", @@ -6376,7 +6376,7 @@ "name": "toChars", "kind": "method", "signature": "toChars(&self): Vector", - "doc": "Copies the underlying bytes into a new `Vector`. \n \nThis is a byte-oriented compatibility helper: each raw byte is cast to \n`char` directly without UTF-8 decoding. Use `charAt()` with byte offsets \nwhen you need Unicode scalar traversal." + "doc": "Copies the underlying bytes into a new `Vector`, one `char` per \nbyte, without UTF-8 decoding: `\"Ć©\"` gives two `char`s. \n \nDeprecated because the name suggests characters. Use `toScalars()` for \nUnicode scalars, or `toBytes()` / `asBytes()` for bytes." }, { "name": "toLowerCase", @@ -6384,6 +6384,12 @@ "signature": "toLowerCase(&self): String", "doc": "Returns a new string with all ASCII uppercase bytes converted to \nlowercase. \n \nConversion is byte-oriented: every byte above 127 becomes U+FFFD, so this \nis only safe on ASCII content. \n \n# Example \n \n```ignis \n \nlet s: String = String::create(\"Hello\"); \nlet lower: String = s.toLowerCase(); // \"hello\" \n```" }, + { + "name": "toScalars", + "kind": "method", + "signature": "toScalars(&self): Vector", + "doc": "Decodes the string into its Unicode scalars, in order. A byte that does \nnot start a well-formed UTF-8 sequence decodes as U+FFFD and advances \none byte, as `charAt` does, so the result never splits a scalar. \n \n# Example \n \n```ignis \nlet text: String = String::create(\"aĆ±šŸ˜€\"); \nlet scalars: Vector = text.toScalars(); // 3 scalars, 7 bytes \n```" + }, { "name": "toStr", "kind": "method", @@ -6400,7 +6406,7 @@ "name": "toVectorChars", "kind": "method", "signature": "toVectorChars(&self): Vector", - "doc": "Compatibility alias for `toChars()`." + "doc": "Compatibility alias for `toChars()`, byte-wise like it." }, { "name": "trimAsciiWhitespace", @@ -6555,7 +6561,7 @@ "kind": "function", "visibility": "private", "signature": "function toChars(value: str): Vector", - "doc": "Copies a NUL-terminated `str` into an owned `Vector`. \n \nThis remains byte-oriented: each byte is cast to `char` directly and the \nscan stops at the first terminating NUL." + "doc": "Copies a NUL-terminated `str` into an owned `Vector`, one `char` \nper byte, without UTF-8 decoding; the scan stops at the first NUL. \n \nDeprecated because the name suggests characters. Use \n`String::create(value).toScalars()` for Unicode scalars." }, { "module": "std::string", diff --git a/test_cases/doc/__doc_baselines__/fixture_written.json b/test_cases/doc/__doc_baselines__/fixture_written.json index 7162c576..e750ee38 100644 --- a/test_cases/doc/__doc_baselines__/fixture_written.json +++ b/test_cases/doc/__doc_baselines__/fixture_written.json @@ -99,7 +99,7 @@ }, { "name": "std::string", - "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toChars` | `Vector` | Copy bytes into a char vector | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" + "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toScalars` | `Vector` | Decode into Unicode scalars | \n| `toChars` | `Vector` | Deprecated: one `char` per byte | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" }, { "name": "std::types", @@ -6355,7 +6355,7 @@ "name": "toChars", "kind": "method", "signature": "toChars(&self): Vector", - "doc": "Copies the underlying bytes into a new `Vector`. \n \nThis is a byte-oriented compatibility helper: each raw byte is cast to \n`char` directly without UTF-8 decoding. Use `charAt()` with byte offsets \nwhen you need Unicode scalar traversal." + "doc": "Copies the underlying bytes into a new `Vector`, one `char` per \nbyte, without UTF-8 decoding: `\"Ć©\"` gives two `char`s. \n \nDeprecated because the name suggests characters. Use `toScalars()` for \nUnicode scalars, or `toBytes()` / `asBytes()` for bytes." }, { "name": "toLowerCase", @@ -6363,6 +6363,12 @@ "signature": "toLowerCase(&self): String", "doc": "Returns a new string with all ASCII uppercase bytes converted to \nlowercase. \n \nConversion is byte-oriented: every byte above 127 becomes U+FFFD, so this \nis only safe on ASCII content. \n \n# Example \n \n```ignis \n \nlet s: String = String::create(\"Hello\"); \nlet lower: String = s.toLowerCase(); // \"hello\" \n```" }, + { + "name": "toScalars", + "kind": "method", + "signature": "toScalars(&self): Vector", + "doc": "Decodes the string into its Unicode scalars, in order. A byte that does \nnot start a well-formed UTF-8 sequence decodes as U+FFFD and advances \none byte, as `charAt` does, so the result never splits a scalar. \n \n# Example \n \n```ignis \nlet text: String = String::create(\"aĆ±šŸ˜€\"); \nlet scalars: Vector = text.toScalars(); // 3 scalars, 7 bytes \n```" + }, { "name": "toStr", "kind": "method", @@ -6379,7 +6385,7 @@ "name": "toVectorChars", "kind": "method", "signature": "toVectorChars(&self): Vector", - "doc": "Compatibility alias for `toChars()`." + "doc": "Compatibility alias for `toChars()`, byte-wise like it." }, { "name": "trimAsciiWhitespace", @@ -6534,7 +6540,7 @@ "kind": "function", "visibility": "private", "signature": "function toChars(value: str): Vector", - "doc": "Copies a NUL-terminated `str` into an owned `Vector`. \n \nThis remains byte-oriented: each byte is cast to `char` directly and the \nscan stops at the first terminating NUL." + "doc": "Copies a NUL-terminated `str` into an owned `Vector`, one `char` \nper byte, without UTF-8 decoding; the scan stops at the first NUL. \n \nDeprecated because the name suggests characters. Use \n`String::create(value).toScalars()` for Unicode scalars." }, { "module": "std::string", diff --git a/test_cases/doc/__doc_baselines__/output__std_io_mod.json b/test_cases/doc/__doc_baselines__/output__std_io_mod.json index 893f48c7..80283e16 100644 --- a/test_cases/doc/__doc_baselines__/output__std_io_mod.json +++ b/test_cases/doc/__doc_baselines__/output__std_io_mod.json @@ -361,7 +361,7 @@ }, { "name": "std::string", - "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toChars` | `Vector` | Copy bytes into a char vector | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" + "doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” \n │ data ────────┼──────► ā”Œā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā” \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ ā””ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”“ā”€ā”€ā”€ā”˜ \n ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector` | Copy bytes into a byte vector | \n| `toScalars` | `Vector` | Decode into Unicode scalars | \n| `toChars` | `Vector` | Deprecated: one `char` per byte | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```" }, { "name": "std::sync", @@ -19563,7 +19563,7 @@ "name": "toChars", "kind": "method", "signature": "toChars(&self): Vector", - "doc": "Copies the underlying bytes into a new `Vector`. \n \nThis is a byte-oriented compatibility helper: each raw byte is cast to \n`char` directly without UTF-8 decoding. Use `charAt()` with byte offsets \nwhen you need Unicode scalar traversal." + "doc": "Copies the underlying bytes into a new `Vector`, one `char` per \nbyte, without UTF-8 decoding: `\"Ć©\"` gives two `char`s. \n \nDeprecated because the name suggests characters. Use `toScalars()` for \nUnicode scalars, or `toBytes()` / `asBytes()` for bytes." }, { "name": "toLowerCase", @@ -19571,6 +19571,12 @@ "signature": "toLowerCase(&self): String", "doc": "Returns a new string with all ASCII uppercase bytes converted to \nlowercase. \n \nConversion is byte-oriented: every byte above 127 becomes U+FFFD, so this \nis only safe on ASCII content. \n \n# Example \n \n```ignis \n \nlet s: String = String::create(\"Hello\"); \nlet lower: String = s.toLowerCase(); // \"hello\" \n```" }, + { + "name": "toScalars", + "kind": "method", + "signature": "toScalars(&self): Vector", + "doc": "Decodes the string into its Unicode scalars, in order. A byte that does \nnot start a well-formed UTF-8 sequence decodes as U+FFFD and advances \none byte, as `charAt` does, so the result never splits a scalar. \n \n# Example \n \n```ignis \nlet text: String = String::create(\"aĆ±šŸ˜€\"); \nlet scalars: Vector = text.toScalars(); // 3 scalars, 7 bytes \n```" + }, { "name": "toStr", "kind": "method", @@ -19587,7 +19593,7 @@ "name": "toVectorChars", "kind": "method", "signature": "toVectorChars(&self): Vector", - "doc": "Compatibility alias for `toChars()`." + "doc": "Compatibility alias for `toChars()`, byte-wise like it." }, { "name": "trimAsciiWhitespace", @@ -19742,7 +19748,7 @@ "kind": "function", "visibility": "private", "signature": "function toChars(value: str): Vector", - "doc": "Copies a NUL-terminated `str` into an owned `Vector`. \n \nThis remains byte-oriented: each byte is cast to `char` directly and the \nscan stops at the first terminating NUL." + "doc": "Copies a NUL-terminated `str` into an owned `Vector`, one `char` \nper byte, without UTF-8 decoding; the scan stops at the first NUL. \n \nDeprecated because the name suggests characters. Use \n`String::create(value).toScalars()` for Unicode scalars." }, { "module": "std::string",