Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion ignis/build/test_runner.ign
Original file line number Diff line number Diff line change
Expand Up @@ -991,7 +991,9 @@ export namespace TestRunner {
}

let display: String = testRunnerTrimEnd(output);
let characters: u64 = display.toChars().length();

// `substring` cuts at a byte offset, so the limit counts bytes.
let characters: u64 = display.length();
let mut shown: String = display.clone();
let mut truncated: boolean = false;

Expand Down
2 changes: 1 addition & 1 deletion ignis/lsp/completion_context.ign
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@ export function matchesPrefix(name: &String, prefix: &String): boolean {

/// The first `count` characters of `text`.
function leadingChars(text: &String, count: u64): String {
let chars: Vector<char> = text.toChars();
let chars: Vector<char> = text.toScalars();
let mut result: String = String::new();
let mut index: u64 = 0;

Expand Down
54 changes: 45 additions & 9 deletions std/string/mod.ign
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@
//! | Method | Returns | Description |
//! |-----------|-----------------|------------------------------------|
//! | `toBytes` | `Vector<u8>` | Copy bytes into a byte vector |
//! | `toChars` | `Vector<char>` | Copy bytes into a char vector |
//! | `toScalars` | `Vector<char>` | Decode into Unicode scalars |
//! | `toChars` | `Vector<char>` | Deprecated: one `char` per byte |
//!
//! ## Numeric Conversions
//!
Expand Down Expand Up @@ -1123,11 +1124,43 @@ export record String {
return @sliceFromParts<u8>(self.data, self.len);
}

/// Copies the underlying bytes into a new `Vector<char>`.
/// Decodes the string into its Unicode scalars, in order. A byte that does
/// not start a well-formed UTF-8 sequence decodes as U+FFFD and advances
/// one byte, as `charAt` does, so the result never splits a scalar.
///
/// This is a byte-oriented compatibility helper: each raw byte is cast to
/// `char` directly without UTF-8 decoding. Use `charAt()` with byte offsets
/// when you need Unicode scalar traversal.
/// # Example
///
/// ```ignis
/// let text: String = String::create("añ😀");
/// let scalars: Vector<char> = text.toScalars(); // 3 scalars, 7 bytes
/// ```
public toScalars(&self): Vector<char> {
let mut result: Vector<char> = Vector::new<char>(self.len);
let mut offset: u64 = 0;

while (offset < self.len) {
let mut end: u64 = offset;

if (let Option::SOME(scalar) = Utf8::decodeAt(self, offset, &mut end)) {
result.push(scalar);
}

if (end <= offset) {
end = offset + 1;
}

offset = end;
}

return result;
}

/// Copies the underlying bytes into a new `Vector<char>`, one `char` per
/// byte, without UTF-8 decoding: `"é"` gives two `char`s.
///
/// Deprecated because the name suggests characters. Use `toScalars()` for
/// Unicode scalars, or `toBytes()` / `asBytes()` for bytes.
@deprecated("byte-wise despite its name; use toScalars() for Unicode scalars or toBytes() for bytes")
public toChars(&self): Vector<char> {
let mut result: Vector<char> = Vector::new<char>(self.len);

Expand All @@ -1140,7 +1173,8 @@ export record String {
return result;
}

/// Compatibility alias for `toChars()`.
/// Compatibility alias for `toChars()`, byte-wise like it.
@deprecated("byte-wise despite its name; use toScalars() for Unicode scalars or toBytes() for bytes")
public toVectorChars(&self): Vector<char> {
return self.toChars();
}
Expand Down Expand Up @@ -1945,10 +1979,12 @@ function asBytes(value: str): u8[] {
return @sliceFromParts<u8>(data, len);
}

/// Copies a NUL-terminated `str` into an owned `Vector<char>`.
/// Copies a NUL-terminated `str` into an owned `Vector<char>`, one `char`
/// per byte, without UTF-8 decoding; the scan stops at the first NUL.
///
/// This remains byte-oriented: each byte is cast to `char` directly and the
/// scan stops at the first terminating NUL.
/// Deprecated because the name suggests characters. Use
/// `String::create(value).toScalars()` for Unicode scalars.
@deprecated("byte-wise despite its name; use String::create(value).toScalars() for Unicode scalars")
@extension(str)
function toChars(value: str): Vector<char> {
let data: *mut u8 = value as *mut u8;
Expand Down
30 changes: 30 additions & 0 deletions std/string/tests.ign
Original file line number Diff line number Diff line change
Expand Up @@ -1027,3 +1027,33 @@ function fromUtf8AcceptsEmptyBytes(): void {

Test::assertEq<u64>(text.length(), 0);
}

@test
function toScalarsDecodesUtf8(): void {
let text: String = String::create("añ😀");
let scalars: Vector<char> = text.toScalars();

Test::assertEq<u64>(text.length(), 7);
Test::assertEq<u64>(scalars.length(), 3);
Test::assertEq<char>(*scalars.get(0).unwrap(), 'a');
Test::assertEq<char>(*scalars.get(1).unwrap(), 'ñ');
Test::assertEq<char>(*scalars.get(2).unwrap(), '😀');
}

@test
function toScalarsReplacesMalformedBytes(): void {
let mut text: String = String::create("a");
text.pushByte(255);
text.pushStr("b");
let scalars: Vector<char> = text.toScalars();

Test::assertEq<u64>(scalars.length(), 3);
Test::assertEq<char>(*scalars.get(1).unwrap(), 65533 as char);
Test::assertEq<char>(*scalars.get(2).unwrap(), 'b');
}

@test
function toScalarsOfEmptyIsEmpty(): void {
let text: String = String::new();
Test::assertEq<u64>(text.toScalars().length(), 0);
}
14 changes: 10 additions & 4 deletions test_cases/doc/__doc_baselines__/fixture_adds.json
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@
},
{
"name": "std::string",
"doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ┌──────────────┐ \n │ data ────────┼──────► ┌───┬───┬───┬───┬───┬───┬───┬───┐ \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ └───┴───┴───┴───┴───┴───┴───┴───┘ \n └──────────────┘ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option<u64>` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option<u64>` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector<String>` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector<u8>` | Copy bytes into a byte vector | \n| `toChars` | `Vector<char>` | Copy bytes into a char vector | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector<String> = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```"
"doc": " \n# String Module \n \nOwned, heap-backed UTF-8 byte string type and related utilities. \n \n## Overview \n \n`String` is a value-type record whose `data` buffer lives on the heap. \nIt implements `Drop` so the buffer is freed automatically when the \nowning binding goes out of scope, and `Clone` for explicit deep copies. \nIts storage is an owned UTF-8 byte buffer with a byte-counted `len` field. \n \n`str` is the primitive immutable string-slice type (`const char*` in C). \nIn v0.4 it remains a borrowed NUL-terminated UTF-8 byte view. \nString literals have type `str`. Use `String::create(s)` to create an \nowned copy from a `str`. \n \nInterior NUL bytes are preserved in owned storage. \ntoStr() is a zero-copy interop view. \nC-string consumers may stop at the first interior NUL. \n \n## Memory Layout \n \n```text \n String \n ┌──────────────┐ \n │ data ────────┼──────► ┌───┬───┬───┬───┬───┬───┬───┬───┐ \n │ len: 5 │ │ h │ e │ l │ l │ o │ │ │ │ \n │ cap: 8 │ └───┴───┴───┴───┴───┴───┴───┴───┘ \n └──────────────┘ ◄── used bytes ──►◄─ reserved ──► \n``` \n \nFields mirror the C runtime's `IgnisString` layout exactly: \n- `data` — pointer to the backing byte buffer (or `null` before init) \n- `len` — number of bytes currently stored \n- `cap` — total bytes allocated \n \n## UTF-8 Scanning and Bytes \n \n`String` stays byte-backed. APIs that need parser or diagnostic offsets keep \nthose byte positions internal to the caller while decoding `char` values at \nexplicit byte boundaries. \n \n`charAt` and `pushChar` work with decoded `char` values, while `byteAt`, \n`pushByte`, `forEachByte`, `findByte`, `trimWhere`, and `split` keep raw byte \nbehavior explicit. \n \n## Higher-Order Iteration Methods \n \nString provides callback-based operations for char/byte iteration, \nsearching, trimming, and splitting: \n \n| Method | Callback signature | Returns | Description | \n|---------------|----------------------|--------------------|------------------------------------| \n| `forEach` | `(char) -> void` | `void` | Iterate scalar `char` values | \n| `forEachByte` | `(u8) -> void` | `void` | Iterate over every byte | \n| `map` | `(char) -> char` | `String` | Transform scalar `char` values | \n| `mapBytes` | `(u8) -> u8` | `String` | Transform bytes directly | \n| `findByte` | `(u8) -> boolean` | `Option<u64>` | Index of first matching byte | \n| `findLastByte`| `(u8) -> boolean` | `Option<u64>` | Index of last matching byte | \n| `trimWhere` | `(u8) -> boolean` | `String` | Strip leading/trailing bytes | \n| `split` | `(u8) -> boolean` | `Vector<String>` | Split into segments | \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Byte Conversion \n \n| Method | Returns | Description | \n|-----------|-----------------|------------------------------------| \n| `toBytes` | `Vector<u8>` | Copy bytes into a byte vector | \n| `toScalars` | `Vector<char>` | Decode into Unicode scalars | \n| `toChars` | `Vector<char>` | Deprecated: one `char` per byte | \n \n## Numeric Conversions \n \n`String::create` is overloaded for all numeric primitives (`i8` through \n`f64`), plus `str`. Extension methods provide `.toString()` on every \nprimitive type. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction main(): i32 { \n let greeting: String = String::create(\"hello\"); \n let world: str = \" world\"; \n let msg: String = greeting.concat(world); \n \n Io::println(msg); \n \n // Byte-level: trim whitespace and split on commas \n let csv: String = String::create(\" a,b,c \"); \n let trimmed: String = csv.trimWhere((b: u8): boolean -> { return b == 32; }); \n let parts: Vector<String> = trimmed.split((b: u8): boolean -> { return b == 44; }); \n // parts: [\"a\", \"b\", \"c\"] \n \n return 0; \n} \n```"
},
{
"name": "std::types",
Expand Down Expand Up @@ -6364,14 +6364,20 @@
"name": "toChars",
"kind": "method",
"signature": "toChars(&self): Vector<char>",
"doc": "Copies the underlying bytes into a new `Vector<char>`. \n \nThis is a byte-oriented compatibility helper: each raw byte is cast to \n`char` directly without UTF-8 decoding. Use `charAt()` with byte offsets \nwhen you need Unicode scalar traversal."
"doc": "Copies the underlying bytes into a new `Vector<char>`, one `char` per \nbyte, without UTF-8 decoding: `\"é\"` gives two `char`s. \n \nDeprecated because the name suggests characters. Use `toScalars()` for \nUnicode scalars, or `toBytes()` / `asBytes()` for bytes."
},
{
"name": "toLowerCase",
"kind": "method",
"signature": "toLowerCase(&self): String",
"doc": "Returns a new string with all ASCII uppercase bytes converted to \nlowercase. \n \nConversion is byte-oriented: every byte above 127 becomes U+FFFD, so this \nis only safe on ASCII content. \n \n# Example \n \n```ignis \n \nlet s: String = String::create(\"Hello\"); \nlet lower: String = s.toLowerCase(); // \"hello\" \n```"
},
{
"name": "toScalars",
"kind": "method",
"signature": "toScalars(&self): Vector<char>",
"doc": "Decodes the string into its Unicode scalars, in order. A byte that does \nnot start a well-formed UTF-8 sequence decodes as U+FFFD and advances \none byte, as `charAt` does, so the result never splits a scalar. \n \n# Example \n \n```ignis \nlet text: String = String::create(\"añ😀\"); \nlet scalars: Vector<char> = text.toScalars(); // 3 scalars, 7 bytes \n```"
},
{
"name": "toStr",
"kind": "method",
Expand All @@ -6388,7 +6394,7 @@
"name": "toVectorChars",
"kind": "method",
"signature": "toVectorChars(&self): Vector<char>",
"doc": "Compatibility alias for `toChars()`."
"doc": "Compatibility alias for `toChars()`, byte-wise like it."
},
{
"name": "trimAsciiWhitespace",
Expand Down Expand Up @@ -6543,7 +6549,7 @@
"kind": "function",
"visibility": "private",
"signature": "function toChars(value: str): Vector<char>",
"doc": "Copies a NUL-terminated `str` into an owned `Vector<char>`. \n \nThis remains byte-oriented: each byte is cast to `char` directly and the \nscan stops at the first terminating NUL."
"doc": "Copies a NUL-terminated `str` into an owned `Vector<char>`, one `char` \nper byte, without UTF-8 decoding; the scan stops at the first NUL. \n \nDeprecated because the name suggests characters. Use \n`String::create(value).toScalars()` for Unicode scalars."
},
{
"module": "std::string",
Expand Down
Loading
Loading