diff --git a/std/regex/compiler.ign b/std/regex/compiler.ign index 6c62cf2b..ff98d3aa 100644 --- a/std/regex/compiler.ign +++ b/std/regex/compiler.ign @@ -51,6 +51,10 @@ export record RegexProgram { /// Two slots per group, the whole match being group 0. public slotCount: u64; + + /// The UTF-8 bytes every match starts with; empty when there is no such + /// literal prefix. The search skips to its occurrences. + public prefix: String; } /// `ASSERT` operand codes. @@ -282,6 +286,53 @@ function regexCompileNode(compiler: &mut RegexCompiler, ast: &RegexAst, id: u64) }; } +/// Appends to `prefix` the literal scalars every match of node `id` starts +/// with, and reports whether the node is exactly those scalars, so that a +/// concatenation may continue past it. +function regexAppendPrefix(ast: &RegexAst, id: u64, prefix: &mut String): boolean { + match (ast.nodes.get(id).unwrap()) { + RegexNode::LITERAL(scalar) -> { + prefix.pushChar(scalar as char); + return true; + }, + RegexNode::CONCAT(children) -> { + let mut index: u64 = 0; + + while (index < children.length()) { + if (!regexAppendPrefix(ast, *children.get(index).unwrap(), prefix)) { + return false; + } + + index += 1; + } + + return true; + }, + RegexNode::GROUP(group) -> { + return regexAppendPrefix(ast, group.child, prefix); + }, + RegexNode::REPEAT(repeat) -> { + // The first iteration is required; what follows it is not known. + if (repeat.min >= 1) { + regexAppendPrefix(ast, repeat.child, prefix); + } + + return false; + }, + _ -> { + return false; + }, + }; +} + +/// The literal prefix of every match of `ast`. Under `(?i)` letters are +/// classes, so a case-insensitive pattern has no letter in its prefix. +function regexLiteralPrefix(ast: &RegexAst): String { + let mut prefix: String = String::new(); + regexAppendPrefix(ast, ast.root, &mut prefix); + return prefix; +} + /// Compiles `ast` to `SAVE 0, , SAVE 1, MATCH`, or fails with /// `pattern too large` when the program would exceed `REGEX_MAX_PROGRAM`. export function compileRegex(ast: &RegexAst): Result { @@ -304,5 +355,6 @@ export function compileRegex(ast: &RegexAst): Result { insts: compiler.insts, ranges: compiler.ranges, slotCount: (ast.groupCount + 1) * 2, + prefix: regexLiteralPrefix(ast), }); } diff --git a/std/regex/mod.ign b/std/regex/mod.ign index 9031a4a5..da75afb1 100644 --- a/std/regex/mod.ign +++ b/std/regex/mod.ign @@ -7,13 +7,23 @@ //! backtracks. //! //! ```ignis -//! import Match, Regex from "std::regex"; +//! import Captures, Match, Regex from "std::regex"; //! -//! let regex: Regex = Regex::new("\\d+").unwrap(); -//! let found: Match = regex.find("abc 123").unwrap(); // 4..7 +//! let digits: Regex = Regex::new("\\d+").unwrap(); +//! let found: Match = digits.find("abc 123").unwrap(); // 4..7 +//! let number: String = found.textIn("abc 123"); // "123" +//! +//! let date: Regex = Regex::new("(?\\d{4})-(\\d\\d)").unwrap(); +//! let captures: Captures = date.captures("on 2024-05").unwrap(); +//! let year: Match = captures.name("year").unwrap(); // 3..7 +//! let swapped: String = date.replaceAll("on 2024-05", "$2/${year}"); // "on 05/2024" +//! +//! let comma: Regex = Regex::new("\\s*,\\s*").unwrap(); +//! let parts: Vector = comma.split("a , b,c"); // "a", "b", "c" //! ``` //! -//! Captures, `replaceAll` and `split` arrive in the next slices. +//! A pattern is a `str`, so a regex backslash is written `\\` in an Ignis +//! string literal: `"\\d+"` is the pattern `\d+`. //! //! ## Matching //! @@ -27,21 +37,59 @@ //! - `findAll` returns non-overlapping matches; after an empty match it //! resumes one scalar further on, so `a*` on `"baa"` gives `0..0`, //! `1..3` and `3..3`. +//! - Group 0 is the whole match; group `n` is the `n`-th `(`. A group that +//! did not take part in the match is `NONE` (`(a)|(b)` on `"b"`: group 1 +//! `NONE`, group 2 `0..1`), and a group in a repetition holds its last +//! iteration (`(a|b)+` on `"ab"`: group 1 `1..2`). +//! - `replaceAll`, `split` and `capturesAll` use the matches `findAll` +//! finds, so `x*` replaced by `-` in `"abc"` gives `"-a-b-c-"`. //! - Matching takes O(text x program) time: the VM runs every thread in //! lockstep and follows each instruction at most once per position, so //! patterns such as `(a*)*b` cannot backtrack. //! +//! ## Replacement +//! +//! In the replacement text of `replace` and `replaceAll`: +//! +//! - `$n`, one or more digits read greedily (`$12` is group 12), and `${n}` +//! insert group `n`; `${1}0` is group 1 followed by `0`. +//! - `${name}` inserts the named group. Braces hold `[A-Za-z0-9_]+`: all +//! digits is a number, anything else a name. +//! - A group that did not take part or does not exist inserts nothing. +//! - `$$` is a literal `$`; any other `$` (`$x`, `${}`, `${a b}`, a final +//! `$`) is a literal `$`. A name needs braces: `$name` is the text +//! `$name`, not the group. +//! //! ## Limits //! -//! Counted repetition copies its body, so a program holds at most -//! `REGEX_MAX_PROGRAM` (100000) instructions; a larger one is -//! `pattern too large` at column 1. +//! - A count in `{n}`, `{n,}` or `{n,m}` is at most `REGEX_MAX_REPETITION` +//! (1000); a larger one is `repetition count above 1000`. +//! - Counted repetition copies its body, so a program holds at most +//! `REGEX_MAX_PROGRAM` (100000) instructions; a larger one, such as +//! `(a{1000}){1000}`, is `pattern too large` at column 1. +//! +//! ## Performance +//! +//! - Matching is linear: a Pike VM runs every thread in lockstep over the +//! text, one scalar per step, and follows each instruction at most once per +//! position, so a search takes O(text x program) time and memory +//! proportional to the program, whatever the pattern. +//! - When every match starts with the same literal scalars (`foo(bar|baz)` +//! starts with `foo`), the search skips straight to the next occurrence of +//! that prefix whenever no thread is alive, and stops when there is none. +//! A pattern that starts with `^`, a class, an alternation, a repetition +//! that may be empty or a case-insensitive letter has no prefix. +//! - `Regex` holds only the compiled program. Every call (`isMatch`, `find`, +//! `findAll`, ...) allocates its own VM state, sized by the program, and +//! copies the text once; a call on many matches reuses that state. +//! - `isMatch` stops at the first match found and is the cheapest test. //! //! ## Exports //! //! | Export | Description | //! |------------------------|----------------------------------------------------------------| -//! | `Regex` | `Regex::new(pattern)`: `Result`; `isMatch(text)`, `find(text)`: `Option`, `findAll(text)`: `Vector` | +//! | `Regex` | `Regex::new(pattern)`: `Result`; `isMatch(text)`, `find(text)`: `Option`, `findAll(text)`: `Vector`, `captures(text)`: `Option`, `capturesAll(text)`: `Vector`, `replace(text, replacement)` and `replaceAll(text, replacement)`: `String`, `split(text)`: `Vector`, `groupCount()`, `groupName(index)`: `Option`, `groupIndex(name)`: `Option` | +//! | `Captures` | `group(index)` and `name(name)`: `Option`, `whole()`: `Match`, `groupCount()` (whole match excluded), `length()` (included) | //! | `Match` | `start()`, `end()`, `length()` in bytes, `isEmpty()`, `textIn(haystack)`: a copy of the matched text | //! | `REGEX_MAX_PROGRAM` | The most instructions a compiled pattern may hold: 100000 | //! | `parseRegex` | `parseRegex(pattern)`: `Result` | @@ -98,9 +146,17 @@ //! 1000`, `invalid group name`, `duplicate group name x`, `unknown group //! flag`, `unsupported: backreference` and `unsupported: lookaround`. //! -//! Backreferences (`\1`) and lookaround (`(?=`, `(?!`, `(?<=`, `(?; + + /// The name of each group, empty when it has none; entry 0 is the whole + /// match. + names: Vector; + + /// Group `index`: 0 is the whole match. `NONE` when the group did not take + /// part in the match or does not exist. In a repetition the group holds + /// its last iteration. + public group(&self, index: u64): Option { + if (index * 2 + 1 >= self.slots.length()) { + return Option::NONE; + } + + let start: u64 = *self.slots.get(index * 2).unwrap(); + let end: u64 = *self.slots.get(index * 2 + 1).unwrap(); + + if (start == REGEX_NO_SLOT || end == REGEX_NO_SLOT) { + return Option::NONE; + } + + return Option::SOME(Match::create(start, end)); + } + + /// The group named `name`; `NONE` when no group has that name or it did not + /// take part in the match. + public name(&self, name: str): Option { + let Option::SOME(index) = regexGroupIndex(&self.names, name) else { + return Option::NONE; + }; + + return self.group(index); + } + + /// The whole match, group 0. + public whole(&self): Match { + return Match::create(*self.slots.get(0).unwrap(), *self.slots.get(1).unwrap()); + } + + /// The number of capturing groups, the whole match not included. + public groupCount(&self): u64 { + return self.slots.length() / 2 - 1; + } + + /// The number of groups, the whole match included: `groupCount() + 1`. + public length(&self): u64 { + return self.slots.length() / 2; + } +} + /// A compiled regular expression. export record Regex { program: RegexProgram; + /// The name of each group, empty when it has none; entry 0 is the whole + /// match. + names: Vector; + /// Parses and compiles `pattern`. public static new(pattern: str): Result { match (parseRegex(pattern)) { Result::OK(ast) -> { match (compileRegex(&ast)) { Result::OK(program) -> { - return Result::OK(Regex { program: program }); + return Result::OK(Regex { program: program, names: regexGroupNames(&ast) }); }, Result::ERROR(error) -> { return Result::ERROR(error); @@ -67,6 +125,30 @@ export record Regex { }; } + /// The number of capturing groups, the whole match not included. + public groupCount(&self): u64 { + return self.names.length() - 1; + } + + /// The name of group `index`; `NONE` when the group has no name or does + /// not exist. + public groupName(&self, index: u64): Option { + let Option::SOME(name) = self.names.get(index) else { + return Option::NONE; + }; + + if (name.length() == 0) { + return Option::NONE; + } + + return Option::SOME(name.clone()); + } + + /// The index of the group named `name`, or `NONE`. + public groupIndex(&self, name: str): Option { + return regexGroupIndex(&self.names, name); + } + /// Whether `text` contains a match. public isMatch(&self, text: str): boolean { let source: String = String::create(text); @@ -78,52 +160,378 @@ export record Regex { public find(&self, text: str): Option { let source: String = String::create(text); let mut vm: RegexVm = regexNewVm(&self.program); - return regexFindFrom(&mut vm, &self.program, &source, 0); + let Option::SOME(slots) = regexSearch(&mut vm, &self.program, &source, 0, false) else { + return Option::NONE; + }; + + return Option::SOME(regexWholeMatch(&slots)); } /// Every non-overlapping match in `text`, left to right. After an empty /// match the search resumes one scalar further on. public findAll(&self, text: str): Vector { - let mut matches: Vector = Vector::new(); + let source: String = String::create(text); + let all: Vector> = regexSearchAll(&self.program, &source, 0); + let mut matches: Vector = Vector::new(all.length()); + let mut index: u64 = 0; + + while (index < all.length()) { + matches.push(regexWholeMatch(all.get(index).unwrap())); + index += 1; + } + + return matches; + } + + /// The leftmost-first match in `text` with every group. + public captures(&self, text: str): Option { let source: String = String::create(text); let mut vm: RegexVm = regexNewVm(&self.program); - let length: u64 = source.length(); - let mut startAt: u64 = 0; + let Option::SOME(slots) = regexSearch(&mut vm, &self.program, &source, 0, false) else { + return Option::NONE; + }; - while (startAt <= length) { - let Option::SOME(found) = regexFindFrom(&mut vm, &self.program, &source, startAt) else { - break; - }; + return Option::SOME(Captures { slots: slots, names: regexCopyNames(&self.names) }); + } - matches.push(found); + /// Every non-overlapping match in `text` with its groups, as `findAll` + /// finds them. + public capturesAll(&self, text: str): Vector { + let source: String = String::create(text); + let all: Vector> = regexSearchAll(&self.program, &source, 0); + let mut result: Vector = Vector::new(all.length()); + let mut index: u64 = 0; - if (!found.isEmpty()) { - startAt = found.end(); - continue; - } + while (index < all.length()) { + let slots: Vector = regexCopySlots(all.get(index).unwrap()); + result.push(Captures { slots: slots, names: regexCopyNames(&self.names) }); + index += 1; + } - if (found.end() >= length) { - break; - } + return result; + } + + /// `text` with every non-overlapping match, as `findAll` finds them, + /// replaced by `replacement` expanded for that match. + /// + /// In `replacement`, `$n` (one or more digits, read greedily) and `${n}` + /// insert group `n`, `${name}` the named group, and `$$` a `$`. Braces + /// hold `[A-Za-z0-9_]+`: all digits is a number, anything else a name. A + /// group that did not take part or does not exist inserts nothing. Any + /// other `$`, such as `$x`, `${}` or `${a b}`, is a literal `$`. + public replaceAll(&self, text: str, replacement: str): String { + return regexReplace(self, text, replacement, true); + } + + /// `text` with its first match replaced by `replacement`, expanded as in + /// `replaceAll`. + public replace(&self, text: str, replacement: str): String { + return regexReplace(self, text, replacement, false); + } + + /// The pieces of `text` between the matches `findAll` finds. A match at + /// the start or end of `text` leaves an empty first or last piece, an + /// empty match splits between scalars, and without a match the result is + /// the whole text. + public split(&self, text: str): Vector { + let source: String = String::create(text); + let all: Vector> = regexSearchAll(&self.program, &source, 0); + let mut pieces: Vector = Vector::new(all.length() + 1); + let mut last: u64 = 0; + let mut index: u64 = 0; - let mut next: u64 = found.end(); - source.charAt(found.end(), &mut next); - startAt = next; + while (index < all.length()) { + let found: Match = regexWholeMatch(all.get(index).unwrap()); + pieces.push(source.substringBytes(last, found.start())); + last = found.end(); + index += 1; } - return matches; + pieces.push(source.substringBytes(last, source.length())); + return pieces; } } -function regexFindFrom( - vm: &mut RegexVm, +/// The literal prefix the search of `regex` skips to, empty when there is +/// none. Not exported by `std::regex`: tests use it. +export function regexLiteralPrefix(regex: &Regex): String { + return regex.program.prefix.clone(); +} + +function regexWholeMatch(slots: &Vector): Match { + return Match::create(*slots.get(0).unwrap(), *slots.get(1).unwrap()); +} + +/// The slots of every non-overlapping match from `startAt` on. After an +/// empty match the search resumes one scalar further on. +function regexSearchAll( program: &RegexProgram, - text: &String, + source: &String, startAt: u64 -): Option { - let Option::SOME(slots) = regexSearch(vm, program, text, startAt, false) else { +): Vector> { + let mut all: Vector> = Vector::new>(); + let mut vm: RegexVm = regexNewVm(program); + let length: u64 = source.length(); + let mut position: u64 = startAt; + + while (position <= length) { + let Option::SOME(slots) = regexSearch(&mut vm, program, source, position, false) else { + break; + }; + + let found: Match = regexWholeMatch(&slots); + all.push(slots); + + if (!found.isEmpty()) { + position = found.end(); + continue; + } + + if (found.end() >= length) { + break; + } + + let mut next: u64 = found.end(); + source.charAt(found.end(), &mut next); + position = next; + } + + return all; +} + +/// The name of every group of `ast`, indexed by group, entry 0 empty. +function regexGroupNames(ast: &RegexAst): Vector { + let mut names: Vector = Vector::new(ast.groupCount + 1); + let mut index: u64 = 0; + + while (index <= ast.groupCount) { + names.push(String::new()); + index += 1; + } + + index = 0; + + while (index < ast.nodes.length()) { + match (ast.nodes.get(index).unwrap()) { + RegexNode::GROUP(group) -> { + if (group.capturing && group.name.length() > 0) { + names.set(group.index, group.name.clone()); + } + }, + _ -> {}, + }; + + index += 1; + } + + return names; +} + +function regexCopySlots(slots: &Vector): Vector { + let mut copy: Vector = Vector::new(slots.length()); + let mut index: u64 = 0; + + while (index < slots.length()) { + copy.push(*slots.get(index).unwrap()); + index += 1; + } + + return copy; +} + +function regexCopyNames(names: &Vector): Vector { + let mut copy: Vector = Vector::new(names.length()); + let mut index: u64 = 0; + + while (index < names.length()) { + copy.push(names.get(index).unwrap().clone()); + index += 1; + } + + return copy; +} + +function regexGroupIndex(names: &Vector, name: str): Option { + if (String::create(name).length() == 0) { return Option::NONE; - }; + } + + let mut index: u64 = 1; + + while (index < names.length()) { + if (names.get(index).unwrap().equals(name)) { + return Option::SOME(index); + } + + index += 1; + } + + return Option::NONE; +} + +function regexIsAsciiDigit(byte: u8): boolean { + return byte >= '0' as u8 && byte <= '9' as u8; +} + +function regexIsNameByte(byte: u8): boolean { + return regexIsAsciiDigit(byte) || (byte >= 'A' as u8 && byte <= 'Z' as u8) || (byte >= 'a' as u8 && byte <= 'z' as u8) || byte == '_' as u8; +} + +/// Group numbers past this are read but can name no group; it keeps the +/// accumulated value from overflowing. +const REGEX_GROUP_NUMBER_CAP: u64 = 1000000; + +function regexPushGroup(out: &mut String, source: &String, slots: &Vector, index: u64): void { + if (index * 2 + 1 >= slots.length()) { + return; + } + + let start: u64 = *slots.get(index * 2).unwrap(); + let end: u64 = *slots.get(index * 2 + 1).unwrap(); + + if (start == REGEX_NO_SLOT || end == REGEX_NO_SLOT) { + return; + } + + let piece: String = source.substringBytes(start, end); + out.push(&piece); +} + +/// Appends `replacement` to `out` with its `$` references expanded against +/// one match; see `Regex::replaceAll`. +function regexExpand( + out: &mut String, + source: &String, + slots: &Vector, + names: &Vector, + replacement: &String +): void { + let length: u64 = replacement.length(); + let mut position: u64 = 0; + + while (position < length) { + let byte: u8 = replacement.byteAt(position).unwrap(); + + if (byte != '$' as u8 || position + 1 >= length) { + out.pushByte(byte); + position += 1; + continue; + } + + let next: u8 = replacement.byteAt(position + 1).unwrap(); + + if (next == '$' as u8) { + out.pushByte('$' as u8); + position += 2; + continue; + } + + if (regexIsAsciiDigit(next)) { + let mut index: u64 = 0; + let mut cursor: u64 = position + 1; + + while (cursor < length && regexIsAsciiDigit(replacement.byteAt(cursor).unwrap())) { + if (index < REGEX_GROUP_NUMBER_CAP) { + index = index * 10 + (replacement.byteAt(cursor).unwrap() - '0' as u8) as u64; + } + + cursor += 1; + } + + regexPushGroup(out, source, slots, index); + position = cursor; + continue; + } + + if (next == '{' as u8) { + let mut cursor: u64 = position + 2; + + while (cursor < length && regexIsNameByte(replacement.byteAt(cursor).unwrap())) { + cursor += 1; + } + + if (cursor < length && cursor > position + 2 && replacement.byteAt(cursor).unwrap() == '}' as u8) { + let reference: String = replacement.substringBytes(position + 2, cursor); + regexPushReference(out, source, slots, names, &reference); + position = cursor + 1; + continue; + } + } + + out.pushByte(byte); + position += 1; + } +} + +/// Appends the group `reference` names: a number when it is all digits, +/// otherwise a group name. +function regexPushReference( + out: &mut String, + source: &String, + slots: &Vector, + names: &Vector, + reference: &String +): void { + let mut index: u64 = 0; + let mut cursor: u64 = 0; + let mut numeric: boolean = true; + + while (cursor < reference.length()) { + let byte: u8 = reference.byteAt(cursor).unwrap(); + + if (!regexIsAsciiDigit(byte)) { + numeric = false; + break; + } + + if (index < REGEX_GROUP_NUMBER_CAP) { + index = index * 10 + (byte - '0' as u8) as u64; + } + + cursor += 1; + } + + if (!numeric) { + let Option::SOME(named) = regexGroupIndex(names, reference.toStr()) else { + return; + }; + + index = named; + } + + regexPushGroup(out, source, slots, index); +} + +function regexReplace(regex: &Regex, text: str, replacement: str, every: boolean): String { + let source: String = String::create(text); + let expansion: String = String::create(replacement); + let mut all: Vector> = Vector::new>(); + + if (every) { + all = regexSearchAll(®ex.program, &source, 0); + } else { + let mut vm: RegexVm = regexNewVm(®ex.program); + + if (let Option::SOME(slots) = regexSearch(&mut vm, ®ex.program, &source, 0, false)) { + all.push(slots); + } + } + + let mut out: String = String::new(); + let mut last: u64 = 0; + let mut index: u64 = 0; + + while (index < all.length()) { + let slots: &Vector = all.get(index).unwrap(); + let found: Match = regexWholeMatch(slots); + let before: String = source.substringBytes(last, found.start()); + out.push(&before); + regexExpand(&mut out, &source, slots, ®ex.names, &expansion); + last = found.end(); + index += 1; + } - return Option::SOME(Match::create(*slots.get(0).unwrap(), *slots.get(1).unwrap())); + let rest: String = source.substringBytes(last, source.length()); + out.push(&rest); + return out; } diff --git a/std/regex/tests.ign b/std/regex/tests.ign index 474e3a8f..05852b3a 100644 --- a/std/regex/tests.ign +++ b/std/regex/tests.ign @@ -1,6 +1,7 @@ import Test from "std::test"; -import Match, parseRegex, Regex from "std::regex"; +import Captures, Match, parseRegex, Regex from "std::regex"; import regexAstDebug from "./ast"; +import regexLiteralPrefix from "./regex"; function regexTestShape(pattern: str, expected: str): void { match (parseRegex(pattern)) { @@ -418,3 +419,217 @@ function matchesNestedStarsInLinearTime(): void { let alternatives: Regex = regexTestCompile("(a|aa)*c"); Test::assertEq(alternatives.isMatch(text.toStr()), false); } + +/// Every group of the first match of `pattern` in `text` as `start..end` or +/// `none`, comma-separated; `no match` without a match. +function regexTestCaptures(pattern: str, text: str, expected: str): void { + let regex: Regex = regexTestCompile(pattern); + let mut shown: String = String::new(); + + match (regex.captures(text)) { + Option::SOME(captures) -> { + let mut index: u64 = 0; + + while (index < captures.length()) { + if (index > 0) { + shown.pushStr(","); + } + + match (captures.group(index)) { + Option::SOME(found) -> { + regexTestPushMatch(&mut shown, &found); + }, + Option::NONE -> { + shown.pushStr("none"); + }, + }; + + index += 1; + } + }, + Option::NONE -> { + shown.pushStr("no match"); + }, + }; + + Test::assertEq(shown.toStr(), expected); +} + +function regexTestReplaceAll(pattern: str, text: str, replacement: str, expected: str): void { + let result: String = regexTestCompile(pattern).replaceAll(text, replacement); + Test::assertEq(result.toStr(), expected); +} + +/// The pieces `split` gives, joined with `|`. +function regexTestSplit(pattern: str, text: str, expected: str): void { + let pieces: Vector = regexTestCompile(pattern).split(text); + let mut shown: String = String::new(); + let mut index: u64 = 0; + + while (index < pieces.length()) { + if (index > 0) { + shown.pushStr("|"); + } + + shown.push(pieces.get(index).unwrap()); + index += 1; + } + + Test::assertEq(shown.toStr(), expected); +} + +@test +function capturesNumberedAndNestedGroups(): void { + regexTestCaptures("(\\d+)-(\\d+)", "x 12-345", "2..8,2..4,5..8"); + regexTestCaptures("((a)(b))c", "abc", "0..3,0..2,0..1,1..2"); + regexTestCaptures("(?:a)(b)", "ab", "0..2,1..2"); + regexTestCaptures("(a)", "xyz", "no match"); +} + +@test +function capturesNonParticipatingGroupsAsNone(): void { + regexTestCaptures("(a)|(b)", "b", "0..1,none,0..1"); + regexTestCaptures("(a)?b", "b", "0..1,none"); + regexTestCaptures("(a)*", "", "0..0,none"); +} + +@test +function capturesTheLastIterationOfARepetition(): void { + regexTestCaptures("(a|b)+", "ab", "0..2,1..2"); + regexTestCaptures("(?:(a)|b)+", "ab", "0..2,0..1"); + regexTestCaptures("(\\w)*!", "xyz!", "0..4,2..3"); +} + +@test +function capturesUtf8Offsets(): void { + regexTestCaptures("(é+)(.)", "aééü", "1..7,1..5,5..7"); +} + +@test +function looksUpNamedGroups(): void { + let regex: Regex = regexTestCompile("(?\\d{4})-(\\d\\d)-(?\\d\\d)"); + Test::assertEq(regex.groupCount(), 3); + Test::assertEq(regex.groupIndex("day").unwrap(), 3); + Test::assertEq(regex.groupIndex("month").isNone(), true); + Test::assertEq(regex.groupName(2).isNone(), true); + let name: String = regex.groupName(1).unwrap(); + Test::assertEq(name.toStr(), "year"); + let captures: Captures = regex.captures("on 2024-05-17").unwrap(); + Test::assertEq(captures.groupCount(), 3); + Test::assertEq(captures.length(), 4); + let year: Match = captures.name("year").unwrap(); + Test::assertEq(year.start(), 3); + Test::assertEq(year.end(), 7); + let day: String = captures.name("day").unwrap().textIn("on 2024-05-17"); + Test::assertEq(day.toStr(), "17"); + Test::assertEq(captures.name("month").isNone(), true); + Test::assertEq(captures.name("").isNone(), true); + Test::assertEq(captures.group(4).isNone(), true); + Test::assertEq(captures.whole().start(), 3); + Test::assertEq(regexTestCompile("abc").groupCount(), 0); +} + +@test +function capturesEveryMatch(): void { + let regex: Regex = regexTestCompile("(\\w)(\\d)?"); + let all: Vector = regex.capturesAll("a1 b c3"); + Test::assertEq(all.length(), 3); + Test::assertEq(all.get(1).unwrap().group(1).unwrap().start(), 3); + Test::assertEq(all.get(1).unwrap().group(2).isNone(), true); + Test::assertEq(all.get(2).unwrap().group(2).unwrap().start(), 6); +} + +@test +function replacesWithGroupReferences(): void { + regexTestReplaceAll("(\\w+)@(\\w+)", "a@b, cd@ef", "$2 at $1", "b at a, ef at cd"); + regexTestReplaceAll("(?\\w+)=(?\\w+)", "x=1 y=2", "${value}:${key}", "1:x 2:y"); + regexTestReplaceAll("\\d+", "a1b22", "$$", "a$b$"); + regexTestReplaceAll("\\d+", "a1", "$x ${ ${} $", "a$x ${ ${} $"); + regexTestReplaceAll("(a)|(b)", "ab", "[$1$2$3${missing}]", "[a][b]"); + regexTestReplaceAll("(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)", "abcdefghijkl", "$12${1}0$10", "la0j"); + regexTestReplaceAll("\\d", "a1", "$99999999999999999999", "a"); + regexTestReplaceAll("x", "abc", "-", "abc"); +} + +@test +function replacesEmptyMatches(): void { + regexTestReplaceAll("x*", "abc", "-", "-a-b-c-"); + regexTestReplaceAll("", "é", "-", "-é-"); +} + +@test +function replacesOnlyTheFirstMatch(): void { + let regex: Regex = regexTestCompile("(\\d)"); + let first: String = regex.replace("a1b2", "<$1>"); + Test::assertEq(first.toStr(), "a<1>b2"); + let none: String = regex.replace("ab", "<$1>"); + Test::assertEq(none.toStr(), "ab"); +} + +@test +function splitsBetweenMatches(): void { + regexTestSplit(",", "a,b,,c", "a|b||c"); + regexTestSplit("\\s*,\\s*", "a , b,c", "a|b|c"); + regexTestSplit(",", ",a,", "|a|"); + regexTestSplit(",", "abc", "abc"); + regexTestSplit(",", "", ""); + regexTestSplit("", "abc", "|a|b|c|"); + regexTestSplit("x*", "aé", "|a|é|"); +} + +function regexTestPrefix(pattern: str, expected: str): void { + let regex: Regex = regexTestCompile(pattern); + let prefix: String = regexLiteralPrefix(®ex); + Test::assertEq(prefix.toStr(), expected); +} + +@test +function extractsTheLiteralPrefix(): void { + regexTestPrefix("abc", "abc"); + regexTestPrefix("foo(bar|baz)", "foo"); + regexTestPrefix("(?:ab)(c)d+e", "abcd"); + regexTestPrefix("ab*c", "a"); + regexTestPrefix("é\\d", "é"); + regexTestPrefix("(?i)1ab", "1"); + regexTestPrefix("", ""); + regexTestPrefix("^abc", ""); + regexTestPrefix("[a]bc", ""); + regexTestPrefix("a*bc", ""); + regexTestPrefix("ab|ac", ""); + regexTestPrefix("(?i)abc", ""); + regexTestPrefix("\\bab", ""); +} + +@test +function skipsToTheLiteralPrefix(): void { + regexTestFind("needle", "hay hay needle hay", "8..14"); + regexTestFind("needle", "hay hay needl", "none"); + regexTestFind("aab", "aaab", "1..4"); + regexTestFindAll("aab", "aaabaab", "1..4,4..7"); + regexTestFind("é+x", "aééx", "1..6"); + regexTestFind("foo(bar|baz)", "foobaq foobaz", "7..13"); + regexTestFindAll("ab", "abxab", "0..2,3..5"); + regexTestCaptures("a(b+)c", "xxabbc", "2..6,3..5"); +} + +@test +function searchesWithoutAPrefix(): void { + regexTestFind("[ab]c", "xxbc", "2..4"); + regexTestFind("a*b", "cccb", "3..4"); + regexTestFind("x|yz", "ayz", "1..3"); + regexTestFind("(?i)abc", "xABC", "1..4"); + regexTestFindAll("", "ab", "0..0,1..1,2..2"); +} + +@test +function matchesLazyAndCaseInsensitiveForms(): void { + regexTestFind("a{2,3}?", "aaaa", "0..2"); + regexTestFind("a{2,}?", "aaaa", "0..2"); + regexTestFind("a+?", "aaa", "0..1"); + regexTestFind("a??b", "ab", "0..2"); + regexTestFind("(?i)straße", "STRAßE", "0..7"); + regexTestFind("(?i)straße", "STRASSE", "none"); + regexTestFind("(?i)ß", "ẞ", "none"); + regexTestFind("(?i:a)b", "Ab", "0..2"); + regexTestFind("(?i:a)b", "AB", "none"); +} diff --git a/std/regex/vm.ign b/std/regex/vm.ign index 5b9c3130..6d944fdc 100644 --- a/std/regex/vm.ign +++ b/std/regex/vm.ign @@ -276,6 +276,32 @@ export function regexNewVm(program: &RegexProgram): RegexVm { }; } +/// The first byte offset at or after `startAt` where `needle` occurs in `text`. +function regexFindBytes(text: &String, needle: &String, startAt: u64): Option { + let length: u64 = text.length(); + let needleLength: u64 = needle.length(); + let first: u8 = needle.byteAt(0).unwrap(); + let mut start: u64 = startAt; + + while (start + needleLength <= length) { + if (text.byteAt(start).unwrap() == first) { + let mut offset: u64 = 1; + + while (offset < needleLength && text.byteAt(start + offset).unwrap() == needle.byteAt(offset).unwrap()) { + offset += 1; + } + + if (offset == needleLength) { + return Option::SOME(start); + } + } + + start += 1; + } + + return Option::NONE; +} + /// Searches `text` from byte offset `startAt` for the leftmost-first match. /// Returns its slots (`slotCount` entries, `REGEX_NO_SLOT` where a group did /// not take part), or `NONE`. With `earliest` it stops at the first match @@ -294,6 +320,19 @@ export function regexSearch( let mut position: u64 = startAt; while (true) { + // With no live thread, nothing can match before the next occurrence of + // the literal prefix, so the search resumes there. + if (best.isNone() && vm.current.pcs.length() == 0 && program.prefix.length() > 0) { + let Option::SOME(found) = regexFindBytes(text, &program.prefix, position) else { + break; + }; + + if (found != position) { + regexClearThreadList(&mut vm.current); + position = found; + } + } + // A new start is the lowest-priority thread, and stops once a match is // known: anything starting later is not leftmost. if (best.isNone()) { diff --git a/test_cases/doc/__doc_baselines__/output__std_io_mod.json b/test_cases/doc/__doc_baselines__/output__std_io_mod.json index 64f5f510..893f48c7 100644 --- a/test_cases/doc/__doc_baselines__/output__std_io_mod.json +++ b/test_cases/doc/__doc_baselines__/output__std_io_mod.json @@ -286,7 +286,7 @@ }, { "name": "std::regex", - "doc": " \n# Regex \n \nRegular expressions with linear-time matching, in the style of RE2: a \npattern is parsed into an AST, compiled to a program and run by a Pike VM \nthat simulates the NFA, so matching takes O(text x program) time and never \nbacktracks. \n \n```ignis \nimport Match, Regex from \"std::regex\"; \n \nlet regex: Regex = Regex::new(\"\\\\d+\").unwrap(); \nlet found: Match = regex.find(\"abc 123\").unwrap(); // 4..7 \n``` \n \nCaptures, `replaceAll` and `split` arrive in the next slices. \n \n## Matching \n \n- Leftmost-first: of the matches starting at the leftmost position, the \n one the pattern prefers wins. Alternatives are tried in order (`a|ab` \n on `\"ab\"` matches `\"a\"`), greedy repetition prefers more and lazy \n repetition fewer. \n- The search is unanchored unless the pattern uses `^`. \n- Offsets are byte offsets into the UTF-8 text and always fall on scalar \n boundaries. An invalid byte is read as U+FFFD and spans one byte. \n- `findAll` returns non-overlapping matches; after an empty match it \n resumes one scalar further on, so `a*` on `\"baa\"` gives `0..0`, \n `1..3` and `3..3`. \n- Matching takes O(text x program) time: the VM runs every thread in \n lockstep and follows each instruction at most once per position, so \n patterns such as `(a*)*b` cannot backtrack. \n \n## Limits \n \nCounted repetition copies its body, so a program holds at most \n`REGEX_MAX_PROGRAM` (100000) instructions; a larger one is \n`pattern too large` at column 1. \n \n## Exports \n \n| Export | Description | \n|------------------------|----------------------------------------------------------------| \n| `Regex` | `Regex::new(pattern)`: `Result`; `isMatch(text)`, `find(text)`: `Option`, `findAll(text)`: `Vector` | \n| `Match` | `start()`, `end()`, `length()` in bytes, `isEmpty()`, `textIn(haystack)`: a copy of the matched text | \n| `REGEX_MAX_PROGRAM` | The most instructions a compiled pattern may hold: 100000 | \n| `parseRegex` | `parseRegex(pattern)`: `Result` | \n| `RegexError` | `column()` (1-based, in Unicode scalars), `message()`, `toString()` | \n| `RegexAst` | `nodes`, `root`, `groupCount` | \n| `RegexNode` | `EMPTY`, `LITERAL`, `ANY_EXCEPT_NEWLINE`, `CLASS`, `CONCAT`, `ALTERNATE`, `REPEAT`, `GROUP`, `ASSERT` | \n| `RegexClass` | Sorted, merged `ranges` and `negated` | \n| `RegexRange` | Inclusive `start` and `end` scalar values | \n| `RegexRepeat` | `child`, `min`, `max`, `bounded`, `greedy` | \n| `RegexGroup` | `child`, `capturing`, `index`, `name` | \n| `RegexAssertion` | `START`, `END`, `WORD_BOUNDARY`, `NOT_WORD_BOUNDARY` | \n| `REGEX_MAX_REPETITION` | The largest count `{n,m}` accepts: 1000 | \n \n## Syntax \n \n| Syntax | Meaning | \n|-----------------------------------|----------------------------------------------------------| \n| `x` | The scalar `x`, any UTF-8 scalar | \n| `.` | Any scalar except `\\n` | \n| `\\n` `\\t` `\\r` `\\f` `\\v` | Control scalars | \n| `\\.` `\\*` `\\\\` ... | Any escaped ASCII punctuation is itself | \n| `[abc]` `[a-z]` `[^...]` | Classes, ranges, negation; escapes work inside | \n| `\\d` `\\w` `\\s` and `\\D` `\\W` `\\S` | `[0-9]`, `[0-9A-Za-z_]`, `[ \\t\\n\\r\\f\\v]` and negations (ASCII) | \n| `^` `$` | Start and end of the whole text, not of a line | \n| `\\b` `\\B` | ASCII word boundary and its negation | \n| `(x)` `(?:x)` `(?x)` | Capturing, non-capturing and named groups | \n| `x\\|y` | Alternation, leftmost-first | \n| `x*` `x+` `x?` | Zero or more, one or more, zero or one | \n| `x{n}` `x{n,}` `x{n,m}` | Counted repetition, counts at most 1000 | \n| `x*?` `x+?` `x??` `x{n,m}?` | Lazy forms: prefer fewer repetitions | \n| `(?i)` `(?i:x)` | ASCII case-insensitive for the rest of the enclosing group, or inside `x` | \n \nGroup names are `[A-Za-z_][A-Za-z0-9_]*` and must be unique. Capturing \ngroups are numbered from 1 in the order of their `(`. \n \nThe parser is strict where a pattern would otherwise be ambiguous: \n \n- `{` always starts a counted repetition; a literal brace is `\\{`. A \n `{` that is not `{n}`, `{n,}` or `{n,m}` is `invalid repetition`. A \n lone `}` or `]` is a literal. \n- A quantifier after a quantifier (`a**`, `a+*`, `a{2}{3}`) and a \n quantifier after `^`, `$`, `\\b` or `\\B` are `nothing to repeat`; the \n `?` of a lazy form is part of its quantifier. \n- `[]` and `[^]` are `empty class`; a `]` inside a class is `\\]`. A `-` \n first or last in a class is a literal. \n \n## Errors \n \nEvery error carries the 1-based column, counted in Unicode scalars, of \nthe part that is wrong: `unclosed group` (at its `(`), `unmatched )`, \n`nothing to repeat`, `invalid range z-a` (at its `-`), `unclosed class` \n(at its `[`), `empty class`, `trailing backslash`, `unknown escape \\q`, \n`invalid repetition`, `invalid repetition range`, `repetition count above \n1000`, `invalid group name`, `duplicate group name x`, `unknown group \nflag`, `unsupported: backreference` and `unsupported: lookaround`. \n \nBackreferences (`\\1`) and lookaround (`(?=`, `(?!`, `(?<=`, `(?\\\\d{4})-(\\\\d\\\\d)\").unwrap(); \nlet captures: Captures = date.captures(\"on 2024-05\").unwrap(); \nlet year: Match = captures.name(\"year\").unwrap(); // 3..7 \nlet swapped: String = date.replaceAll(\"on 2024-05\", \"$2/${year}\"); // \"on 05/2024\" \n \nlet comma: Regex = Regex::new(\"\\\\s*,\\\\s*\").unwrap(); \nlet parts: Vector = comma.split(\"a , b,c\"); // \"a\", \"b\", \"c\" \n``` \n \nA pattern is a `str`, so a regex backslash is written `\\\\` in an Ignis \nstring literal: `\"\\\\d+\"` is the pattern `\\d+`. \n \n## Matching \n \n- Leftmost-first: of the matches starting at the leftmost position, the \n one the pattern prefers wins. Alternatives are tried in order (`a|ab` \n on `\"ab\"` matches `\"a\"`), greedy repetition prefers more and lazy \n repetition fewer. \n- The search is unanchored unless the pattern uses `^`. \n- Offsets are byte offsets into the UTF-8 text and always fall on scalar \n boundaries. An invalid byte is read as U+FFFD and spans one byte. \n- `findAll` returns non-overlapping matches; after an empty match it \n resumes one scalar further on, so `a*` on `\"baa\"` gives `0..0`, \n `1..3` and `3..3`. \n- Group 0 is the whole match; group `n` is the `n`-th `(`. A group that \n did not take part in the match is `NONE` (`(a)|(b)` on `\"b\"`: group 1 \n `NONE`, group 2 `0..1`), and a group in a repetition holds its last \n iteration (`(a|b)+` on `\"ab\"`: group 1 `1..2`). \n- `replaceAll`, `split` and `capturesAll` use the matches `findAll` \n finds, so `x*` replaced by `-` in `\"abc\"` gives `\"-a-b-c-\"`. \n- Matching takes O(text x program) time: the VM runs every thread in \n lockstep and follows each instruction at most once per position, so \n patterns such as `(a*)*b` cannot backtrack. \n \n## Replacement \n \nIn the replacement text of `replace` and `replaceAll`: \n \n- `$n`, one or more digits read greedily (`$12` is group 12), and `${n}` \n insert group `n`; `${1}0` is group 1 followed by `0`. \n- `${name}` inserts the named group. Braces hold `[A-Za-z0-9_]+`: all \n digits is a number, anything else a name. \n- A group that did not take part or does not exist inserts nothing. \n- `$$` is a literal `$`; any other `$` (`$x`, `${}`, `${a b}`, a final \n `$`) is a literal `$`. A name needs braces: `$name` is the text \n `$name`, not the group. \n \n## Limits \n \n- A count in `{n}`, `{n,}` or `{n,m}` is at most `REGEX_MAX_REPETITION` \n (1000); a larger one is `repetition count above 1000`. \n- Counted repetition copies its body, so a program holds at most \n `REGEX_MAX_PROGRAM` (100000) instructions; a larger one, such as \n `(a{1000}){1000}`, is `pattern too large` at column 1. \n \n## Performance \n \n- Matching is linear: a Pike VM runs every thread in lockstep over the \n text, one scalar per step, and follows each instruction at most once per \n position, so a search takes O(text x program) time and memory \n proportional to the program, whatever the pattern. \n- When every match starts with the same literal scalars (`foo(bar|baz)` \n starts with `foo`), the search skips straight to the next occurrence of \n that prefix whenever no thread is alive, and stops when there is none. \n A pattern that starts with `^`, a class, an alternation, a repetition \n that may be empty or a case-insensitive letter has no prefix. \n- `Regex` holds only the compiled program. Every call (`isMatch`, `find`, \n `findAll`, ...) allocates its own VM state, sized by the program, and \n copies the text once; a call on many matches reuses that state. \n- `isMatch` stops at the first match found and is the cheapest test. \n \n## Exports \n \n| Export | Description | \n|------------------------|----------------------------------------------------------------| \n| `Regex` | `Regex::new(pattern)`: `Result`; `isMatch(text)`, `find(text)`: `Option`, `findAll(text)`: `Vector`, `captures(text)`: `Option`, `capturesAll(text)`: `Vector`, `replace(text, replacement)` and `replaceAll(text, replacement)`: `String`, `split(text)`: `Vector`, `groupCount()`, `groupName(index)`: `Option`, `groupIndex(name)`: `Option` | \n| `Captures` | `group(index)` and `name(name)`: `Option`, `whole()`: `Match`, `groupCount()` (whole match excluded), `length()` (included) | \n| `Match` | `start()`, `end()`, `length()` in bytes, `isEmpty()`, `textIn(haystack)`: a copy of the matched text | \n| `REGEX_MAX_PROGRAM` | The most instructions a compiled pattern may hold: 100000 | \n| `parseRegex` | `parseRegex(pattern)`: `Result` | \n| `RegexError` | `column()` (1-based, in Unicode scalars), `message()`, `toString()` | \n| `RegexAst` | `nodes`, `root`, `groupCount` | \n| `RegexNode` | `EMPTY`, `LITERAL`, `ANY_EXCEPT_NEWLINE`, `CLASS`, `CONCAT`, `ALTERNATE`, `REPEAT`, `GROUP`, `ASSERT` | \n| `RegexClass` | Sorted, merged `ranges` and `negated` | \n| `RegexRange` | Inclusive `start` and `end` scalar values | \n| `RegexRepeat` | `child`, `min`, `max`, `bounded`, `greedy` | \n| `RegexGroup` | `child`, `capturing`, `index`, `name` | \n| `RegexAssertion` | `START`, `END`, `WORD_BOUNDARY`, `NOT_WORD_BOUNDARY` | \n| `REGEX_MAX_REPETITION` | The largest count `{n,m}` accepts: 1000 | \n \n## Syntax \n \n| Syntax | Meaning | \n|-----------------------------------|----------------------------------------------------------| \n| `x` | The scalar `x`, any UTF-8 scalar | \n| `.` | Any scalar except `\\n` | \n| `\\n` `\\t` `\\r` `\\f` `\\v` | Control scalars | \n| `\\.` `\\*` `\\\\` ... | Any escaped ASCII punctuation is itself | \n| `[abc]` `[a-z]` `[^...]` | Classes, ranges, negation; escapes work inside | \n| `\\d` `\\w` `\\s` and `\\D` `\\W` `\\S` | `[0-9]`, `[0-9A-Za-z_]`, `[ \\t\\n\\r\\f\\v]` and negations (ASCII) | \n| `^` `$` | Start and end of the whole text, not of a line | \n| `\\b` `\\B` | ASCII word boundary and its negation | \n| `(x)` `(?:x)` `(?x)` | Capturing, non-capturing and named groups | \n| `x\\|y` | Alternation, leftmost-first | \n| `x*` `x+` `x?` | Zero or more, one or more, zero or one | \n| `x{n}` `x{n,}` `x{n,m}` | Counted repetition, counts at most 1000 | \n| `x*?` `x+?` `x??` `x{n,m}?` | Lazy forms: prefer fewer repetitions | \n| `(?i)` `(?i:x)` | ASCII case-insensitive for the rest of the enclosing group, or inside `x` | \n \nGroup names are `[A-Za-z_][A-Za-z0-9_]*` and must be unique. Capturing \ngroups are numbered from 1 in the order of their `(`. \n \nThe parser is strict where a pattern would otherwise be ambiguous: \n \n- `{` always starts a counted repetition; a literal brace is `\\{`. A \n `{` that is not `{n}`, `{n,}` or `{n,m}` is `invalid repetition`. A \n lone `}` or `]` is a literal. \n- A quantifier after a quantifier (`a**`, `a+*`, `a{2}{3}`) and a \n quantifier after `^`, `$`, `\\b` or `\\B` are `nothing to repeat`; the \n `?` of a lazy form is part of its quantifier. \n- `[]` and `[^]` are `empty class`; a `]` inside a class is `\\]`. A `-` \n first or last in a class is a literal. \n \n## Errors \n \nEvery error carries the 1-based column, counted in Unicode scalars, of \nthe part that is wrong: `unclosed group` (at its `(`), `unmatched )`, \n`nothing to repeat`, `invalid range z-a` (at its `-`), `unclosed class` \n(at its `[`), `empty class`, `trailing backslash`, `unknown escape \\q`, \n`invalid repetition`, `invalid repetition range`, `repetition count above \n1000`, `invalid group name`, `duplicate group name x`, `unknown group \nflag`, `unsupported: backreference` and `unsupported: lookaround`. \n \n## Unsupported \n \n- Backreferences (`\\1`) and lookaround (`(?=`, `(?!`, `(?<=`, `(?", "doc": "Compiles `ast` to `SAVE 0, , SAVE 1, MATCH`, or fails with \n`pattern too large` when the program would exceed `REGEX_MAX_PROGRAM`." }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::regexAppendPrefix", + "name": "regexAppendPrefix", + "kind": "function", + "visibility": "private", + "signature": "function regexAppendPrefix(ast: &RegexAst, id: u64, prefix: &mut String): boolean", + "doc": "Appends to `prefix` the literal scalars every match of node `id` starts \nwith, and reports whether the node is exactly those scalars, so that a \nconcatenation may continue past it." + }, { "module": "std::regex::compiler", "path": "std::regex::compiler::regexAssertionCode", @@ -16702,6 +16717,15 @@ "visibility": "private", "signature": "function regexEmitRanges(compiler: &mut RegexCompiler, ranges: &Vector, negated: boolean): void" }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::regexLiteralPrefix", + "name": "regexLiteralPrefix", + "kind": "function", + "visibility": "private", + "signature": "function regexLiteralPrefix(ast: &RegexAst): String", + "doc": "The literal prefix of every match of `ast`. Under `(?i)` letters are \nclasses, so a case-insensitive pattern has no letter in its prefix." + }, { "module": "std::regex::compiler", "path": "std::regex::compiler::regexPatchExit", @@ -17100,6 +17124,58 @@ "signature": "function regexShorthand(letter: u32, ranges: &mut Vector): boolean", "doc": "The ranges of `\\d`, `\\w`, `\\s` or their negations, or `false` when \n`letter` is none of them." }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::Captures", + "name": "Captures", + "kind": "record", + "visibility": "public", + "signature": "record Captures", + "doc": "The groups of one match: `slotCount` byte offsets, a start and an end per \ngroup, group 0 being the whole match. It stores no view of the text.", + "members": [ + { + "name": "slots", + "kind": "field", + "signature": "slots: Vector" + }, + { + "name": "names", + "kind": "field", + "signature": "names: Vector", + "doc": "The name of each group, empty when it has none; entry 0 is the whole \nmatch." + }, + { + "name": "group", + "kind": "method", + "signature": "group(&self, index: u64): Option", + "doc": "Group `index`: 0 is the whole match. `NONE` when the group did not take \npart in the match or does not exist. In a repetition the group holds \nits last iteration." + }, + { + "name": "groupCount", + "kind": "method", + "signature": "groupCount(&self): u64", + "doc": "The number of capturing groups, the whole match not included." + }, + { + "name": "length", + "kind": "method", + "signature": "length(&self): u64", + "doc": "The number of groups, the whole match included: `groupCount() + 1`." + }, + { + "name": "name", + "kind": "method", + "signature": "name(&self, name: str): Option", + "doc": "The group named `name`; `NONE` when no group has that name or it did not \ntake part in the match." + }, + { + "name": "whole", + "kind": "method", + "signature": "whole(&self): Match", + "doc": "The whole match, group 0." + } + ] + }, { "module": "std::regex::regex", "path": "std::regex::regex::Match", @@ -17157,6 +17233,14 @@ } ] }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::REGEX_GROUP_NUMBER_CAP", + "name": "REGEX_GROUP_NUMBER_CAP", + "kind": "constant", + "visibility": "private", + "signature": "const REGEX_GROUP_NUMBER_CAP: u64" + }, { "module": "std::regex::regex", "path": "std::regex::regex::Regex", @@ -17171,12 +17255,30 @@ "kind": "field", "signature": "program: RegexProgram" }, + { + "name": "names", + "kind": "field", + "signature": "names: Vector", + "doc": "The name of each group, empty when it has none; entry 0 is the whole \nmatch." + }, { "name": "new", "kind": "method", "signature": "static new(pattern: str): Result", "doc": "Parses and compiles `pattern`." }, + { + "name": "captures", + "kind": "method", + "signature": "captures(&self, text: str): Option", + "doc": "The leftmost-first match in `text` with every group." + }, + { + "name": "capturesAll", + "kind": "method", + "signature": "capturesAll(&self, text: str): Vector", + "doc": "Every non-overlapping match in `text` with its groups, as `findAll` \nfinds them." + }, { "name": "find", "kind": "method", @@ -17189,21 +17291,158 @@ "signature": "findAll(&self, text: str): Vector", "doc": "Every non-overlapping match in `text`, left to right. After an empty \nmatch the search resumes one scalar further on." }, + { + "name": "groupCount", + "kind": "method", + "signature": "groupCount(&self): u64", + "doc": "The number of capturing groups, the whole match not included." + }, + { + "name": "groupIndex", + "kind": "method", + "signature": "groupIndex(&self, name: str): Option", + "doc": "The index of the group named `name`, or `NONE`." + }, + { + "name": "groupName", + "kind": "method", + "signature": "groupName(&self, index: u64): Option", + "doc": "The name of group `index`; `NONE` when the group has no name or does \nnot exist." + }, { "name": "isMatch", "kind": "method", "signature": "isMatch(&self, text: str): boolean", "doc": "Whether `text` contains a match." + }, + { + "name": "replace", + "kind": "method", + "signature": "replace(&self, text: str, replacement: str): String", + "doc": "`text` with its first match replaced by `replacement`, expanded as in \n`replaceAll`." + }, + { + "name": "replaceAll", + "kind": "method", + "signature": "replaceAll(&self, text: str, replacement: str): String", + "doc": "`text` with every non-overlapping match, as `findAll` finds them, \nreplaced by `replacement` expanded for that match. \n \nIn `replacement`, `$n` (one or more digits, read greedily) and `${n}` \ninsert group `n`, `${name}` the named group, and `$$` a `$`. Braces \nhold `[A-Za-z0-9_]+`: all digits is a number, anything else a name. A \ngroup that did not take part or does not exist inserts nothing. Any \nother `$`, such as `$x`, `${}` or `${a b}`, is a literal `$`." + }, + { + "name": "split", + "kind": "method", + "signature": "split(&self, text: str): Vector", + "doc": "The pieces of `text` between the matches `findAll` finds. A match at \nthe start or end of `text` leaves an empty first or last piece, an \nempty match splits between scalars, and without a match the result is \nthe whole text." } ] }, { "module": "std::regex::regex", - "path": "std::regex::regex::regexFindFrom", - "name": "regexFindFrom", + "path": "std::regex::regex::regexCopyNames", + "name": "regexCopyNames", + "kind": "function", + "visibility": "private", + "signature": "function regexCopyNames(names: &Vector): Vector" + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::regexCopySlots", + "name": "regexCopySlots", + "kind": "function", + "visibility": "private", + "signature": "function regexCopySlots(slots: &Vector): Vector" + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::regexExpand", + "name": "regexExpand", + "kind": "function", + "visibility": "private", + "signature": "function regexExpand(out: &mut String, source: &String, slots: &Vector, names: &Vector, replacement: &String): void", + "doc": "Appends `replacement` to `out` with its `$` references expanded against \none match; see `Regex::replaceAll`." + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::regexGroupIndex", + "name": "regexGroupIndex", + "kind": "function", + "visibility": "private", + "signature": "function regexGroupIndex(names: &Vector, name: str): Option" + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::regexGroupNames", + "name": "regexGroupNames", + "kind": "function", + "visibility": "private", + "signature": "function regexGroupNames(ast: &RegexAst): Vector", + "doc": "The name of every group of `ast`, indexed by group, entry 0 empty." + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::regexIsAsciiDigit", + "name": "regexIsAsciiDigit", + "kind": "function", + "visibility": "private", + "signature": "function regexIsAsciiDigit(byte: u8): boolean" + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::regexIsNameByte", + "name": "regexIsNameByte", "kind": "function", "visibility": "private", - "signature": "function regexFindFrom(vm: &mut RegexVm, program: &RegexProgram, text: &String, startAt: u64): Option" + "signature": "function regexIsNameByte(byte: u8): boolean" + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::regexLiteralPrefix", + "name": "regexLiteralPrefix", + "kind": "function", + "visibility": "public", + "signature": "function regexLiteralPrefix(regex: &Regex): String", + "doc": "The literal prefix the search of `regex` skips to, empty when there is \nnone. Not exported by `std::regex`: tests use it." + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::regexPushGroup", + "name": "regexPushGroup", + "kind": "function", + "visibility": "private", + "signature": "function regexPushGroup(out: &mut String, source: &String, slots: &Vector, index: u64): void" + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::regexPushReference", + "name": "regexPushReference", + "kind": "function", + "visibility": "private", + "signature": "function regexPushReference(out: &mut String, source: &String, slots: &Vector, names: &Vector, reference: &String): void", + "doc": "Appends the group `reference` names: a number when it is all digits, \notherwise a group name." + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::regexReplace", + "name": "regexReplace", + "kind": "function", + "visibility": "private", + "signature": "function regexReplace(regex: &Regex, text: str, replacement: str, every: boolean): String" + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::regexSearchAll", + "name": "regexSearchAll", + "kind": "function", + "visibility": "private", + "signature": "function regexSearchAll(program: &RegexProgram, source: &String, startAt: u64): Vector>", + "doc": "The slots of every non-overlapping match from `startAt` on. After an \nempty match the search resumes one scalar further on." + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::regexWholeMatch", + "name": "regexWholeMatch", + "kind": "function", + "visibility": "private", + "signature": "function regexWholeMatch(slots: &Vector): Match" }, { "module": "std::regex::vm", @@ -17351,6 +17590,15 @@ "visibility": "private", "signature": "function regexClearThreadList(list: &mut RegexThreadList): void" }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::regexFindBytes", + "name": "regexFindBytes", + "kind": "function", + "visibility": "private", + "signature": "function regexFindBytes(text: &String, needle: &String, startAt: u64): Option", + "doc": "The first byte offset at or after `startAt` where `needle` occurs in `text`." + }, { "module": "std::regex::vm", "path": "std::regex::vm::regexIsWordByte",