From b4919ce998d536680219961c5a54aeb53d64fe5d Mon Sep 17 00:00:00 2001 From: Ignacio Perez Date: Wed, 28 Oct 2026 15:59:18 -0300 Subject: [PATCH] feat(std): match regular expressions with a Pike VM Regex::new parses and compiles a pattern to a program (RANGES, ANY, SPLIT with priority, JUMP, SAVE, ASSERT, MATCH; at most 100000 instructions, else 'pattern too large'), and a Pike VM runs every thread in lockstep over the UTF-8 text, so matching takes time linear in the text for a given pattern and never backtracks. isMatch, find and findAll give leftmost-first, non-overlapping matches as byte offsets (Match keeps offsets only and copies with textIn). Capture slots for every group are already saved, for the captures slice. --- std/regex/compiler.ign | 308 +++++++++ std/regex/mod.ign | 41 +- std/regex/regex.ign | 129 ++++ std/regex/tests.ign | 203 +++++- std/regex/vm.ign | 372 +++++++++++ .../__parse_verdicts__/std_regex_compiler.txt | 2 + .../__parse_verdicts__/std_regex_regex.txt | 2 + .../__parse_verdicts__/std_regex_vm.txt | 2 + .../__doc_baselines__/output__std_io_mod.json | 592 +++++++++++++++++- 9 files changed, 1646 insertions(+), 5 deletions(-) create mode 100644 std/regex/compiler.ign create mode 100644 std/regex/regex.ign create mode 100644 std/regex/vm.ign create mode 100644 test_cases/__parse_verdicts__/std_regex_compiler.txt create mode 100644 test_cases/__parse_verdicts__/std_regex_regex.txt create mode 100644 test_cases/__parse_verdicts__/std_regex_vm.txt diff --git a/std/regex/compiler.ign b/std/regex/compiler.ign new file mode 100644 index 000000000..6c62cf2bd --- /dev/null +++ b/std/regex/compiler.ign @@ -0,0 +1,308 @@ +//! Compiler: `RegexAst` -> `RegexProgram`, the instruction list the Pike VM +//! runs. + +import RegexAssertion, RegexAst, RegexError, RegexNode, RegexRange from "./ast"; + +/// The most instructions a compiled pattern may hold. Counted repetition +/// copies its body, so nesting (`(a{1000}){1000}`) could otherwise produce +/// a program too large to run; such a pattern is `pattern too large`. +export const REGEX_MAX_PROGRAM: u64 = 100000; + +/// What an instruction does. Operands live in `RegexInst.first` and +/// `RegexInst.second`. +export enum RegexOpcode { + /// Consume one scalar inside the ranges `first .. first + second` of + /// `RegexProgram.ranges`, or outside them when `negated`. + RANGES, + + /// Consume one scalar other than `\n`. + ANY_EXCEPT_NEWLINE, + + /// Continue at `first`, and with lower priority at `second`. + SPLIT, + + /// Continue at `first`. + JUMP, + + /// Record the current offset in slot `first`. + SAVE, + + /// Continue only when the assertion coded in `first` holds here. + ASSERT, + + /// The pattern matched. + MATCH, +} + +/// One instruction of a program. +export record RegexInst { + public opcode: RegexOpcode; + public first: u64; + public second: u64; + public negated: boolean; +} + +/// A compiled pattern. Execution starts at instruction 0. +export record RegexProgram { + public insts: Vector; + + /// The ranges `RANGES` instructions refer to. + public ranges: Vector; + + /// Two slots per group, the whole match being group 0. + public slotCount: u64; +} + +/// `ASSERT` operand codes. +export const REGEX_ASSERT_START: u64 = 0; +export const REGEX_ASSERT_END: u64 = 1; +export const REGEX_ASSERT_WORD_BOUNDARY: u64 = 2; +export const REGEX_ASSERT_NOT_WORD_BOUNDARY: u64 = 3; + +record RegexCompiler { + insts: Vector; + ranges: Vector; + tooLarge: boolean; +} + +function regexEmit( + compiler: &mut RegexCompiler, + opcode: RegexOpcode, + first: u64, + second: u64 +): u64 { + let pc: u64 = compiler.insts.length(); + + if (pc >= REGEX_MAX_PROGRAM) { + compiler.tooLarge = true; + return pc; + } + + compiler.insts.push(RegexInst { opcode: opcode, first: first, second: second, negated: false }); + return pc; +} + +function regexEmitRanges( + compiler: &mut RegexCompiler, + ranges: &Vector, + negated: boolean +): void { + let start: u64 = compiler.ranges.length(); + let mut index: u64 = 0; + + while (index < ranges.length()) { + let range: &RegexRange = ranges.get(index).unwrap(); + compiler.ranges.push(RegexRange { start: range.start, end: range.end }); + index += 1; + } + + let pc: u64 = regexEmit(compiler, RegexOpcode::RANGES, start, ranges.length()); + + if (!compiler.tooLarge) { + compiler.insts.getMut(pc).unwrap().negated = negated; + } +} + +function regexPatchFirst(compiler: &mut RegexCompiler, pc: u64, target: u64): void { + if (!compiler.tooLarge) { + compiler.insts.getMut(pc).unwrap().first = target; + } +} + +function regexPatchSecond(compiler: &mut RegexCompiler, pc: u64, target: u64): void { + if (!compiler.tooLarge) { + compiler.insts.getMut(pc).unwrap().second = target; + } +} + +/// A `SPLIT` whose body starts right after it; the other target is patched +/// later. Greedy prefers the body, lazy the exit. +function regexEmitBodySplit(compiler: &mut RegexCompiler, greedy: boolean): u64 { + let pc: u64 = regexEmit(compiler, RegexOpcode::SPLIT, 0, 0); + + if (greedy) { + regexPatchFirst(compiler, pc, pc + 1); + } else { + regexPatchSecond(compiler, pc, pc + 1); + } + + return pc; +} + +function regexPatchExit( + compiler: &mut RegexCompiler, + split: u64, + greedy: boolean, + target: u64 +): void { + if (greedy) { + regexPatchSecond(compiler, split, target); + } else { + regexPatchFirst(compiler, split, target); + } +} + +function regexCompileAlternate( + compiler: &mut RegexCompiler, + ast: &RegexAst, + children: &Vector +): void { + let mut jumps: Vector = Vector::new(); + let mut index: u64 = 0; + + while (index < children.length() && !compiler.tooLarge) { + let child: u64 = *children.get(index).unwrap(); + + if (index + 1 == children.length()) { + regexCompileNode(compiler, ast, child); + } else { + let split: u64 = regexEmit(compiler, RegexOpcode::SPLIT, 0, 0); + regexPatchFirst(compiler, split, split + 1); + regexCompileNode(compiler, ast, child); + jumps.push(regexEmit(compiler, RegexOpcode::JUMP, 0, 0)); + regexPatchSecond(compiler, split, compiler.insts.length()); + } + + index += 1; + } + + let end: u64 = compiler.insts.length(); + let mut jump: u64 = 0; + + while (jump < jumps.length()) { + regexPatchFirst(compiler, *jumps.get(jump).unwrap(), end); + jump += 1; + } +} + +function regexCompileRepeat( + compiler: &mut RegexCompiler, + ast: &RegexAst, + child: u64, + min: u32, + max: u32, + bounded: boolean, + greedy: boolean +): void { + let mut count: u32 = 0; + + while (count < min && !compiler.tooLarge) { + regexCompileNode(compiler, ast, child); + count += 1; + } + + if (!bounded) { + // An empty-matching body cannot loop forever: the VM visits each + // instruction at most once per position. + let split: u64 = regexEmitBodySplit(compiler, greedy); + regexCompileNode(compiler, ast, child); + regexEmit(compiler, RegexOpcode::JUMP, split, 0); + regexPatchExit(compiler, split, greedy, compiler.insts.length()); + return; + } + + let mut splits: Vector = Vector::new(); + + while (count < max && !compiler.tooLarge) { + splits.push(regexEmitBodySplit(compiler, greedy)); + regexCompileNode(compiler, ast, child); + count += 1; + } + + let end: u64 = compiler.insts.length(); + let mut index: u64 = 0; + + while (index < splits.length()) { + regexPatchExit(compiler, *splits.get(index).unwrap(), greedy, end); + index += 1; + } +} + +function regexAssertionCode(assertion: RegexAssertion): u64 { + match (assertion) { + RegexAssertion::START -> { + return REGEX_ASSERT_START; + }, + RegexAssertion::END -> { + return REGEX_ASSERT_END; + }, + RegexAssertion::WORD_BOUNDARY -> { + return REGEX_ASSERT_WORD_BOUNDARY; + }, + RegexAssertion::NOT_WORD_BOUNDARY -> { + return REGEX_ASSERT_NOT_WORD_BOUNDARY; + }, + }; +} + +function regexCompileNode(compiler: &mut RegexCompiler, ast: &RegexAst, id: u64): void { + if (compiler.tooLarge) { + return; + } + + match (ast.nodes.get(id).unwrap()) { + RegexNode::EMPTY -> {}, + RegexNode::LITERAL(scalar) -> { + let mut ranges: Vector = Vector::new(); + ranges.push(RegexRange { start: scalar, end: scalar }); + regexEmitRanges(compiler, &ranges, false); + }, + RegexNode::ANY_EXCEPT_NEWLINE -> { + regexEmit(compiler, RegexOpcode::ANY_EXCEPT_NEWLINE, 0, 0); + }, + RegexNode::CLASS(class) -> { + regexEmitRanges(compiler, &class.ranges, class.negated); + }, + RegexNode::CONCAT(children) -> { + let mut index: u64 = 0; + + while (index < children.length() && !compiler.tooLarge) { + regexCompileNode(compiler, ast, *children.get(index).unwrap()); + index += 1; + } + }, + RegexNode::ALTERNATE(children) -> { + regexCompileAlternate(compiler, ast, &children); + }, + RegexNode::REPEAT(repeat) -> { + regexCompileRepeat(compiler, ast, repeat.child, repeat.min, repeat.max, repeat.bounded, repeat.greedy); + }, + RegexNode::GROUP(group) -> { + if (group.capturing) { + regexEmit(compiler, RegexOpcode::SAVE, group.index * 2, 0); + regexCompileNode(compiler, ast, group.child); + regexEmit(compiler, RegexOpcode::SAVE, group.index * 2 + 1, 0); + } else { + regexCompileNode(compiler, ast, group.child); + } + }, + RegexNode::ASSERT(assertion) -> { + regexEmit(compiler, RegexOpcode::ASSERT, regexAssertionCode(assertion), 0); + }, + }; +} + +/// Compiles `ast` to `SAVE 0, , SAVE 1, MATCH`, or fails with +/// `pattern too large` when the program would exceed `REGEX_MAX_PROGRAM`. +export function compileRegex(ast: &RegexAst): Result { + let mut compiler: RegexCompiler = RegexCompiler { + insts: Vector::new(), + ranges: Vector::new(), + tooLarge: false, + }; + + regexEmit(&mut compiler, RegexOpcode::SAVE, 0, 0); + regexCompileNode(&mut compiler, ast, ast.root); + regexEmit(&mut compiler, RegexOpcode::SAVE, 1, 0); + regexEmit(&mut compiler, RegexOpcode::MATCH, 0, 0); + + if (compiler.tooLarge) { + return Result::ERROR(RegexError::create(1, String::create("pattern too large"))); + } + + return Result::OK(RegexProgram { + insts: compiler.insts, + ranges: compiler.ranges, + slotCount: (ast.groupCount + 1) * 2, + }); +} diff --git a/std/regex/mod.ign b/std/regex/mod.ign index 85d098ae4..9031a4a58 100644 --- a/std/regex/mod.ign +++ b/std/regex/mod.ign @@ -6,14 +6,44 @@ //! that simulates the NFA, so matching takes O(text x program) time and never //! backtracks. //! -//! This slice holds the parser only. Compiling and matching (`Regex::new`, -//! `isMatch`, `find`, `findAll`, captures, `replaceAll`, `split`) arrive in -//! the next slices. +//! ```ignis +//! import Match, Regex from "std::regex"; +//! +//! let regex: Regex = Regex::new("\\d+").unwrap(); +//! let found: Match = regex.find("abc 123").unwrap(); // 4..7 +//! ``` +//! +//! Captures, `replaceAll` and `split` arrive in the next slices. +//! +//! ## Matching +//! +//! - Leftmost-first: of the matches starting at the leftmost position, the +//! one the pattern prefers wins. Alternatives are tried in order (`a|ab` +//! on `"ab"` matches `"a"`), greedy repetition prefers more and lazy +//! repetition fewer. +//! - The search is unanchored unless the pattern uses `^`. +//! - Offsets are byte offsets into the UTF-8 text and always fall on scalar +//! boundaries. An invalid byte is read as U+FFFD and spans one byte. +//! - `findAll` returns non-overlapping matches; after an empty match it +//! resumes one scalar further on, so `a*` on `"baa"` gives `0..0`, +//! `1..3` and `3..3`. +//! - Matching takes O(text x program) time: the VM runs every thread in +//! lockstep and follows each instruction at most once per position, so +//! patterns such as `(a*)*b` cannot backtrack. +//! +//! ## Limits +//! +//! Counted repetition copies its body, so a program holds at most +//! `REGEX_MAX_PROGRAM` (100000) instructions; a larger one is +//! `pattern too large` at column 1. //! //! ## Exports //! //! | Export | Description | //! |------------------------|----------------------------------------------------------------| +//! | `Regex` | `Regex::new(pattern)`: `Result`; `isMatch(text)`, `find(text)`: `Option`, `findAll(text)`: `Vector` | +//! | `Match` | `start()`, `end()`, `length()` in bytes, `isEmpty()`, `textIn(haystack)`: a copy of the matched text | +//! | `REGEX_MAX_PROGRAM` | The most instructions a compiled pattern may hold: 100000 | //! | `parseRegex` | `parseRegex(pattern)`: `Result` | //! | `RegexError` | `column()` (1-based, in Unicode scalars), `message()`, `toString()` | //! | `RegexAst` | `nodes`, `root`, `groupCount` | @@ -74,6 +104,9 @@ import _ from "std::regex::ast"; import _ from "std::regex::parser"; +import _ from "std::regex::compiler"; +import _ from "std::regex::vm"; +import _ from "std::regex::regex"; export RegexAssertion, @@ -86,3 +119,5 @@ export RegexRepeat, from "./ast"; export REGEX_MAX_REPETITION, parseRegex from "./parser"; +export REGEX_MAX_PROGRAM from "./compiler"; +export Match, Regex from "./regex"; diff --git a/std/regex/regex.ign b/std/regex/regex.ign new file mode 100644 index 000000000..04cde567f --- /dev/null +++ b/std/regex/regex.ign @@ -0,0 +1,129 @@ +//! `Regex`, the compiled pattern, and `Match`, the offsets of one match. + +import RegexError from "./ast"; +import compileRegex, RegexProgram from "./compiler"; +import parseRegex from "./parser"; +import regexNewVm, regexSearch, RegexVm from "./vm"; + +/// One match: byte offsets into the searched text, always on scalar +/// boundaries. It stores no view of the text. +export record Match { + matchStart: u64; + matchEnd: u64; + + /// A match of the bytes `start .. end`. + public static create(start: u64, end: u64): Match { + return Match { matchStart: start, matchEnd: end }; + } + + /// The byte offset where the match starts. + public start(&self): u64 { + return self.matchStart; + } + + /// The byte offset just past the match. + public end(&self): u64 { + return self.matchEnd; + } + + /// The length of the match in bytes. + public length(&self): u64 { + return self.matchEnd - self.matchStart; + } + + /// Whether the match is empty. + public isEmpty(&self): boolean { + return self.matchEnd == self.matchStart; + } + + /// A copy of the matched bytes of `haystack`, which must be the searched + /// text. + public textIn(&self, haystack: str): String { + let source: String = String::create(haystack); + return source.substringBytes(self.matchStart, self.matchEnd); + } +} + +/// A compiled regular expression. +export record Regex { + program: RegexProgram; + + /// Parses and compiles `pattern`. + public static new(pattern: str): Result { + match (parseRegex(pattern)) { + Result::OK(ast) -> { + match (compileRegex(&ast)) { + Result::OK(program) -> { + return Result::OK(Regex { program: program }); + }, + Result::ERROR(error) -> { + return Result::ERROR(error); + }, + }; + }, + Result::ERROR(error) -> { + return Result::ERROR(error); + }, + }; + } + + /// Whether `text` contains a match. + public isMatch(&self, text: str): boolean { + let source: String = String::create(text); + let mut vm: RegexVm = regexNewVm(&self.program); + return regexSearch(&mut vm, &self.program, &source, 0, true).isSome(); + } + + /// The leftmost-first match in `text`. + public find(&self, text: str): Option { + let source: String = String::create(text); + let mut vm: RegexVm = regexNewVm(&self.program); + return regexFindFrom(&mut vm, &self.program, &source, 0); + } + + /// Every non-overlapping match in `text`, left to right. After an empty + /// match the search resumes one scalar further on. + public findAll(&self, text: str): Vector { + let mut matches: Vector = Vector::new(); + let source: String = String::create(text); + let mut vm: RegexVm = regexNewVm(&self.program); + let length: u64 = source.length(); + let mut startAt: u64 = 0; + + while (startAt <= length) { + let Option::SOME(found) = regexFindFrom(&mut vm, &self.program, &source, startAt) else { + break; + }; + + matches.push(found); + + if (!found.isEmpty()) { + startAt = found.end(); + continue; + } + + if (found.end() >= length) { + break; + } + + let mut next: u64 = found.end(); + source.charAt(found.end(), &mut next); + startAt = next; + } + + return matches; + } +} + +function regexFindFrom( + vm: &mut RegexVm, + program: &RegexProgram, + text: &String, + startAt: u64 +): Option { + let Option::SOME(slots) = regexSearch(vm, program, text, startAt, false) else { + return Option::NONE; + }; + + return Option::SOME(Match::create(*slots.get(0).unwrap(), *slots.get(1).unwrap())); +} diff --git a/std/regex/tests.ign b/std/regex/tests.ign index 24f0b851d..474e3a8fc 100644 --- a/std/regex/tests.ign +++ b/std/regex/tests.ign @@ -1,5 +1,5 @@ import Test from "std::test"; -import parseRegex from "std::regex"; +import Match, parseRegex, Regex from "std::regex"; import regexAstDebug from "./ast"; function regexTestShape(pattern: str, expected: str): void { @@ -217,3 +217,204 @@ function rejectsBadGroupSyntax(): void { regexTestError("(?x)(?y)", "duplicate group name a", 11); } + +function regexTestCompile(pattern: str): Regex { + match (Regex::new(pattern)) { + Result::OK(regex) -> { + return regex; + }, + Result::ERROR(error) -> { + let text: String = error.toString(); + Test::assertEq(text.toStr(), "no error"); + return Regex::new("").unwrap(); + }, + }; +} + +function regexTestPushMatch(text: &mut String, found: &Match): void { + let startOffset: u64 = found.start(); + let endOffset: u64 = found.end(); + let start: String = startOffset.toString(); + let end: String = endOffset.toString(); + text.push(&start); + text.pushStr(".."); + text.push(&end); +} + +/// The first match of `pattern` in `text` as `start..end`, or `none`. +function regexTestFind(pattern: str, text: str, expected: str): void { + let regex: Regex = regexTestCompile(pattern); + let mut shown: String = String::new(); + + match (regex.find(text)) { + Option::SOME(found) -> { + regexTestPushMatch(&mut shown, &found); + }, + Option::NONE -> { + shown.pushStr("none"); + }, + }; + + Test::assertEq(shown.toStr(), expected); + Test::assertEq(regex.isMatch(text), !shown.equals("none")); +} + +/// Every match of `pattern` in `text` as `start..end`, comma-separated. +function regexTestFindAll(pattern: str, text: str, expected: str): void { + let regex: Regex = regexTestCompile(pattern); + let matches: Vector = regex.findAll(text); + let mut shown: String = String::new(); + let mut index: u64 = 0; + + while (index < matches.length()) { + if (index > 0) { + shown.pushStr(","); + } + + regexTestPushMatch(&mut shown, matches.get(index).unwrap()); + index += 1; + } + + Test::assertEq(shown.toStr(), expected); +} + +@test +function matchesLiterals(): void { + regexTestFind("abc", "xxabcabc", "2..5"); + regexTestFind("abc", "ab", "none"); + regexTestFind("", "abc", "0..0"); + regexTestFind("b", "", "none"); +} + +@test +function matchesClasses(): void { + regexTestFind("[a-c]+", "xxbcaz", "2..5"); + regexTestFind("[^a-c]+", "abxyc", "2..4"); + regexTestFind("\\d+", "ab123c", "2..5"); + regexTestFind("\\W", "ab_9 x", "4..5"); + regexTestFind("[^a]", "aaa", "none"); +} + +@test +function matchesAnyExceptNewline(): void { + regexTestFind("a.c", "a\nc abc", "4..7"); + regexTestFind(".", "\n", "none"); + regexTestFind("[^x]", "\n", "0..1"); +} + +@test +function matchesAnchors(): void { + regexTestFind("^ab", "abab", "0..2"); + regexTestFind("^b", "ab", "none"); + regexTestFind("ab$", "abab", "2..4"); + regexTestFind("a$", "a\n", "none"); + regexTestFind("^$", "", "0..0"); + regexTestFindAll("^a", "aaa", "0..1"); +} + +@test +function matchesWordBoundaries(): void { + regexTestFind("\\bcat\\b", "concat cat", "7..10"); + regexTestFind("\\Bcat", "cat concat", "7..10"); + regexTestFindAll("\\b", "ab cd", "0..0,2..2,3..3,5..5"); +} + +@test +function prefersTheFirstAlternative(): void { + regexTestFind("a|ab", "ab", "0..1"); + regexTestFind("ab|a", "ab", "0..2"); + regexTestFind("x|ab|a", "zab", "1..3"); +} + +@test +function matchesGreedyAndLazyRepetition(): void { + regexTestFind("a+", "aaa", "0..3"); + regexTestFind("a+?", "aaa", "0..1"); + regexTestFind("a*?b", "aab", "0..3"); + regexTestFind("a??", "a", "0..0"); + regexTestFind("a?", "a", "0..1"); +} + +@test +function matchesCountedRepetition(): void { + regexTestFind("a{3}", "aaaaa", "0..3"); + regexTestFind("a{2,}", "aaaaa", "0..5"); + regexTestFind("a{2,3}", "aaaaa", "0..3"); + regexTestFind("a{2,3}?", "aaaaa", "0..2"); + regexTestFind("a{3}", "aa", "none"); + regexTestFind("(?:ab){2}", "abaabab", "3..7"); +} + +@test +function matchesUtf8Scalars(): void { + regexTestFind("Γ©+", "ééx", "0..4"); + regexTestFind(".", "πŸ˜€x", "0..4"); + regexTestFind("[Γ -ΓΌ]", "aΓ©", "1..3"); + regexTestFindAll("", "Γ©", "0..0,2..2"); +} + +@test +function matchesCaseInsensitively(): void { + regexTestFind("(?i)abc", "xABc", "1..4"); + regexTestFind("a(?i:b)c", "aBc aBC", "0..3"); + regexTestFind("(?i)[a-c]+", "xBaC", "1..4"); +} + +@test +function findsEveryMatch(): void { + regexTestFindAll("a*", "baa", "0..0,1..3,3..3"); + regexTestFindAll("\\d+", "a1b22c333", "1..2,3..5,6..9"); + regexTestFindAll("x", "abc", ""); + regexTestFindAll("aa", "aaaaa", "0..2,2..4"); +} + +@test +function exposesMatchOffsetsAndText(): void { + let regex: Regex = regexTestCompile("b+"); + let found: Match = regex.find("abbc").unwrap(); + Test::assertEq(found.start(), 1); + Test::assertEq(found.end(), 3); + Test::assertEq(found.length(), 2); + Test::assertEq(found.isEmpty(), false); + let text: String = found.textIn("abbc"); + Test::assertEq(text.toStr(), "bb"); + let empty: Match = regexTestCompile("x*").find("abc").unwrap(); + Test::assertEq(empty.isEmpty(), true); +} + +@test +function rejectsPatternsThatCompileTooLarge(): void { + match (Regex::new("(?:a{1000}){1000}")) { + Result::OK(_) -> { + Test::assertEq("compiled", "pattern too large"); + }, + Result::ERROR(error) -> { + Test::assertEq(error.message(), "pattern too large"); + }, + }; + + match (Regex::new("x{3}(")) { + Result::OK(_) -> { + Test::assertEq("compiled", "unclosed group"); + }, + Result::ERROR(error) -> { + Test::assertEq(error.message(), "unclosed group"); + }, + }; +} + +@test +function matchesNestedStarsInLinearTime(): void { + let mut text: String = String::new(); + let mut count: u64 = 0; + + while (count < 10000) { + text.pushStr("a"); + count += 1; + } + + let nested: Regex = regexTestCompile("(a*)*b"); + Test::assertEq(nested.isMatch(text.toStr()), false); + let alternatives: Regex = regexTestCompile("(a|aa)*c"); + Test::assertEq(alternatives.isMatch(text.toStr()), false); +} diff --git a/std/regex/vm.ign b/std/regex/vm.ign new file mode 100644 index 000000000..5b9c3130f --- /dev/null +++ b/std/regex/vm.ign @@ -0,0 +1,372 @@ +//! Pike VM: runs a `RegexProgram` over UTF-8 text with every thread in +//! lockstep, one scalar per step, so a search takes O(text x program) time. + +import + REGEX_ASSERT_END, + REGEX_ASSERT_NOT_WORD_BOUNDARY, + REGEX_ASSERT_START, + REGEX_ASSERT_WORD_BOUNDARY, + RegexInst, + RegexOpcode, + RegexProgram, +from "./compiler"; +import RegexRange from "./ast"; + +/// A slot no `SAVE` has written. +export const REGEX_NO_SLOT: u64 = 0xFFFFFFFFFFFFFFFF; + +/// The threads of one step, in priority order, with the instructions +/// already visited at this position so that each is followed once. +record RegexThreadList { + sparse: Vector; + dense: Vector; + visitedCount: u64; + pcs: Vector; + + /// `slotCount` slots per entry of `pcs`. + slots: Vector; +} + +/// A pending step of `regexAddThread`: follow `pc`, or restore `slot` to +/// `value` once the branch that saved it is done. +record RegexFrame { + pc: u64; + restore: boolean; + slot: u64; + value: u64; +} + +/// The working state of searches with one program, reused across them. +export record RegexVm { + slotCount: u64; + stack: Vector; + scratch: Vector; + current: RegexThreadList; + next: RegexThreadList; +} + +function regexNewThreadList(size: u64): RegexThreadList { + let mut sparse: Vector = Vector::new(size); + let mut dense: Vector = Vector::new(size); + let mut index: u64 = 0; + + while (index < size) { + sparse.push(0); + dense.push(0); + index += 1; + } + + return RegexThreadList { + sparse: sparse, + dense: dense, + visitedCount: 0, + pcs: Vector::new(), + slots: Vector::new(), + }; +} + +function regexClearThreadList(list: &mut RegexThreadList): void { + list.visitedCount = 0; + list.pcs.clear(); + list.slots.clear(); +} + +/// Marks `pc` visited and reports whether it already was. +function regexVisit(list: &mut RegexThreadList, pc: u64): boolean { + let index: u64 = *list.sparse.get(pc).unwrap(); + + if (index < list.visitedCount && *list.dense.get(index).unwrap() == pc) { + return true; + } + + list.sparse.set(pc, list.visitedCount); + list.dense.set(list.visitedCount, pc); + list.visitedCount += 1; + return false; +} + +function regexIsWordByte(byte: u8): boolean { + return (byte >= '0' as u8 && byte <= '9' as u8) || (byte >= 'A' as u8 && byte <= 'Z' as u8) || (byte >= 'a' as u8 && byte <= 'z' as u8) || byte == '_' as u8; +} + +function regexAssertionHolds(text: &String, code: u64, position: u64): boolean { + let length: u64 = text.length(); + + if (code == REGEX_ASSERT_START) { + return position == 0; + } + + if (code == REGEX_ASSERT_END) { + return position == length; + } + + let before: boolean = position > 0 && regexIsWordByte(text.byteAt(position - 1).unwrap()); + let after: boolean = position < length && regexIsWordByte(text.byteAt(position).unwrap()); + + if (code == REGEX_ASSERT_WORD_BOUNDARY) { + return before != after; + } + + if (code == REGEX_ASSERT_NOT_WORD_BOUNDARY) { + return before == after; + } + + return false; +} + +/// Follows `pc` at `position` through every non-consuming instruction, with +/// the slots in `vm.scratch`, and appends the consuming instructions reached +/// to `list` in priority order. An explicit stack keeps deep programs off the +/// call stack. +function regexAddThread( + vm: &mut RegexVm, + program: &RegexProgram, + text: &String, + list: &mut RegexThreadList, + startPc: u64, + position: u64 +): void { + vm.stack.push(RegexFrame { pc: startPc, restore: false, slot: 0, value: 0 }); + + while (true) { + let Option::SOME(frame) = vm.stack.pop() else { + break; + }; + + if (frame.restore) { + vm.scratch.set(frame.slot, frame.value); + continue; + } + + let mut pc: u64 = frame.pc; + + while (!regexVisit(list, pc)) { + let inst: &RegexInst = program.insts.get(pc).unwrap(); + let first: u64 = inst.first; + let second: u64 = inst.second; + + match (inst.opcode) { + RegexOpcode::JUMP -> { + pc = first; + }, + RegexOpcode::SPLIT -> { + vm.stack.push(RegexFrame { pc: second, restore: false, slot: 0, value: 0 }); + pc = first; + }, + RegexOpcode::SAVE -> { + let previous: u64 = *vm.scratch.get(first).unwrap(); + vm.stack.push(RegexFrame { pc: 0, restore: true, slot: first, value: previous }); + vm.scratch.set(first, position); + pc += 1; + }, + RegexOpcode::ASSERT -> { + if (!regexAssertionHolds(text, first, position)) { + break; + } + + pc += 1; + }, + _ -> { + list.pcs.push(pc); + let mut slot: u64 = 0; + + while (slot < vm.slotCount) { + list.slots.push(*vm.scratch.get(slot).unwrap()); + slot += 1; + } + + break; + }, + }; + } + } +} + +function regexRangesContain(program: &RegexProgram, inst: &RegexInst, scalar: u32): boolean { + let mut index: u64 = inst.first; + let end: u64 = inst.first + inst.second; + let mut found: boolean = false; + + while (index < end) { + let range: &RegexRange = program.ranges.get(index).unwrap(); + + if (scalar < range.start) { + break; + } + + if (scalar <= range.end) { + found = true; + break; + } + + index += 1; + } + + return found != inst.negated; +} + +/// Makes the threads of the next position current, and empties `next` for +/// the step after. The visited set comes along: a new start thread added at +/// that position must not revisit what those threads already followed. The +/// copy costs as much as building `next` did, so the step stays linear. +function regexAdvanceThreadLists(vm: &mut RegexVm): void { + regexClearThreadList(&mut vm.current); + let mut index: u64 = 0; + + while (index < vm.next.visitedCount) { + let pc: u64 = *vm.next.dense.get(index).unwrap(); + vm.current.dense.set(index, pc); + vm.current.sparse.set(pc, index); + index += 1; + } + + vm.current.visitedCount = vm.next.visitedCount; + index = 0; + + while (index < vm.next.pcs.length()) { + vm.current.pcs.push(*vm.next.pcs.get(index).unwrap()); + index += 1; + } + + index = 0; + + while (index < vm.next.slots.length()) { + vm.current.slots.push(*vm.next.slots.get(index).unwrap()); + index += 1; + } + + regexClearThreadList(&mut vm.next); +} + +function regexResetScratch(vm: &mut RegexVm): void { + let mut slot: u64 = 0; + + while (slot < vm.slotCount) { + vm.scratch.set(slot, REGEX_NO_SLOT); + slot += 1; + } +} + +function regexLoadScratch(vm: &mut RegexVm, thread: u64): void { + let mut slot: u64 = 0; + + while (slot < vm.slotCount) { + let value: u64 = *vm.current.slots.get(thread * vm.slotCount + slot).unwrap(); + vm.scratch.set(slot, value); + slot += 1; + } +} + +/// The state for searching with `program`. +export function regexNewVm(program: &RegexProgram): RegexVm { + let mut scratch: Vector = Vector::new(program.slotCount); + let mut slot: u64 = 0; + + while (slot < program.slotCount) { + scratch.push(REGEX_NO_SLOT); + slot += 1; + } + + return RegexVm { + slotCount: program.slotCount, + stack: Vector::new(), + scratch: scratch, + current: regexNewThreadList(program.insts.length()), + next: regexNewThreadList(program.insts.length()), + }; +} + +/// Searches `text` from byte offset `startAt` for the leftmost-first match. +/// Returns its slots (`slotCount` entries, `REGEX_NO_SLOT` where a group did +/// not take part), or `NONE`. With `earliest` it stops at the first match +/// found, whose end may not be the preferred one. +export function regexSearch( + vm: &mut RegexVm, + program: &RegexProgram, + text: &String, + startAt: u64, + earliest: boolean +): Option> { + regexClearThreadList(&mut vm.current); + regexClearThreadList(&mut vm.next); + let mut best: Option> = Option::NONE; + let length: u64 = text.length(); + let mut position: u64 = startAt; + + while (true) { + // A new start is the lowest-priority thread, and stops once a match is + // known: anything starting later is not leftmost. + if (best.isNone()) { + regexResetScratch(vm); + regexAddThread(vm, program, text, &mut vm.current, 0, position); + } + + if (vm.current.pcs.length() == 0 && best.isSome()) { + break; + } + + let mut scalar: u32 = 0; + let mut nextPosition: u64 = position; + + if (position < length) { + scalar = text.charAt(position, &mut nextPosition).unwrap() as u32; + } + + let mut thread: u64 = 0; + + while (thread < vm.current.pcs.length()) { + let pc: u64 = *vm.current.pcs.get(thread).unwrap(); + let inst: &RegexInst = program.insts.get(pc).unwrap(); + let mut consumes: boolean = false; + let mut matched: boolean = false; + + match (inst.opcode) { + RegexOpcode::MATCH -> { + matched = true; + }, + RegexOpcode::ANY_EXCEPT_NEWLINE -> { + consumes = position < length && scalar != '\n' as u32; + }, + RegexOpcode::RANGES -> { + consumes = position < length && regexRangesContain(program, inst, scalar); + }, + _ -> {}, + }; + + if (matched) { + let mut slots: Vector = Vector::new(vm.slotCount); + let mut index: u64 = 0; + + while (index < vm.slotCount) { + slots.push(*vm.current.slots.get(thread * vm.slotCount + index).unwrap()); + index += 1; + } + + best = Option::SOME(slots); + + if (earliest) { + return best; + } + + // Lower-priority threads lose to this match. + break; + } + + if (consumes) { + regexLoadScratch(vm, thread); + regexAddThread(vm, program, text, &mut vm.next, pc + 1, nextPosition); + } + + thread += 1; + } + + if (position >= length) { + break; + } + + regexAdvanceThreadLists(vm); + position = nextPosition; + } + + return best; +} diff --git a/test_cases/__parse_verdicts__/std_regex_compiler.txt b/test_cases/__parse_verdicts__/std_regex_compiler.txt new file mode 100644 index 000000000..3d8e43b24 --- /dev/null +++ b/test_cases/__parse_verdicts__/std_regex_compiler.txt @@ -0,0 +1,2 @@ +parse-verdict v1 +accepted diff --git a/test_cases/__parse_verdicts__/std_regex_regex.txt b/test_cases/__parse_verdicts__/std_regex_regex.txt new file mode 100644 index 000000000..3d8e43b24 --- /dev/null +++ b/test_cases/__parse_verdicts__/std_regex_regex.txt @@ -0,0 +1,2 @@ +parse-verdict v1 +accepted diff --git a/test_cases/__parse_verdicts__/std_regex_vm.txt b/test_cases/__parse_verdicts__/std_regex_vm.txt new file mode 100644 index 000000000..3d8e43b24 --- /dev/null +++ b/test_cases/__parse_verdicts__/std_regex_vm.txt @@ -0,0 +1,2 @@ +parse-verdict v1 +accepted diff --git a/test_cases/doc/__doc_baselines__/output__std_io_mod.json b/test_cases/doc/__doc_baselines__/output__std_io_mod.json index 1fe99de6b..64f5f5109 100644 --- a/test_cases/doc/__doc_baselines__/output__std_io_mod.json +++ b/test_cases/doc/__doc_baselines__/output__std_io_mod.json @@ -286,16 +286,28 @@ }, { "name": "std::regex", - "doc": " \n# Regex \n \nRegular expressions with linear-time matching, in the style of RE2: a \npattern is parsed into an AST, compiled to a program and run by a Pike VM \nthat simulates the NFA, so matching takes O(text x program) time and never \nbacktracks. \n \nThis slice holds the parser only. Compiling and matching (`Regex::new`, \n`isMatch`, `find`, `findAll`, captures, `replaceAll`, `split`) arrive in \nthe next slices. \n \n## Exports \n \n| Export | Description | \n|------------------------|----------------------------------------------------------------| \n| `parseRegex` | `parseRegex(pattern)`: `Result` | \n| `RegexError` | `column()` (1-based, in Unicode scalars), `message()`, `toString()` | \n| `RegexAst` | `nodes`, `root`, `groupCount` | \n| `RegexNode` | `EMPTY`, `LITERAL`, `ANY_EXCEPT_NEWLINE`, `CLASS`, `CONCAT`, `ALTERNATE`, `REPEAT`, `GROUP`, `ASSERT` | \n| `RegexClass` | Sorted, merged `ranges` and `negated` | \n| `RegexRange` | Inclusive `start` and `end` scalar values | \n| `RegexRepeat` | `child`, `min`, `max`, `bounded`, `greedy` | \n| `RegexGroup` | `child`, `capturing`, `index`, `name` | \n| `RegexAssertion` | `START`, `END`, `WORD_BOUNDARY`, `NOT_WORD_BOUNDARY` | \n| `REGEX_MAX_REPETITION` | The largest count `{n,m}` accepts: 1000 | \n \n## Syntax \n \n| Syntax | Meaning | \n|-----------------------------------|----------------------------------------------------------| \n| `x` | The scalar `x`, any UTF-8 scalar | \n| `.` | Any scalar except `\\n` | \n| `\\n` `\\t` `\\r` `\\f` `\\v` | Control scalars | \n| `\\.` `\\*` `\\\\` ... | Any escaped ASCII punctuation is itself | \n| `[abc]` `[a-z]` `[^...]` | Classes, ranges, negation; escapes work inside | \n| `\\d` `\\w` `\\s` and `\\D` `\\W` `\\S` | `[0-9]`, `[0-9A-Za-z_]`, `[ \\t\\n\\r\\f\\v]` and negations (ASCII) | \n| `^` `$` | Start and end of the whole text, not of a line | \n| `\\b` `\\B` | ASCII word boundary and its negation | \n| `(x)` `(?:x)` `(?x)` | Capturing, non-capturing and named groups | \n| `x\\|y` | Alternation, leftmost-first | \n| `x*` `x+` `x?` | Zero or more, one or more, zero or one | \n| `x{n}` `x{n,}` `x{n,m}` | Counted repetition, counts at most 1000 | \n| `x*?` `x+?` `x??` `x{n,m}?` | Lazy forms: prefer fewer repetitions | \n| `(?i)` `(?i:x)` | ASCII case-insensitive for the rest of the enclosing group, or inside `x` | \n \nGroup names are `[A-Za-z_][A-Za-z0-9_]*` and must be unique. Capturing \ngroups are numbered from 1 in the order of their `(`. \n \nThe parser is strict where a pattern would otherwise be ambiguous: \n \n- `{` always starts a counted repetition; a literal brace is `\\{`. A \n `{` that is not `{n}`, `{n,}` or `{n,m}` is `invalid repetition`. A \n lone `}` or `]` is a literal. \n- A quantifier after a quantifier (`a**`, `a+*`, `a{2}{3}`) and a \n quantifier after `^`, `$`, `\\b` or `\\B` are `nothing to repeat`; the \n `?` of a lazy form is part of its quantifier. \n- `[]` and `[^]` are `empty class`; a `]` inside a class is `\\]`. A `-` \n first or last in a class is a literal. \n \n## Errors \n \nEvery error carries the 1-based column, counted in Unicode scalars, of \nthe part that is wrong: `unclosed group` (at its `(`), `unmatched )`, \n`nothing to repeat`, `invalid range z-a` (at its `-`), `unclosed class` \n(at its `[`), `empty class`, `trailing backslash`, `unknown escape \\q`, \n`invalid repetition`, `invalid repetition range`, `repetition count above \n1000`, `invalid group name`, `duplicate group name x`, `unknown group \nflag`, `unsupported: backreference` and `unsupported: lookaround`. \n \nBackreferences (`\\1`) and lookaround (`(?=`, `(?!`, `(?<=`, `(?`; `isMatch(text)`, `find(text)`: `Option`, `findAll(text)`: `Vector` | \n| `Match` | `start()`, `end()`, `length()` in bytes, `isEmpty()`, `textIn(haystack)`: a copy of the matched text | \n| `REGEX_MAX_PROGRAM` | The most instructions a compiled pattern may hold: 100000 | \n| `parseRegex` | `parseRegex(pattern)`: `Result` | \n| `RegexError` | `column()` (1-based, in Unicode scalars), `message()`, `toString()` | \n| `RegexAst` | `nodes`, `root`, `groupCount` | \n| `RegexNode` | `EMPTY`, `LITERAL`, `ANY_EXCEPT_NEWLINE`, `CLASS`, `CONCAT`, `ALTERNATE`, `REPEAT`, `GROUP`, `ASSERT` | \n| `RegexClass` | Sorted, merged `ranges` and `negated` | \n| `RegexRange` | Inclusive `start` and `end` scalar values | \n| `RegexRepeat` | `child`, `min`, `max`, `bounded`, `greedy` | \n| `RegexGroup` | `child`, `capturing`, `index`, `name` | \n| `RegexAssertion` | `START`, `END`, `WORD_BOUNDARY`, `NOT_WORD_BOUNDARY` | \n| `REGEX_MAX_REPETITION` | The largest count `{n,m}` accepts: 1000 | \n \n## Syntax \n \n| Syntax | Meaning | \n|-----------------------------------|----------------------------------------------------------| \n| `x` | The scalar `x`, any UTF-8 scalar | \n| `.` | Any scalar except `\\n` | \n| `\\n` `\\t` `\\r` `\\f` `\\v` | Control scalars | \n| `\\.` `\\*` `\\\\` ... | Any escaped ASCII punctuation is itself | \n| `[abc]` `[a-z]` `[^...]` | Classes, ranges, negation; escapes work inside | \n| `\\d` `\\w` `\\s` and `\\D` `\\W` `\\S` | `[0-9]`, `[0-9A-Za-z_]`, `[ \\t\\n\\r\\f\\v]` and negations (ASCII) | \n| `^` `$` | Start and end of the whole text, not of a line | \n| `\\b` `\\B` | ASCII word boundary and its negation | \n| `(x)` `(?:x)` `(?x)` | Capturing, non-capturing and named groups | \n| `x\\|y` | Alternation, leftmost-first | \n| `x*` `x+` `x?` | Zero or more, one or more, zero or one | \n| `x{n}` `x{n,}` `x{n,m}` | Counted repetition, counts at most 1000 | \n| `x*?` `x+?` `x??` `x{n,m}?` | Lazy forms: prefer fewer repetitions | \n| `(?i)` `(?i:x)` | ASCII case-insensitive for the rest of the enclosing group, or inside `x` | \n \nGroup names are `[A-Za-z_][A-Za-z0-9_]*` and must be unique. Capturing \ngroups are numbered from 1 in the order of their `(`. \n \nThe parser is strict where a pattern would otherwise be ambiguous: \n \n- `{` always starts a counted repetition; a literal brace is `\\{`. A \n `{` that is not `{n}`, `{n,}` or `{n,m}` is `invalid repetition`. A \n lone `}` or `]` is a literal. \n- A quantifier after a quantifier (`a**`, `a+*`, `a{2}{3}`) and a \n quantifier after `^`, `$`, `\\b` or `\\B` are `nothing to repeat`; the \n `?` of a lazy form is part of its quantifier. \n- `[]` and `[^]` are `empty class`; a `]` inside a class is `\\]`. A `-` \n first or last in a class is a literal. \n \n## Errors \n \nEvery error carries the 1-based column, counted in Unicode scalars, of \nthe part that is wrong: `unclosed group` (at its `(`), `unmatched )`, \n`nothing to repeat`, `invalid range z-a` (at its `-`), `unclosed class` \n(at its `[`), `empty class`, `trailing backslash`, `unknown escape \\q`, \n`invalid repetition`, `invalid repetition range`, `repetition count above \n1000`, `invalid group name`, `duplicate group name x`, `unknown group \nflag`, `unsupported: backreference` and `unsupported: lookaround`. \n \nBackreferences (`\\1`) and lookaround (`(?=`, `(?!`, `(?<=`, `(? `RegexProgram`, the instruction list the Pike VM \nruns." + }, { "name": "std::regex::parser", "doc": "Pattern parser: pattern text -> `RegexAst`, or the first `RegexError`." }, + { + "name": "std::regex::regex", + "doc": "`Regex`, the compiled pattern, and `Match`, the offsets of one match." + }, + { + "name": "std::regex::vm", + "doc": "Pike VM: runs a `RegexProgram` over UTF-8 text with every thread in \nlockstep, one scalar per step, so a search takes O(text x program) time." + }, { "name": "std::result", "doc": " \n# Result Module \n \nSuccess/error sum type for fallible operations. \n \n## Overview \n \n`Result` models operations that can either produce a value (`T`) \nor an error (`E`). It is an enum with two variants: \n \n- `Result::OK(value)` β€” the operation succeeded \n- `Result::ERROR(err)` β€” the operation failed \n \nUse `Result` when a function can fail in a recoverable way, forcing \nthe caller to explicitly handle the error path. \n \n## Variants \n \n```text \n Result \n β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” \n β”‚ β”‚ \n β”‚ OK(T) ── success value of type T β”‚ \n β”‚ β”‚ \n β”‚ ERROR(E) ── error value of type E β”‚ \n β”‚ β”‚ \n β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ \n``` \n \n## Ownership \n \nEvery method that can hand a payload out takes `self` by value and \nconsumes the result: `unwrap`, `unwrapErr`, `unwrapOr`, `unwrapOrElse`, \n`inspect`, `inspectErr`, `map`, `mapErr`, `andThen`, and `orElse`. The \nreceiver cannot be used again afterwards, so an owning payload is moved \nexactly once. \n \nOnly `isOk` and `isError` observe through `&self` and leave the result \nusable. \n \nA borrowed `Result` therefore cannot be fed to a consuming \ncombinator directly. Match on the borrow instead, or clone the result when \nthe value is genuinely needed twice. \n \n## Higher-Order Methods \n \nResult provides callback-based methods for transforming, inspecting, \nand chaining fallible operations: \n \n| Method | Signature | Description | \n|------------------|----------------------------------------|--------------------------------------| \n| `unwrapOrElse` | `(E) -> T` β†’ `T` | Recover from error with callback | \n| `inspect` | `(&T) -> void` β†’ `Result` | Side-effect on OK value | \n| `inspectErr` | `(&E) -> void` β†’ `Result` | Side-effect on ERROR value | \n| `map` | `(T) -> U` β†’ `Result` | Transform OK value | \n| `mapErr` | `(E) -> F` β†’ `Result` | Transform ERROR value | \n| `andThen` | `(T) -> Result` β†’ `Result` | Chain fallible OK operations | \n| `orElse` | `(E) -> Result` β†’ `Result` | Chain fallible ERROR recovery | \n \n`inspect` and `inspectErr` observe the payload through a reference because \nthey hand the original result back. The other callbacks receive the payload \nby value because it is given away. \n \nAll callbacks are `@noescape`: the closure must not outlive the method call. \n \n## Example \n \n```ignis \nimport Io from \"std::io\"; \n \nfunction parseInt(s: str): Result { \n // ... parsing logic ... \n return Result::OK(42); \n} \n \nfunction main(): i32 { \n let result: Result = parseInt(\"42\"); \n \n // Chain: parse, double, inspect \n let doubled: Result = result \n .inspect((val: &i32): void -> { Io::printI32(*val); }) \n .map((val: i32): i32 -> { return val * 2; }); \n \n // Recover from error \n let value: i32 = doubled.unwrapOrElse((err: str): i32 -> { return 0; }); \n \n return 0; \n} \n```" @@ -16454,6 +16466,266 @@ "visibility": "private", "signature": "function regexHexDigit(value: u32): u8" }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::REGEX_ASSERT_END", + "name": "REGEX_ASSERT_END", + "kind": "constant", + "visibility": "public", + "signature": "const REGEX_ASSERT_END: u64" + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::REGEX_ASSERT_NOT_WORD_BOUNDARY", + "name": "REGEX_ASSERT_NOT_WORD_BOUNDARY", + "kind": "constant", + "visibility": "public", + "signature": "const REGEX_ASSERT_NOT_WORD_BOUNDARY: u64" + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::REGEX_ASSERT_START", + "name": "REGEX_ASSERT_START", + "kind": "constant", + "visibility": "public", + "signature": "const REGEX_ASSERT_START: u64" + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::REGEX_ASSERT_WORD_BOUNDARY", + "name": "REGEX_ASSERT_WORD_BOUNDARY", + "kind": "constant", + "visibility": "public", + "signature": "const REGEX_ASSERT_WORD_BOUNDARY: u64" + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::REGEX_MAX_PROGRAM", + "name": "REGEX_MAX_PROGRAM", + "kind": "constant", + "visibility": "public", + "signature": "const REGEX_MAX_PROGRAM: u64" + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::RegexCompiler", + "name": "RegexCompiler", + "kind": "record", + "visibility": "private", + "signature": "record RegexCompiler", + "members": [ + { + "name": "insts", + "kind": "field", + "signature": "insts: Vector" + }, + { + "name": "ranges", + "kind": "field", + "signature": "ranges: Vector" + }, + { + "name": "tooLarge", + "kind": "field", + "signature": "tooLarge: boolean" + } + ] + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::RegexInst", + "name": "RegexInst", + "kind": "record", + "visibility": "public", + "signature": "record RegexInst", + "doc": "One instruction of a program.", + "members": [ + { + "name": "opcode", + "kind": "field", + "signature": "opcode: RegexOpcode" + }, + { + "name": "first", + "kind": "field", + "signature": "first: u64" + }, + { + "name": "second", + "kind": "field", + "signature": "second: u64" + }, + { + "name": "negated", + "kind": "field", + "signature": "negated: boolean" + } + ] + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::RegexOpcode", + "name": "RegexOpcode", + "kind": "enum", + "visibility": "public", + "signature": "enum RegexOpcode", + "doc": "What an instruction does. Operands live in `RegexInst.first` and \n`RegexInst.second`.", + "members": [ + { + "name": "RANGES", + "kind": "variant", + "signature": "RANGES" + }, + { + "name": "ANY_EXCEPT_NEWLINE", + "kind": "variant", + "signature": "ANY_EXCEPT_NEWLINE" + }, + { + "name": "SPLIT", + "kind": "variant", + "signature": "SPLIT" + }, + { + "name": "JUMP", + "kind": "variant", + "signature": "JUMP" + }, + { + "name": "SAVE", + "kind": "variant", + "signature": "SAVE" + }, + { + "name": "ASSERT", + "kind": "variant", + "signature": "ASSERT" + }, + { + "name": "MATCH", + "kind": "variant", + "signature": "MATCH" + } + ] + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::RegexProgram", + "name": "RegexProgram", + "kind": "record", + "visibility": "public", + "signature": "record RegexProgram", + "doc": "A compiled pattern. Execution starts at instruction 0.", + "members": [ + { + "name": "insts", + "kind": "field", + "signature": "insts: Vector" + }, + { + "name": "ranges", + "kind": "field", + "signature": "ranges: Vector", + "doc": "The ranges `RANGES` instructions refer to." + }, + { + "name": "slotCount", + "kind": "field", + "signature": "slotCount: u64", + "doc": "Two slots per group, the whole match being group 0." + } + ] + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::compileRegex", + "name": "compileRegex", + "kind": "function", + "visibility": "public", + "signature": "function compileRegex(ast: &RegexAst): Result", + "doc": "Compiles `ast` to `SAVE 0, , SAVE 1, MATCH`, or fails with \n`pattern too large` when the program would exceed `REGEX_MAX_PROGRAM`." + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::regexAssertionCode", + "name": "regexAssertionCode", + "kind": "function", + "visibility": "private", + "signature": "function regexAssertionCode(assertion: RegexAssertion): u64" + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::regexCompileAlternate", + "name": "regexCompileAlternate", + "kind": "function", + "visibility": "private", + "signature": "function regexCompileAlternate(compiler: &mut RegexCompiler, ast: &RegexAst, children: &Vector): void" + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::regexCompileNode", + "name": "regexCompileNode", + "kind": "function", + "visibility": "private", + "signature": "function regexCompileNode(compiler: &mut RegexCompiler, ast: &RegexAst, id: u64): void" + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::regexCompileRepeat", + "name": "regexCompileRepeat", + "kind": "function", + "visibility": "private", + "signature": "function regexCompileRepeat(compiler: &mut RegexCompiler, ast: &RegexAst, child: u64, min: u32, max: u32, bounded: boolean, greedy: boolean): void" + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::regexEmit", + "name": "regexEmit", + "kind": "function", + "visibility": "private", + "signature": "function regexEmit(compiler: &mut RegexCompiler, opcode: RegexOpcode, first: u64, second: u64): u64" + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::regexEmitBodySplit", + "name": "regexEmitBodySplit", + "kind": "function", + "visibility": "private", + "signature": "function regexEmitBodySplit(compiler: &mut RegexCompiler, greedy: boolean): u64", + "doc": "A `SPLIT` whose body starts right after it; the other target is patched \nlater. Greedy prefers the body, lazy the exit." + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::regexEmitRanges", + "name": "regexEmitRanges", + "kind": "function", + "visibility": "private", + "signature": "function regexEmitRanges(compiler: &mut RegexCompiler, ranges: &Vector, negated: boolean): void" + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::regexPatchExit", + "name": "regexPatchExit", + "kind": "function", + "visibility": "private", + "signature": "function regexPatchExit(compiler: &mut RegexCompiler, split: u64, greedy: boolean, target: u64): void" + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::regexPatchFirst", + "name": "regexPatchFirst", + "kind": "function", + "visibility": "private", + "signature": "function regexPatchFirst(compiler: &mut RegexCompiler, pc: u64, target: u64): void" + }, + { + "module": "std::regex::compiler", + "path": "std::regex::compiler::regexPatchSecond", + "name": "regexPatchSecond", + "kind": "function", + "visibility": "private", + "signature": "function regexPatchSecond(compiler: &mut RegexCompiler, pc: u64, target: u64): void" + }, { "module": "std::regex::parser", "path": "std::regex::parser::REGEX_MAX_REPETITION", @@ -16828,6 +17100,324 @@ "signature": "function regexShorthand(letter: u32, ranges: &mut Vector): boolean", "doc": "The ranges of `\\d`, `\\w`, `\\s` or their negations, or `false` when \n`letter` is none of them." }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::Match", + "name": "Match", + "kind": "record", + "visibility": "public", + "signature": "record Match", + "doc": "One match: byte offsets into the searched text, always on scalar \nboundaries. It stores no view of the text.", + "members": [ + { + "name": "matchStart", + "kind": "field", + "signature": "matchStart: u64" + }, + { + "name": "matchEnd", + "kind": "field", + "signature": "matchEnd: u64" + }, + { + "name": "create", + "kind": "method", + "signature": "static create(start: u64, end: u64): Match", + "doc": "A match of the bytes `start .. end`." + }, + { + "name": "end", + "kind": "method", + "signature": "end(&self): u64", + "doc": "The byte offset just past the match." + }, + { + "name": "isEmpty", + "kind": "method", + "signature": "isEmpty(&self): boolean", + "doc": "Whether the match is empty." + }, + { + "name": "length", + "kind": "method", + "signature": "length(&self): u64", + "doc": "The length of the match in bytes." + }, + { + "name": "start", + "kind": "method", + "signature": "start(&self): u64", + "doc": "The byte offset where the match starts." + }, + { + "name": "textIn", + "kind": "method", + "signature": "textIn(&self, haystack: str): String", + "doc": "A copy of the matched bytes of `haystack`, which must be the searched \ntext." + } + ] + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::Regex", + "name": "Regex", + "kind": "record", + "visibility": "public", + "signature": "record Regex", + "doc": "A compiled regular expression.", + "members": [ + { + "name": "program", + "kind": "field", + "signature": "program: RegexProgram" + }, + { + "name": "new", + "kind": "method", + "signature": "static new(pattern: str): Result", + "doc": "Parses and compiles `pattern`." + }, + { + "name": "find", + "kind": "method", + "signature": "find(&self, text: str): Option", + "doc": "The leftmost-first match in `text`." + }, + { + "name": "findAll", + "kind": "method", + "signature": "findAll(&self, text: str): Vector", + "doc": "Every non-overlapping match in `text`, left to right. After an empty \nmatch the search resumes one scalar further on." + }, + { + "name": "isMatch", + "kind": "method", + "signature": "isMatch(&self, text: str): boolean", + "doc": "Whether `text` contains a match." + } + ] + }, + { + "module": "std::regex::regex", + "path": "std::regex::regex::regexFindFrom", + "name": "regexFindFrom", + "kind": "function", + "visibility": "private", + "signature": "function regexFindFrom(vm: &mut RegexVm, program: &RegexProgram, text: &String, startAt: u64): Option" + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::REGEX_NO_SLOT", + "name": "REGEX_NO_SLOT", + "kind": "constant", + "visibility": "public", + "signature": "const REGEX_NO_SLOT: u64" + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::RegexFrame", + "name": "RegexFrame", + "kind": "record", + "visibility": "private", + "signature": "record RegexFrame", + "doc": "A pending step of `regexAddThread`: follow `pc`, or restore `slot` to \n`value` once the branch that saved it is done.", + "members": [ + { + "name": "pc", + "kind": "field", + "signature": "pc: u64" + }, + { + "name": "restore", + "kind": "field", + "signature": "restore: boolean" + }, + { + "name": "slot", + "kind": "field", + "signature": "slot: u64" + }, + { + "name": "value", + "kind": "field", + "signature": "value: u64" + } + ] + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::RegexThreadList", + "name": "RegexThreadList", + "kind": "record", + "visibility": "private", + "signature": "record RegexThreadList", + "doc": "The threads of one step, in priority order, with the instructions \nalready visited at this position so that each is followed once.", + "members": [ + { + "name": "sparse", + "kind": "field", + "signature": "sparse: Vector" + }, + { + "name": "dense", + "kind": "field", + "signature": "dense: Vector" + }, + { + "name": "visitedCount", + "kind": "field", + "signature": "visitedCount: u64" + }, + { + "name": "pcs", + "kind": "field", + "signature": "pcs: Vector" + }, + { + "name": "slots", + "kind": "field", + "signature": "slots: Vector", + "doc": "`slotCount` slots per entry of `pcs`." + } + ] + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::RegexVm", + "name": "RegexVm", + "kind": "record", + "visibility": "public", + "signature": "record RegexVm", + "doc": "The working state of searches with one program, reused across them.", + "members": [ + { + "name": "slotCount", + "kind": "field", + "signature": "slotCount: u64" + }, + { + "name": "stack", + "kind": "field", + "signature": "stack: Vector" + }, + { + "name": "scratch", + "kind": "field", + "signature": "scratch: Vector" + }, + { + "name": "current", + "kind": "field", + "signature": "current: RegexThreadList" + }, + { + "name": "next", + "kind": "field", + "signature": "next: RegexThreadList" + } + ] + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::regexAddThread", + "name": "regexAddThread", + "kind": "function", + "visibility": "private", + "signature": "function regexAddThread(vm: &mut RegexVm, program: &RegexProgram, text: &String, list: &mut RegexThreadList, startPc: u64, position: u64): void", + "doc": "Follows `pc` at `position` through every non-consuming instruction, with \nthe slots in `vm.scratch`, and appends the consuming instructions reached \nto `list` in priority order. An explicit stack keeps deep programs off the \ncall stack." + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::regexAdvanceThreadLists", + "name": "regexAdvanceThreadLists", + "kind": "function", + "visibility": "private", + "signature": "function regexAdvanceThreadLists(vm: &mut RegexVm): void", + "doc": "Makes the threads of the next position current, and empties `next` for \nthe step after. The visited set comes along: a new start thread added at \nthat position must not revisit what those threads already followed. The \ncopy costs as much as building `next` did, so the step stays linear." + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::regexAssertionHolds", + "name": "regexAssertionHolds", + "kind": "function", + "visibility": "private", + "signature": "function regexAssertionHolds(text: &String, code: u64, position: u64): boolean" + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::regexClearThreadList", + "name": "regexClearThreadList", + "kind": "function", + "visibility": "private", + "signature": "function regexClearThreadList(list: &mut RegexThreadList): void" + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::regexIsWordByte", + "name": "regexIsWordByte", + "kind": "function", + "visibility": "private", + "signature": "function regexIsWordByte(byte: u8): boolean" + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::regexLoadScratch", + "name": "regexLoadScratch", + "kind": "function", + "visibility": "private", + "signature": "function regexLoadScratch(vm: &mut RegexVm, thread: u64): void" + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::regexNewThreadList", + "name": "regexNewThreadList", + "kind": "function", + "visibility": "private", + "signature": "function regexNewThreadList(size: u64): RegexThreadList" + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::regexNewVm", + "name": "regexNewVm", + "kind": "function", + "visibility": "public", + "signature": "function regexNewVm(program: &RegexProgram): RegexVm", + "doc": "The state for searching with `program`." + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::regexRangesContain", + "name": "regexRangesContain", + "kind": "function", + "visibility": "private", + "signature": "function regexRangesContain(program: &RegexProgram, inst: &RegexInst, scalar: u32): boolean" + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::regexResetScratch", + "name": "regexResetScratch", + "kind": "function", + "visibility": "private", + "signature": "function regexResetScratch(vm: &mut RegexVm): void" + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::regexSearch", + "name": "regexSearch", + "kind": "function", + "visibility": "public", + "signature": "function regexSearch(vm: &mut RegexVm, program: &RegexProgram, text: &String, startAt: u64, earliest: boolean): Option>", + "doc": "Searches `text` from byte offset `startAt` for the leftmost-first match. \nReturns its slots (`slotCount` entries, `REGEX_NO_SLOT` where a group did \nnot take part), or `NONE`. With `earliest` it stops at the first match \nfound, whose end may not be the preferred one." + }, + { + "module": "std::regex::vm", + "path": "std::regex::vm::regexVisit", + "name": "regexVisit", + "kind": "function", + "visibility": "private", + "signature": "function regexVisit(list: &mut RegexThreadList, pc: u64): boolean", + "doc": "Marks `pc` visited and reports whether it already was." + }, { "module": "std::result", "path": "std::result::Result",