Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions std/regex/compiler.ign
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ export record RegexProgram {

/// Two slots per group, the whole match being group 0.
public slotCount: u64;

/// The UTF-8 bytes every match starts with; empty when there is no such
/// literal prefix. The search skips to its occurrences.
public prefix: String;
}

/// `ASSERT` operand codes.
Expand Down Expand Up @@ -282,6 +286,53 @@ function regexCompileNode(compiler: &mut RegexCompiler, ast: &RegexAst, id: u64)
};
}

/// Appends to `prefix` the literal scalars every match of node `id` starts
/// with, and reports whether the node is exactly those scalars, so that a
/// concatenation may continue past it.
function regexAppendPrefix(ast: &RegexAst, id: u64, prefix: &mut String): boolean {
match (ast.nodes.get(id).unwrap()) {
RegexNode::LITERAL(scalar) -> {
prefix.pushChar(scalar as char);
return true;
},
RegexNode::CONCAT(children) -> {
let mut index: u64 = 0;

while (index < children.length()) {
if (!regexAppendPrefix(ast, *children.get(index).unwrap(), prefix)) {
return false;
}

index += 1;
}

return true;
},
RegexNode::GROUP(group) -> {
return regexAppendPrefix(ast, group.child, prefix);
},
RegexNode::REPEAT(repeat) -> {
// The first iteration is required; what follows it is not known.
if (repeat.min >= 1) {
regexAppendPrefix(ast, repeat.child, prefix);
}

return false;
},
_ -> {
return false;
},
};
}

/// The literal prefix of every match of `ast`. Under `(?i)` letters are
/// classes, so a case-insensitive pattern has no letter in its prefix.
function regexLiteralPrefix(ast: &RegexAst): String {
let mut prefix: String = String::new();
regexAppendPrefix(ast, ast.root, &mut prefix);
return prefix;
}

/// Compiles `ast` to `SAVE 0, <pattern>, SAVE 1, MATCH`, or fails with
/// `pattern too large` when the program would exceed `REGEX_MAX_PROGRAM`.
export function compileRegex(ast: &RegexAst): Result<RegexProgram, RegexError> {
Expand All @@ -304,5 +355,6 @@ export function compileRegex(ast: &RegexAst): Result<RegexProgram, RegexError> {
insts: compiler.insts,
ranges: compiler.ranges,
slotCount: (ast.groupCount + 1) * 2,
prefix: regexLiteralPrefix(ast),
});
}
80 changes: 68 additions & 12 deletions std/regex/mod.ign
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,23 @@
//! backtracks.
//!
//! ```ignis
//! import Match, Regex from "std::regex";
//! import Captures, Match, Regex from "std::regex";
//!
//! let regex: Regex = Regex::new("\\d+").unwrap();
//! let found: Match = regex.find("abc 123").unwrap(); // 4..7
//! let digits: Regex = Regex::new("\\d+").unwrap();
//! let found: Match = digits.find("abc 123").unwrap(); // 4..7
//! let number: String = found.textIn("abc 123"); // "123"
//!
//! let date: Regex = Regex::new("(?<year>\\d{4})-(\\d\\d)").unwrap();
//! let captures: Captures = date.captures("on 2024-05").unwrap();
//! let year: Match = captures.name("year").unwrap(); // 3..7
//! let swapped: String = date.replaceAll("on 2024-05", "$2/${year}"); // "on 05/2024"
//!
//! let comma: Regex = Regex::new("\\s*,\\s*").unwrap();
//! let parts: Vector<String> = comma.split("a , b,c"); // "a", "b", "c"
//! ```
//!
//! Captures, `replaceAll` and `split` arrive in the next slices.
//! A pattern is a `str`, so a regex backslash is written `\\` in an Ignis
//! string literal: `"\\d+"` is the pattern `\d+`.
//!
//! ## Matching
//!
Expand All @@ -27,21 +37,59 @@
//! - `findAll` returns non-overlapping matches; after an empty match it
//! resumes one scalar further on, so `a*` on `"baa"` gives `0..0`,
//! `1..3` and `3..3`.
//! - Group 0 is the whole match; group `n` is the `n`-th `(`. A group that
//! did not take part in the match is `NONE` (`(a)|(b)` on `"b"`: group 1
//! `NONE`, group 2 `0..1`), and a group in a repetition holds its last
//! iteration (`(a|b)+` on `"ab"`: group 1 `1..2`).
//! - `replaceAll`, `split` and `capturesAll` use the matches `findAll`
//! finds, so `x*` replaced by `-` in `"abc"` gives `"-a-b-c-"`.
//! - Matching takes O(text x program) time: the VM runs every thread in
//! lockstep and follows each instruction at most once per position, so
//! patterns such as `(a*)*b` cannot backtrack.
//!
//! ## Replacement
//!
//! In the replacement text of `replace` and `replaceAll`:
//!
//! - `$n`, one or more digits read greedily (`$12` is group 12), and `${n}`
//! insert group `n`; `${1}0` is group 1 followed by `0`.
//! - `${name}` inserts the named group. Braces hold `[A-Za-z0-9_]+`: all
//! digits is a number, anything else a name.
//! - A group that did not take part or does not exist inserts nothing.
//! - `$$` is a literal `$`; any other `$` (`$x`, `${}`, `${a b}`, a final
//! `$`) is a literal `$`. A name needs braces: `$name` is the text
//! `$name`, not the group.
//!
//! ## Limits
//!
//! Counted repetition copies its body, so a program holds at most
//! `REGEX_MAX_PROGRAM` (100000) instructions; a larger one is
//! `pattern too large` at column 1.
//! - A count in `{n}`, `{n,}` or `{n,m}` is at most `REGEX_MAX_REPETITION`
//! (1000); a larger one is `repetition count above 1000`.
//! - Counted repetition copies its body, so a program holds at most
//! `REGEX_MAX_PROGRAM` (100000) instructions; a larger one, such as
//! `(a{1000}){1000}`, is `pattern too large` at column 1.
//!
//! ## Performance
//!
//! - Matching is linear: a Pike VM runs every thread in lockstep over the
//! text, one scalar per step, and follows each instruction at most once per
//! position, so a search takes O(text x program) time and memory
//! proportional to the program, whatever the pattern.
//! - When every match starts with the same literal scalars (`foo(bar|baz)`
//! starts with `foo`), the search skips straight to the next occurrence of
//! that prefix whenever no thread is alive, and stops when there is none.
//! A pattern that starts with `^`, a class, an alternation, a repetition
//! that may be empty or a case-insensitive letter has no prefix.
//! - `Regex` holds only the compiled program. Every call (`isMatch`, `find`,
//! `findAll`, ...) allocates its own VM state, sized by the program, and
//! copies the text once; a call on many matches reuses that state.
//! - `isMatch` stops at the first match found and is the cheapest test.
//!
//! ## Exports
//!
//! | Export | Description |
//! |------------------------|----------------------------------------------------------------|
//! | `Regex` | `Regex::new(pattern)`: `Result<Regex, RegexError>`; `isMatch(text)`, `find(text)`: `Option<Match>`, `findAll(text)`: `Vector<Match>` |
//! | `Regex` | `Regex::new(pattern)`: `Result<Regex, RegexError>`; `isMatch(text)`, `find(text)`: `Option<Match>`, `findAll(text)`: `Vector<Match>`, `captures(text)`: `Option<Captures>`, `capturesAll(text)`: `Vector<Captures>`, `replace(text, replacement)` and `replaceAll(text, replacement)`: `String`, `split(text)`: `Vector<String>`, `groupCount()`, `groupName(index)`: `Option<String>`, `groupIndex(name)`: `Option<u64>` |
//! | `Captures` | `group(index)` and `name(name)`: `Option<Match>`, `whole()`: `Match`, `groupCount()` (whole match excluded), `length()` (included) |
//! | `Match` | `start()`, `end()`, `length()` in bytes, `isEmpty()`, `textIn(haystack)`: a copy of the matched text |
//! | `REGEX_MAX_PROGRAM` | The most instructions a compiled pattern may hold: 100000 |
//! | `parseRegex` | `parseRegex(pattern)`: `Result<RegexAst, RegexError>` |
Expand Down Expand Up @@ -98,9 +146,17 @@
//! 1000`, `invalid group name`, `duplicate group name x`, `unknown group
//! flag`, `unsupported: backreference` and `unsupported: lookaround`.
//!
//! Backreferences (`\1`) and lookaround (`(?=`, `(?!`, `(?<=`, `(?<!`) are
//! rejected because they cannot be matched in linear time. Classes are
//! ASCII; Unicode classes are not supported in v1.
//! ## Unsupported
//!
//! - Backreferences (`\1`) and lookaround (`(?=`, `(?!`, `(?<=`, `(?<!`)
//! are rejected with `unsupported: backreference` and
//! `unsupported: lookaround`: they cannot be matched in linear time.
//! - `\d`, `\w`, `\s`, `\b` and `(?i)` are ASCII-only in v1: `\d` does not
//! match other Unicode digits, and `(?i)` folds only `A-Z` and `a-z`, so
//! `(?i)straße` matches `"STRAßE"` but not `"STRASSE"`. Unicode classes
//! (`\p{...}`) are not supported.
//! - Other flags (`(?m)`, `(?s)`, `(?x)`) are `unknown group flag`; `^` and
//! `$` always mean the whole text.

import _ from "std::regex::ast";
import _ from "std::regex::parser";
Expand All @@ -120,4 +176,4 @@ export
from "./ast";
export REGEX_MAX_REPETITION, parseRegex from "./parser";
export REGEX_MAX_PROGRAM from "./compiler";
export Match, Regex from "./regex";
export Captures, Match, Regex from "./regex";
Loading
Loading