interpreter: implement RS regexp and blank-line record splitting

interpreter/src/ir.rs

@@ -178,6 +178,7 @@ pub enum BuiltInVar {
     Fpat = var_n(Variable::Fpat),
     Fieldwidths = var_n(Variable::Fieldwidths),
     Rs = var_n(Variable::Rs),
+    Rt = var_n(Variable::Rt),
     Ofs = var_n(Variable::Ofs),
     Ors = var_n(Variable::Ors),
     Filename = var_n(Variable::Filename),
@@ -296,6 +297,7 @@ impl<'a, 'r> TryFrom<&'r Variable<'a>> for BuiltInVar {
             Variable::Fpat => Ok(Self::Fpat),
             Variable::Fieldwidths => Ok(Self::Fieldwidths),
             Variable::Rs => Ok(Self::Rs),
+            Variable::Rt => Ok(Self::Rt),
             Variable::Ofs => Ok(Self::Ofs),
             Variable::Ors => Ok(Self::Ors),
             Variable::Filename => Ok(Self::Filename),

interpreter/src/vm.rs

@@ -54,6 +54,10 @@ pub struct Interpreter<'a> {
     registers: Registers<'a>,
     pub(crate) symbols: SymbolTable<'a>,
     pub(crate) record: Record,
+    /// Unconsumed input left after a regexp `RS` match; cleared on each new file.
+    rs_leftover: StdVec<u8>,
+    /// Cached regexp `RS` automaton (`pattern`, `regex`); invalidated on change.
+    rs_regex_cache: Option<(StdVec<u8>, minrx::Regex)>,
     consts: Consts<'a>,
     mode: ExecMode,
     frames: StdVec<CallFrame>,
@@ -114,6 +118,8 @@ impl<'a> Interpreter<'a> {
             registers: Registers(bumpalo::vec![in code.arena; Value::new_untyped(); n_regs + 1]),
             symbols: code.symbols,
             record: Record::new(),
+            rs_leftover: StdVec::new(),
+            rs_regex_cache: None,
             consts: code.consts,
             mode,
             frames: StdVec::new(),

interpreter/src/vm/io.rs

@@ -6,15 +6,16 @@
 #[cfg(unix)]
 use std::{ffi::OsStr, os::unix::ffi::OsStrExt};
 use std::{
-    io::{self, BufRead, Result},
+    io::{self, BufRead, ErrorKind},
     num::NonZero,
+    ops::Range,
     path::{Path, PathBuf},
 };
 
 use atoi::atoi;
 
 use super::ExecMode;
-use crate::{Interpreter, first_char, vm::types::Value};
+use crate::{Interpreter, exactly_one_char, first_char, vm::regex, vm::types::Value};
 
 #[derive(Debug)]
 pub enum FilePath {
@@ -66,71 +67,281 @@ impl Interpreter<'_> {
         self.symbols.errno = Value::new_int(errno);
         self.symbols.fnr = Value::new_int(0);
         self.record.clear();
+        self.rs_leftover.clear();
     }
 
-    // TODO: consider caching the result of all this validation, so we can
-    // directly dispatch to the correct functions and avoid extra writes to RT.
     // TODO: on Windows, we must strip `\r` depending on BINMODE.
-    pub fn read_record(&mut self, reader: impl BufRead) -> Result<bool> {
+    pub fn read_record(&mut self, reader: impl BufRead) -> io::Result<bool> {
         // Update vars
         // TODO: optimize and make more ergonomic
         self.symbols.nr = &self.symbols.fnr + &Value::new_int(1);
         self.symbols.fnr = &self.symbols.fnr + &Value::new_int(1);
 
         let rs = self.symbols.rs.clone();
         let rs_bytes = &*rs.to_bytes();
-        let first = first_char(rs_bytes);
 
-        match self.mode {
-            // Regex matching (GNU extension)
-            ExecMode::Uu | ExecMode::Gnu if first.is_none() => {
-                self.read_record_regex(rs_bytes, reader)
-            }
-            // Single char matching
-            // TODO: what if POSIX & invalid UTF-8?
-            _ if let Some(c) = first => {
-                self.symbols.rt.clone_from(&rs);
+        if rs_bytes.is_empty() {
+            // Blank-line separated records (`RS = ""`).
+            self.read_record_blank_lines(reader)
+        } else if self.mode == ExecMode::Posix {
+            // POSIX: never a regexp — only the first character (or first byte).
+            if let Some(c) = first_char(rs_bytes) {
                 self.read_record_until_char(c, reader)
+            } else {
+                self.read_record_until_byte(rs_bytes[0], reader)
             }
-            // Empty RS
-            _ => self.read_record_blank_lines(reader),
+        } else if let Some(c) = exactly_one_char(rs_bytes) {
+            self.read_record_until_char(c, reader)
+        } else {
+            // Multi-character (or non-UTF-8) `RS` is a regexp (gawk extension).
+            self.read_record_regex(rs_bytes, reader)
         }
     }
 
-    pub fn read_record_regex(&mut self, _rs: &[u8], mut _reader: impl BufRead) -> Result<bool> {
-        todo!()
+    pub fn read_record_regex(&mut self, rs: &[u8], mut reader: impl BufRead) -> io::Result<bool> {
+        self.ensure_rs_regex(rs)?;
+
+        // Grow the leftover buffer until we see a usable non-empty match or EOF.
+        // Empty matches are ignored: `RS = "()"` must not split between chars.
+        // A match that ends at the buffer end may still grow (e.g. `RS = "\n+"`),
+        // so keep reading unless we have reached EOF.
+        let mut at_eof = false;
+        loop {
+            let range = {
+                let re = &self.rs_regex_cache.as_ref().unwrap().1;
+                find_nonempty_match(re, &self.rs_leftover)?
+            };
+            if let Some(range) = range {
+                let touches_end = range.end == self.rs_leftover.len();
+                if !touches_end || at_eof {
+                    let rec = self.record.write_new();
+                    rec.extend_from_slice(&self.rs_leftover[..range.start]);
+                    self.symbols.rt =
+                        Value::new_string(self.rs_leftover[range.start..range.end].into());
+                    self.rs_leftover.drain(..range.end);
+                    return Ok(true);
+                }
+                // Match reaches the end of the buffered input; try to extend it.
+            }
+
+            if at_eof {
+                if self.rs_leftover.is_empty() {
+                    self.symbols.rt = Value::new_str(b"");
+                    return Ok(false);
+                }
+                let rec = self.record.write_new();
+                rec.append(&mut self.rs_leftover);
+                self.symbols.rt = Value::new_str(b"");
+                return Ok(true);
+            }
+
+            let buf = reader.fill_buf()?;
+            if buf.is_empty() {
+                at_eof = true;
+                continue;
+            }
+            let n = buf.len();
+            self.rs_leftover.extend_from_slice(buf);
+            reader.consume(n);
+        }
     }
 
-    pub fn read_record_until_char(&mut self, c: char, mut reader: impl BufRead) -> Result<bool> {
+    /// Compiles `RS` once and reuses the automaton while the pattern is unchanged.
+    fn ensure_rs_regex(&mut self, rs: &[u8]) -> io::Result<()> {
+        let needs_compile = self
+            .rs_regex_cache
+            .as_ref()
+            .is_none_or(|(pat, _)| pat.as_slice() != rs);
+        if needs_compile {
+            let re = regex::automaton(rs, self.mode, false)
+                .map_err(|e| io::Error::new(ErrorKind::InvalidInput, e))?;
+            self.rs_regex_cache = Some((rs.to_vec(), re));
+        }
+        Ok(())
+    }
+
+    pub fn read_record_until_char(
+        &mut self,
+        c: char,
+        mut reader: impl BufRead,
+    ) -> io::Result<bool> {
         let mut bytes = [0; 4];
         let bytes = c.encode_utf8(&mut bytes).as_bytes();
-        let rec = self.record.write_new();
+        self.read_record_until_delim(bytes, &mut reader)
+    }
+
+    fn read_record_until_byte(&mut self, byte: u8, mut reader: impl BufRead) -> io::Result<bool> {
+        self.read_record_until_delim(&[byte], &mut reader)
+    }
+
+    /// Character / byte `RS`, consuming any `rs_leftover` from a previous regexp
+    /// `RS` before reading the underlying stream.
+    fn read_record_until_delim(
+        &mut self,
+        delim: &[u8],
+        reader: &mut impl BufRead,
+    ) -> io::Result<bool> {
+        // Drain leftover from a prior regexp RS first so mid-file RS changes
+        // do not drop already-buffered input.
+        if let Some(pos) = find_delim(&self.rs_leftover, delim) {
+            let rec = self.record.write_new();
+            rec.extend_from_slice(&self.rs_leftover[..pos]);
+            self.symbols.rt = Value::new_string(delim.into());
+            self.rs_leftover.drain(..pos + delim.len());
+            return Ok(true);
+        }
 
         // It is correct behavior that we terminate the record on EOF too,
         // even if the text file is malformed (no trailing newline).
-        match *bytes {
-            [byte] => {
-                // fast path: single search
-                let read = reader.read_until(byte, rec)?;
-                rec.pop_if(|&mut last| last == byte);
-                Ok(read > 0)
+        let (read_ok, found_delim) = {
+            let rec = self.record.write_new();
+            rec.append(&mut self.rs_leftover);
+            match *delim {
+                [byte] => {
+                    // fast path: single search
+                    let read = reader.read_until(byte, rec)?;
+                    let found_delim = rec.pop_if(|&mut last| last == byte).is_some();
+                    (read > 0 || !rec.is_empty() || found_delim, found_delim)
+                }
+                [.., last] => {
+                    // slow path: loop searching multiple bytes or EOF.
+                    let mut found = false;
+                    let mut read_any = false;
+                    while reader.read_until(last, rec)? > 0 {
+                        read_any = true;
+                        if rec.ends_with(delim) {
+                            rec.truncate(rec.len() - delim.len());
+                            found = true;
+                            break;
+                        }
+                    }
+                    (found || read_any || !rec.is_empty(), found)
+                }
+                // This would be a bug in the std. It is often optimized away.
+                _ => unreachable!(),
+            }
+        };
+        self.symbols.rt = if found_delim {
+            Value::new_string(delim.into())
+        } else {
+            Value::new_str(b"")
+        };
+        Ok(read_ok)
+    }
+
+    /// Blank-line record separation for `RS = ""`.
+    ///
+    /// Leading newlines are skipped. A record ends at the first blank line; the
+    /// run of separating newlines becomes `RT`. A final newline at EOF (without
+    /// a following blank line) is stripped from the record and stored in `RT`.
+    ///
+    /// Any `rs_leftover` from a previous regexp `RS` is consumed first.
+    pub fn read_record_blank_lines(&mut self, mut reader: impl BufRead) -> io::Result<bool> {
+        // Merge leftover into a working buffer so mid-file RS changes keep bytes.
+        let mut pending = std::mem::take(&mut self.rs_leftover);
+        let start = pending
+            .iter()
+            .position(|&b| b != b'\n')
+            .unwrap_or(pending.len());
+        pending.drain(..start);
+
+        if pending.is_empty() {
+            // Skip leading newlines on the stream.
+            loop {
+                let buf = reader.fill_buf()?;
+                if buf.is_empty() {
+                    self.symbols.rt = Value::new_str(b"");
+                    return Ok(false);
+                }
+                if buf[0] == b'\n' {
+                    reader.consume(1);
+                    continue;
+                }
+                break;
             }
-            [.., last] => {
-                // slow path: loop searching multiple bytes or EOF.
-                while reader.read_until(last, rec)? > 0 {
-                    if rec.ends_with(bytes) {
-                        rec.truncate(rec.len() - bytes.len());
-                        return Ok(true);
+        }
+
+        let (rt, tail) = {
+            let rec = self.record.write_new();
+            rec.append(&mut pending);
+
+            loop {
+                if let Some((content_len, rt_len)) = blank_line_split(rec) {
+                    let rt = rec[content_len..content_len + rt_len].to_vec();
+                    let tail = rec.split_off(content_len + rt_len);
+                    rec.truncate(content_len);
+                    break (rt, tail);
+                }
+
+                let n = reader.read_until(b'\n', rec)?;
+                if n == 0 || !rec.ends_with(b"\n") {
+                    break (Vec::new(), Vec::new());
+                }
+
+                let buf = reader.fill_buf()?;
+                if buf.is_empty() {
+                    rec.pop();
+                    break (Vec::from(b"\n".as_slice()), Vec::new());
+                }
+                if buf[0] == b'\n' {
+                    rec.pop();
+                    let mut rt = Vec::from(b"\n".as_slice());
+                    while {
+                        let b = reader.fill_buf()?;
+                        !b.is_empty() && b[0] == b'\n'
+                    } {
+                        rt.push(b'\n');
+                        reader.consume(1);
                     }
+                    break (rt, Vec::new());
                 }
-                Ok(!rec.is_empty())
             }
-            // This would be a bug in the std. It is often optimized away.
-            _ => unreachable!(),
+        };
+
+        self.rs_leftover = tail;
+        self.symbols.rt = Value::new_string(rt.as_slice().into());
+        Ok(true)
+    }
+}
+
+/// Finds a blank-line boundary in `buf`.
+///
+/// Returns `(content_len, rt_len)` where `buf[..content_len]` is the record and
+/// `buf[content_len..content_len + rt_len]` is the run of separator newlines.
+fn blank_line_split(buf: &[u8]) -> Option<(usize, usize)> {
+    let mut i = 0;
+    while i < buf.len() {
+        if buf[i] != b'\n' {
+            i += 1;
+            continue;
+        }
+        if i + 1 < buf.len() && buf[i + 1] == b'\n' {
+            let mut j = i;
+            while j < buf.len() && buf[j] == b'\n' {
+                j += 1;
+            }
+            return Some((i, j - i));
         }
+        i += 1;
     }
+    None
+}
 
-    pub fn read_record_blank_lines(&mut self, mut _reader: impl BufRead) -> Result<bool> {
-        todo!()
+fn find_delim(haystack: &[u8], delim: &[u8]) -> Option<usize> {
+    match delim {
+        [] => None,
+        [b] => haystack.iter().position(|x| x == b),
+        d => haystack.windows(d.len()).position(|w| w == d),
+    }
+}
+
+fn find_nonempty_match(re: &minrx::Regex, haystack: &[u8]) -> io::Result<Option<Range<usize>>> {
+    for m in re.find_iter(haystack) {
+        let m = m.map_err(|e| io::Error::new(ErrorKind::InvalidInput, e))?;
+        let range = m.range();
+        if !range.is_empty() {
+            return Ok(Some(range.into()));
+        }
     }
+    Ok(None)
 }

interpreter/src/vm/symbols.rs

@@ -285,6 +285,7 @@ impl<'a> SymbolTable<'a> {
             BuiltInVar::Fpat => &self.fpat,
             BuiltInVar::Fieldwidths => &self.fieldwidths,
             BuiltInVar::Rs => &self.rs,
+            BuiltInVar::Rt => &self.rt,
             BuiltInVar::Ofs => &self.ofs,
             BuiltInVar::Ors => &self.ors,
             BuiltInVar::Filename => &self.filename,
@@ -307,6 +308,7 @@ impl<'a> SymbolTable<'a> {
             BuiltInVar::Fpat => &mut self.fpat,
             BuiltInVar::Fieldwidths => &mut self.fieldwidths,
             BuiltInVar::Rs => &mut self.rs,
+            BuiltInVar::Rt => &mut self.rt,
             BuiltInVar::Ofs => &mut self.ofs,
             BuiltInVar::Ors => &mut self.ors,
             BuiltInVar::Filename => &mut self.filename,

lexer/src/lib.rs

@@ -195,6 +195,8 @@ pub enum Token<'a> {
     FieldwidthsVariable,
     #[regex("(awk::)?RS", accept_expression)]
     RsVariable,
+    #[regex("(awk::)?RT", accept_expression)]
+    RtVariable,
     #[regex("(awk::)?OFS", accept_expression)]
     OfsVariable,
     #[regex("(awk::)?ORS", accept_expression)]