diff --git a/librio-wasm/src/lib.rs b/librio-wasm/src/lib.rs index f9ca7cd333..077f6d84cd 100644 --- a/librio-wasm/src/lib.rs +++ b/librio-wasm/src/lib.rs @@ -69,9 +69,16 @@ pub const COLOR_INDEXED: u32 = 1; pub const COLOR_RGB: u32 = 2; /// u32 words per cell in [`RioTerm::write_cells`]: -/// `[codepoint | wide << 21, fg, bg, style_flags]`. +/// `[codepoint | wide << 21 | flags, fg, bg, style_flags]`. pub const CELL_WORDS: usize = 4; +/// Word-0 flag: the cell carries attached cluster codepoints +/// (combining marks, or a mode-2027 grapheme cluster tail) beyond the +/// base codepoint in bits 0..21. Fetch the full text with +/// [`RioTerm::cluster_text`] and draw that instead of the base char; +/// a renderer that ignores the bit simply keeps drawing bases. +pub const CELL_HAS_CLUSTER: u32 = 1 << 23; + enum Event { Output(Vec), Wakeup, @@ -658,6 +665,16 @@ impl RioTerm { pub fn kitty_image_rgba(&self, image_id: u32, out: &mut [u8]) -> usize { self.state.kitty_image_rgba(image_id, out) } + + /// The full text of a cell flagged [`CELL_HAS_CLUSTER`]: the base + /// codepoint followed by its attached cluster codepoints + /// (combining marks, or a mode-2027 grapheme cluster). Draw this + /// string in place of the base char so a ZWJ emoji or a + /// decomposed accent renders as the glyph the sequence means. + /// `undefined` for cells without attachments. + pub fn cluster_text(&self, line: usize, column: usize) -> Option { + self.state.cell_cluster_text(line, column) + } } impl RioTerm { @@ -667,8 +684,14 @@ impl RioTerm { match self.state.square(line, col) { Some(square) => { let style = self.state.style_of(square); - out[base] = - (square.c() as u32 & 0x1F_FFFF) | ((square.wide() as u32) << 21); + let cluster = if self.state.cluster_of(square).is_some() { + CELL_HAS_CLUSTER + } else { + 0 + }; + out[base] = (square.c() as u32 & 0x1F_FFFF) + | ((square.wide() as u32) << 21) + | cluster; out[base + 1] = pack_color(style.fg); out[base + 2] = pack_color(style.bg); out[base + 3] = style.flags.bits() as u32; diff --git a/librio/include/librio.h b/librio/include/librio.h index 761721b454..95dbab29c4 100644 --- a/librio/include/librio.h +++ b/librio/include/librio.h @@ -255,6 +255,21 @@ uint16_t rio_render_state_lines(const rio_render_state_t *state); uint16_t rio_render_state_columns(const rio_render_state_t *state); bool rio_render_state_row_dirty(const rio_render_state_t *state, uint16_t line); void rio_render_state_reset_dirty(rio_render_state_t *state); +/* Set in rio_cell_s.style_flags when the cell carries attached cluster + * codepoints (combining marks, or a DEC-2027 grapheme cluster) beyond + * `codepoint`. Fetch the full text with rio_render_state_cell_cluster + * and draw that instead of the base char. StyleFlags proper occupies + * bits 0..10; this is bit 15. */ +#define RIO_CELL_HAS_CLUSTER (1u << 15) + +/* Write the full text of a cell (base codepoint plus attached cluster + * codepoints) as UTF-32 into `out` (capacity `cap` code units). + * Returns the total codepoint count, which may exceed `cap` (call + * again with a larger buffer); 0 for plain cells. */ +size_t rio_render_state_cell_cluster(const rio_render_state_t *state, + uint16_t line, uint16_t column, + uint32_t *out, size_t cap); + rio_cell_s rio_render_state_cell(const rio_render_state_t *state, uint16_t line, uint16_t column); rio_cursor_s rio_render_state_cursor(const rio_render_state_t *state); diff --git a/librio/src/capi.rs b/librio/src/capi.rs index d55d4a5918..30588f179a 100644 --- a/librio/src/capi.rs +++ b/librio/src/capi.rs @@ -941,11 +941,16 @@ pub unsafe extern "C" fn rio_render_state_cell( return empty; }; let style = state.style_of(square); + let cluster = if state.cluster_of(square).is_some() { + RIO_CELL_HAS_CLUSTER + } else { + 0 + }; rio_cell_s { codepoint: square.c() as u32, fg: color_to_c(style.fg), bg: color_to_c(style.bg), - style_flags: style.flags.bits(), + style_flags: style.flags.bits() | cluster, } })) .unwrap_or(rio_cell_s { @@ -956,6 +961,51 @@ pub unsafe extern "C" fn rio_render_state_cell( }) } +/// Set in `rio_cell_s.style_flags` when the cell carries attached +/// cluster codepoints (combining marks, or a mode-2027 grapheme +/// cluster) beyond `codepoint`. Fetch the full text with +/// [`rio_render_state_cell_cluster`] and draw that instead of the +/// base char. StyleFlags proper occupies bits 0..10; this is bit 15. +pub const RIO_CELL_HAS_CLUSTER: u16 = 1 << 15; + +/// Write the full text of a cell (base codepoint plus attached +/// cluster codepoints) as UTF-32 into `out` (capacity `cap` code +/// units). Returns the total codepoint count, which may exceed `cap` +/// (call again with a larger buffer); 0 for plain cells, so callers +/// can treat 0 as "draw `codepoint` as usual". +#[no_mangle] +pub unsafe extern "C" fn rio_render_state_cell_cluster( + state: *const RenderState, + line: u16, + column: u16, + out: *mut u32, + cap: usize, +) -> usize { + catch_unwind(AssertUnwindSafe(|| { + if state.is_null() { + return 0; + } + let state = unsafe { &*state }; + let Some(square) = state.square(line as usize, column as usize) else { + return 0; + }; + let Some(cluster) = state.cluster_of(square) else { + return 0; + }; + let total = 1 + cluster.len(); + if !out.is_null() { + let write = total.min(cap); + let dst = unsafe { core::slice::from_raw_parts_mut(out, write) }; + let mut src = core::iter::once(square.c()).chain(cluster.iter().copied()); + for slot in dst.iter_mut() { + *slot = src.next().unwrap_or('\0') as u32; + } + } + total + })) + .unwrap_or(0) +} + /// Lines the view is scrolled up into history; 0 means the live screen. /// Renderers use this to hide the cursor while scrolled. #[no_mangle] diff --git a/librio/src/lib.rs b/librio/src/lib.rs index 7a50585882..f58f611b94 100644 --- a/librio/src/lib.rs +++ b/librio/src/lib.rs @@ -1236,6 +1236,33 @@ mod tests { assert!(state.display_offset() > 0, "the view should have moved"); } + // The wasm renderer draws a cell's full cluster text (base + + // attached codepoints) when the wire flag says one exists; the + // accessor is the source of that text. + #[test] + fn render_state_exposes_cluster_text() { + let engine = Engine::new(Arc::new(CountingDelegate { + wakeups: AtomicUsize::new(0), + })); + let surface = engine + .create_surface(&SurfaceDesc::default()) + .expect("spawn shell"); + let mut state = RenderState::new(&surface); + + // Mode 2027 on, then a ZWJ emoji and a decomposed accent. + surface + .inject_output("\x1b[?2027h\u{1F9D1}\u{200D}\u{1F33E}e\u{301}x".as_bytes()); + state.update(); + + assert_eq!( + state.cell_cluster_text(0, 0).as_deref(), + Some("\u{1F9D1}\u{200D}\u{1F33E}") + ); + assert_eq!(state.cell_cluster_text(0, 2).as_deref(), Some("e\u{301}")); + // Plain cells report nothing. + assert_eq!(state.cell_cluster_text(0, 3), None); + } + // SGR is the modern form; the X10 fallback offsets by 32. #[test] fn mouse_reports_encode_both_forms() { diff --git a/librio/src/render_state.rs b/librio/src/render_state.rs index 656b519f3b..e2c4991df3 100644 --- a/librio/src/render_state.rs +++ b/librio/src/render_state.rs @@ -206,6 +206,38 @@ impl RenderState { } } + /// The codepoints attached to a square's grapheme cluster + /// (combining marks, or a full mode-2027 cluster tail) when it + /// carries any. Bg-only squares reuse the extras-id bits for + /// color and never report a cluster. + pub fn cluster_of(&self, square: &Square) -> Option<&[char]> { + if !square.has_grapheme() + || !matches!( + square.content_tag(), + rio_vt::crosswords::square::ContentTag::Codepoint + ) + { + return None; + } + let extras = square.extras_id().and_then(|eid| self.extras.get(&eid))?; + if extras.zerowidth.is_empty() { + None + } else { + Some(&extras.zerowidth) + } + } + + /// The cell's full text (base codepoint plus attached cluster + /// codepoints), or `None` for cells with no attachments. + pub fn cell_cluster_text(&self, line: usize, column: usize) -> Option { + let square = self.square(line, column)?; + let cluster = self.cluster_of(square)?; + let mut text = String::with_capacity(4 * (1 + cluster.len())); + text.push(square.c()); + text.extend(cluster.iter()); + Some(text) + } + pub fn square(&self, line: usize, column: usize) -> Option<&Square> { let row = self.rows.get(line)?; if column >= self.columns { diff --git a/rio-unicode/src/grapheme.rs b/rio-unicode/src/grapheme.rs index 181fac2864..1f52bf0723 100644 --- a/rio-unicode/src/grapheme.rs +++ b/rio-unicode/src/grapheme.rs @@ -49,6 +49,38 @@ pub enum GraphemeClass { Consonant, } +impl GraphemeClass { + /// Number of classes; `as u8` values are contiguous in + /// `0..COUNT`, so classes can index precomputed tables. + pub const COUNT: usize = 18; + + /// Inverse of `as u8`, for table builders enumerating classes. + pub fn from_u8(value: u8) -> Option { + use GraphemeClass as G; + Some(match value { + 0 => G::Other, + 1 => G::CR, + 2 => G::LF, + 3 => G::Control, + 4 => G::Extend, + 5 => G::ExtendIncb, + 6 => G::Linker, + 7 => G::Zwj, + 8 => G::RegionalIndicator, + 9 => G::Prepend, + 10 => G::SpacingMark, + 11 => G::L, + 12 => G::V, + 13 => G::T, + 14 => G::LV, + 15 => G::LVT, + 16 => G::ExtPic, + 17 => G::Consonant, + _ => return None, + }) + } +} + /// Class lookup: binary search over the generated ranges. pub fn grapheme_class(c: char) -> GraphemeClass { let cp = c as u32; @@ -101,6 +133,45 @@ pub struct BreakState { } impl BreakState { + /// Number of distinct states; `pack` values are contiguous in + /// `0..COUNT`, so states can index precomputed transition tables. + pub const COUNT: usize = 18; + + /// Pack into `0..COUNT` for table indexing. + pub fn pack(self) -> u8 { + let emoji = match self.emoji { + EmojiSeq::None => 0u8, + EmojiSeq::Emoji => 1, + EmojiSeq::EmojiZwj => 2, + }; + let incb = match self.incb { + Incb::None => 0u8, + Incb::Consonant => 1, + Incb::LinkerSeen => 2, + }; + emoji * 6 + incb * 2 + self.ri_odd as u8 + } + + /// Inverse of [`pack`](Self::pack). + pub fn unpack(value: u8) -> Option { + if value as usize >= Self::COUNT { + return None; + } + Some(Self { + emoji: match value / 6 { + 0 => EmojiSeq::None, + 1 => EmojiSeq::Emoji, + _ => EmojiSeq::EmojiZwj, + }, + incb: match (value % 6) / 2 { + 0 => Incb::None, + 1 => Incb::Consonant, + _ => Incb::LinkerSeen, + }, + ri_odd: value % 2 == 1, + }) + } + /// State after the first codepoint of a sequence. pub fn start(first: GraphemeClass) -> Self { let mut state = Self::default(); diff --git a/rio-vt/Cargo.toml b/rio-vt/Cargo.toml index 9b67e55305..d8e28b3e7c 100644 --- a/rio-vt/Cargo.toml +++ b/rio-vt/Cargo.toml @@ -24,6 +24,8 @@ parking_lot = { workspace = true } serde = { workspace = true } rio-graphics = { workspace = true } rio-grapheme-width = { workspace = true } +# The workspace aliases this to the rio-unicode crate, which also +# carries the UAX #29 grapheme segmentation used by mode 2027. unicode-width = { workspace = true } regex-automata = "0.4.16" cursor-icon = { version = "1.1.0", default-features = false } diff --git a/rio-vt/src/ansi/mode.rs b/rio-vt/src/ansi/mode.rs index f029cb2c69..0c01d373d1 100644 --- a/rio-vt/src/ansi/mode.rs +++ b/rio-vt/src/ansi/mode.rs @@ -70,6 +70,7 @@ impl PrivateMode { 1049 => Self::Named(NamedPrivateMode::SwapScreenAndSetRestoreCursor), 2004 => Self::Named(NamedPrivateMode::BracketedPaste), 2026 => Self::Named(NamedPrivateMode::SyncUpdate), + 2027 => Self::Named(NamedPrivateMode::GraphemeCluster), _ => Self::Unknown(mode), } } @@ -127,6 +128,14 @@ pub enum NamedPrivateMode { BracketedPaste = 2004, /// The mode is handled automatically by [`Processor`]. SyncUpdate = 2026, + /// Grapheme cluster processing (contour's terminal-unicode-core). + /// + /// When set, incoming text is segmented into UAX #29 extended + /// grapheme clusters: a multi-codepoint cluster (a ZWJ emoji, a + /// conjunct) occupies one cell slot instead of one slot per + /// width-bearing codepoint. Queryable via DECRQM, which is how + /// applications detect support. Default: reset. + GraphemeCluster = 2027, } /// Mode for clearing line. diff --git a/rio-vt/src/crosswords/mod.rs b/rio-vt/src/crosswords/mod.rs index 30b1d68f66..6b9bb353ea 100644 --- a/rio-vt/src/crosswords/mod.rs +++ b/rio-vt/src/crosswords/mod.rs @@ -98,6 +98,10 @@ bitflags! { const REPORT_ALL_KEYS_AS_ESC = 1 << 21; const REPORT_ASSOCIATED_TEXT = 1 << 22; const MOUSE_REPORT_X10 = 1 << 23; + /// DEC private mode 2027: grapheme cluster processing. Shared + /// across main/alt screens (the mode lives on the terminal, + /// not the grid), matching ghostty and contour. + const GRAPHEME_CLUSTER = 1 << 24; const MOUSE_MODE = Self::MOUSE_REPORT_CLICK.bits() | Self::MOUSE_MOTION.bits() | Self::MOUSE_DRAG.bits() | Self::MOUSE_REPORT_X10.bits(); const KITTY_KEYBOARD_PROTOCOL = Self::DISAMBIGUATE_ESC_CODES.bits() | Self::REPORT_EVENT_TYPES.bits() @@ -1523,6 +1527,144 @@ impl Crosswords { } } + /// Append `c` to the cluster of the cell most recently written at + /// the cursor (legacy zero-width semantics: `saturating_sub` + /// column math, so a mark at column 0 targets column 0). + fn attach_to_prev_cell(&mut self, c: char) { + let mut column = self.grid.cursor.pos.col; + if !self.grid.cursor.should_wrap { + column.0 = column.saturating_sub(1); + } + + let row = self.grid.cursor.pos.row; + if matches!(self.grid[row][column].wide(), Wide::Spacer) { + column.0 = column.saturating_sub(1); + } + + // A bg-only cell reuses the extras-id bits for its color + // channel: reading them as an id would clone an unrelated + // slot onto this cell, and writing one back would corrupt + // the color. A combining mark with no base character has + // nothing to attach to, so drop it. + if !matches!( + self.grid[row][column].content_tag(), + crate::crosswords::square::ContentTag::Codepoint + ) { + return; + } + // Copy-on-write: slots are interned and shared; every + // cell written under one OSC 8 template references the + // same slot, so pushing into it in place would attach the + // mark to the whole hyperlink span. Clone, extend, and + // re-intern instead; the marked cell gets its own + // (marks + hyperlink) slot and its neighbors keep theirs. + let existing_id = self.grid[row][column].extras_id(); + let mut extras = existing_id + .and_then(|id| self.grid.extras_table.get(id).cloned()) + .unwrap_or_default(); + extras.zerowidth.push(c); + let id = self.grid.alloc_extras(extras); + if id != 0 { + let cell = &mut self.grid[row][column]; + cell.set_extras_id(Some(id)); + cell.insert_cell_flag(CellFlags::GRAPHEME); + self.grid[row].has_extras = true; + } + // id == 0: slot space exhausted. Keep the cell's existing + // extras (hyperlink, earlier marks) rather than erasing + // them; only the new mark is lost. + } + + /// Mode-2027 cluster continuation. `true` means `c` was consumed: + /// appended to the previous cell's cluster (possibly widening it), + /// or deliberately ignored (an invalid variation selector, the + /// ghostty `.ignore` contract). `false` means a grapheme break: + /// the caller writes `c` through the normal paths. + /// + /// There is no cross-call segmentation state. Any no-break + /// sequence accumulates into a single cell, so everything the + /// break rules can look behind at, an emoji ZWJ run (GB11), + /// regional-indicator parity (GB12/13), an Indic conjunct chain + /// (GB9c), is exactly the previous cell's contents, and the + /// `BreakState` is rebuilt from them (a handful of codepoints). + /// Cursor movement, clears, scrolling, and screen switches + /// invalidate a cluster automatically: whatever cell precedes the + /// cursor *is* the truth, with no reset hooks to forget. + fn try_cluster_append(&mut self, c: char, width: usize) -> bool { + use crate::grapheme_lut::{class_of, is_break_lut, start_state}; + + let row = self.grid.cursor.pos.row; + let Some(base_col) = self.prev_cell_col() else { + return false; + }; + let cell = self.grid[row][Column(base_col)]; + if !matches!( + cell.content_tag(), + crate::crosswords::square::ContentTag::Codepoint + ) { + return false; + } + let base = cell.c(); + if base == '\0' { + // Never-written cell: nothing to continue. + return false; + } + + // Reconstruct the segmentation state from the cluster itself. + // Flat-table hops (ghostty devlog-006): one class lookup per + // codepoint and one transition index per step, no rule chain. + let mut prev = class_of(base); + let mut state = start_state(prev); + let mut last_cp = base; + if let Some(id) = cell.extras_id() { + if let Some(extras) = self.grid.extras_table.get(id) { + for &attached in &extras.zerowidth { + let class = class_of(attached); + let _ = is_break_lut(prev, class, &mut state); + prev = class; + last_cp = attached; + } + } + } + if is_break_lut(prev, class_of(c), &mut state) { + return false; + } + + // Joined. Decide the width effect (ghostty + // `graphemeWidthEffect`): variation selectors flip a valid + // base's width and are otherwise ignored outright; any other + // width-bearing continuation makes the whole cluster wide + // (the base already contributed one column); zero-width + // continuations change nothing. + match c { + '\u{FE0F}' => { + // A selector modifies the codepoint immediately before + // it, so validity is judged against the *last* codepoint + // of the cluster, not its base (ghostty checks `prev` + // in `graphemeWidthEffect`). The width helpers are + // no-ops when the cell is already in the target state. + if vs_is_valid_base(last_cp, c) { + self.widen_prev_cell(); + self.attach_to_prev_cell(c); + } + // Invalid selector: ignore. The cell is untouched, so + // the reconstructed state next time is identical. + } + '\u{FE0E}' => { + if vs_is_valid_base(last_cp, c) { + self.narrow_prev_cell(); + self.attach_to_prev_cell(c); + } + } + _ if width == 0 => self.attach_to_prev_cell(c), + _ => { + self.widen_prev_cell(); + self.attach_to_prev_cell(c); + } + } + true + } + fn write_cell(&mut self, c: char, template: crate::crosswords::square::Square) { use crate::crosswords::square::{Square, Wide}; @@ -1811,6 +1953,43 @@ impl Crosswords { /// via cmap format 14, so the grid must budget two cells for it. #[inline(never)] fn apply_emoji_vs16(&mut self) { + let Some(base_col) = self.prev_cell_col() else { + return; + }; + let base_cell = &self.grid[self.grid.cursor.pos.row][Column(base_col)]; + if !matches!(base_cell.wide(), Wide::Narrow) { + return; + } + if !vs_is_valid_base(base_cell.c(), '\u{FE0F}') { + return; + } + self.widen_prev_cell(); + } + + /// The column of the cell most recently written at the cursor: the + /// cursor column itself while a pending wrap holds the cursor past + /// the edge, otherwise the column before it, stepping over a wide + /// pair's spacer to its lead. `None` when no such cell exists. + fn prev_cell_col(&self) -> Option { + let col = self.grid.cursor.pos.col.0; + let col = if self.grid.cursor.should_wrap { + col + } else { + col.checked_sub(1)? + }; + let row = self.grid.cursor.pos.row; + Some(match self.grid[row][Column(col)].wide() { + Wide::Spacer => col.checked_sub(1)?, + _ => col, + }) + } + + /// Widen the narrow cell preceding the cursor into a wide pair, + /// handling the row-edge wrap. Shared by VS16 promotion and + /// mode-2027 cluster continuation (a width-bearing codepoint + /// joining a narrow cluster makes the whole cluster wide). + #[inline(never)] + fn widen_prev_cell(&mut self) { let columns = self.grid.columns(); // No wide pair fits on one column, so leave the base narrow: the // wrap branch below would place the trailing Spacer at column 1 of @@ -1819,23 +1998,10 @@ impl Crosswords { return; } let row = self.grid.cursor.pos.row; - let cursor_col = self.grid.cursor.pos.col.0; - let should_wrap = self.grid.cursor.should_wrap; - - let base_col = if should_wrap { - cursor_col - } else if cursor_col == 0 { + let Some(base_col) = self.prev_cell_col() else { return; - } else { - cursor_col - 1 }; - - let base_cell = &self.grid[row][Column(base_col)]; - if !matches!(base_cell.wide(), Wide::Narrow) { - return; - } - let base_char = base_cell.c(); - if !vs_is_valid_base(base_char, '\u{FE0F}') { + if !matches!(self.grid[row][Column(base_col)].wide(), Wide::Narrow) { return; } @@ -1915,24 +2081,53 @@ impl Crosswords { let cursor_col = self.grid.cursor.pos.col.0; let should_wrap = self.grid.cursor.should_wrap; - let (base_col, spacer_col) = if should_wrap { + let base_col = if should_wrap { if cursor_col == 0 { return; } - (cursor_col - 1, cursor_col) + cursor_col - 1 } else { if cursor_col < 2 { return; } - (cursor_col - 2, cursor_col - 1) + cursor_col - 2 }; let base_cell = &self.grid[row][Column(base_col)]; if !matches!(base_cell.wide(), Wide::Wide) { return; } - let base_char = base_cell.c(); - if !vs_is_valid_base(base_char, '\u{FE0E}') { + if !vs_is_valid_base(base_cell.c(), '\u{FE0E}') { + return; + } + + self.narrow_prev_cell(); + } + + /// Shrink the wide pair preceding the cursor back to a narrow cell, + /// clearing its spacer and retracting the cursor. No-op unless a + /// wide pair is there. Validity-free counterpart of + /// [`widen_prev_cell`]: VS15 demotion (via `apply_emoji_vs15`, + /// which gates on the base) and mode-2027 cluster continuation + /// (which gates on the cluster's last codepoint) share it. + fn narrow_prev_cell(&mut self) { + let row = self.grid.cursor.pos.row; + let cursor_col = self.grid.cursor.pos.col.0; + let should_wrap = self.grid.cursor.should_wrap; + + let (base_col, spacer_col) = if should_wrap { + if cursor_col == 0 { + return; + } + (cursor_col - 1, cursor_col) + } else { + if cursor_col < 2 { + return; + } + (cursor_col - 2, cursor_col - 1) + }; + + if !matches!(self.grid[row][Column(base_col)].wide(), Wide::Wide) { return; } @@ -2682,6 +2877,7 @@ impl Handler for Crosswords { self.event_proxy .send_event(RioEvent::CursorBlinkingChange, self.window_id); } + NamedPrivateMode::GraphemeCluster => self.mode.insert(Mode::GRAPHEME_CLUSTER), NamedPrivateMode::SyncUpdate => (), } } @@ -2750,6 +2946,7 @@ impl Handler for Crosswords { self.event_proxy .send_event(RioEvent::CursorBlinkingChange, self.window_id); } + NamedPrivateMode::GraphemeCluster => self.mode.remove(Mode::GRAPHEME_CLUSTER), NamedPrivateMode::SyncUpdate => (), } } @@ -2799,6 +2996,13 @@ impl Handler for Crosswords { NamedPrivateMode::BracketedPaste => { self.mode.contains(Mode::BRACKETED_PASTE).into() } + NamedPrivateMode::GraphemeCluster => { + if self.mode.contains(Mode::GRAPHEME_CLUSTER) { + ModeState::Set + } else { + ModeState::Reset + } + } NamedPrivateMode::SyncUpdate => ModeState::Reset, NamedPrivateMode::ColumnMode => ModeState::NotSupported, }, @@ -3343,8 +3547,35 @@ impl Handler for Crosswords { None => return, }; + // Mode 2027: try to continue the previous cell's grapheme + // cluster first. On a break (or nothing to continue) fall + // through to the legacy paths: zero-width codepoints attach + // wcwidth-style, everything else writes a fresh cell. + // + // Codepoints <= 0xFF never continue a cluster (matching + // ghostty; the only UAX29 rule this waives is Prepend x + // Latin-1, which the bulk ASCII writer already waives; this + // keeps scalar and bulk agreeing regardless of how the parser + // chunks the stream) and are the common case, so that check + // goes first. + if c > '\u{FF}' + && self.mode.contains(Mode::GRAPHEME_CLUSTER) + && self.try_cluster_append(c, width) + { + return; + } + // Handle zero-width characters. if width == 0 { + // Mode 2027: the cluster path above is the only legitimate + // attach. Reaching here means there was no base to join: + // a column-0 orphan, an empty or bg-only previous cell. + // Attaching wcwidth-style would contradict the boundary + // the mode just computed, so drop the codepoint (matching + // ghostty). + if self.mode.contains(Mode::GRAPHEME_CLUSTER) { + return; + } // Emoji presentation variation selectors flip the *width* of // the preceding cell before being attached as combining data. // Matches kitty/ghostty; see emoji-variation-sequences.txt. @@ -3357,48 +3588,7 @@ impl Handler for Crosswords { _ => {} } - let mut column = self.grid.cursor.pos.col; - if !self.grid.cursor.should_wrap { - column.0 = column.saturating_sub(1); - } - - let row = self.grid.cursor.pos.row; - if matches!(self.grid[row][column].wide(), Wide::Spacer) { - column.0 = column.saturating_sub(1); - } - - // A bg-only cell reuses the extras-id bits for its color - // channel: reading them as an id would clone an unrelated - // slot onto this cell, and writing one back would corrupt - // the color. A combining mark with no base character has - // nothing to attach to — drop it. - if !matches!( - self.grid[row][column].content_tag(), - crate::crosswords::square::ContentTag::Codepoint - ) { - return; - } - // Copy-on-write: slots are interned and shared — every - // cell written under one OSC 8 template references the - // same slot, so pushing into it in place would attach the - // mark to the whole hyperlink span. Clone, extend, and - // re-intern instead; the marked cell gets its own - // (marks + hyperlink) slot and its neighbors keep theirs. - let existing_id = self.grid[row][column].extras_id(); - let mut extras = existing_id - .and_then(|id| self.grid.extras_table.get(id).cloned()) - .unwrap_or_default(); - extras.zerowidth.push(c); - let id = self.grid.alloc_extras(extras); - if id != 0 { - let cell = &mut self.grid[row][column]; - cell.set_extras_id(Some(id)); - cell.insert_cell_flag(CellFlags::GRAPHEME); - self.grid[row].has_extras = true; - } - // id == 0: slot space exhausted. Keep the cell's existing - // extras (hyperlink, earlier marks) rather than erasing - // them; only the new mark is lost. + self.attach_to_prev_cell(c); return; } @@ -3471,6 +3661,13 @@ impl Handler for Crosswords { let active = self.grid.cursor.charsets[self.active_charset]; if self.mode.contains(Mode::INSERT) || active != crate::crosswords::pos::StandardCharset::Ascii + // Grapheme clustering decides cell layout per codepoint + // against the previous cell; the bulk writers place whole + // width-runs at once and would split clusters. ASCII bulk + // paths stay fast: ASCII never continues a cluster, and + // the cluster check anchors on the previous cell, so it + // self-invalidates across a bulk run. + || self.mode.contains(Mode::GRAPHEME_CLUSTER) { for &cp in codepoints { let c = char::from_u32(cp).unwrap_or('\u{FFFD}'); @@ -7755,6 +7952,234 @@ mod tests { /// re-interned into the alt grid's table: extras ids are /// table-local, and the seeded cursor template would otherwise /// stamp a foreign id onto every alt-screen cell. + fn term_2027(cols: usize, rows: usize) -> Crosswords { + use crate::ansi::mode::PrivateMode; + use crate::performer::handler::Handler; + let mut cw = new_term(cols, rows); + cw.set_private_mode(PrivateMode::new(2027)); + cw + } + + fn extras_of(cw: &Crosswords, line: i32, col: usize) -> Vec { + cw.grid[Line(line)][Column(col)] + .extras_id() + .and_then(|id| cw.grid.extras_table.get(id)) + .map(|e| e.zerowidth.clone()) + .unwrap_or_default() + } + + /// ZWJ emoji occupy one two-cell slot under mode 2027 (GB11), + /// and shatter into per-emoji cells without it. + #[test] + fn mode_2027_zwj_emoji_is_one_cluster() { + use crate::performer::handler::Handler; + let farmer = ['\u{1F9D1}', '\u{200D}', '\u{1F33E}']; + + let mut cw = term_2027(10, 2); + for c in farmer { + cw.input(c); + } + assert_eq!(cw.grid[Line(0)][Column(0)].c(), '\u{1F9D1}'); + assert_eq!(cw.grid[Line(0)][Column(0)].wide(), Wide::Wide); + assert_eq!(cw.grid[Line(0)][Column(1)].wide(), Wide::Spacer); + assert_eq!(extras_of(&cw, 0, 0), ['\u{200D}', '\u{1F33E}']); + assert_eq!(cw.grid.cursor.pos.col, Column(2)); + + // Legacy: the trailing emoji starts its own pair. + let mut cw = new_term(10, 2); + for c in farmer { + cw.input(c); + } + assert_eq!(cw.grid[Line(0)][Column(2)].c(), '\u{1F33E}'); + assert_eq!(cw.grid.cursor.pos.col, Column(4)); + } + + /// Regional indicators pair up (GB12/13): two RIs form one wide + /// cluster, the third starts a new cell. + #[test] + fn mode_2027_regional_indicators_pair() { + use crate::performer::handler::Handler; + let mut cw = term_2027(10, 2); + let ri = ['\u{1F1E7}', '\u{1F1F7}', '\u{1F1E6}']; + for c in ri { + cw.input(c); + } + // First pair: one wide cluster. + assert_eq!(cw.grid[Line(0)][Column(0)].wide(), Wide::Wide); + assert_eq!(extras_of(&cw, 0, 0), ['\u{1F1F7}']); + // Third RI: fresh narrow cell (parity even at the boundary). + assert_eq!(cw.grid[Line(0)][Column(2)].c(), '\u{1F1E6}'); + assert_eq!(cw.grid[Line(0)][Column(2)].wide(), Wide::Narrow); + } + + /// Indic conjuncts join across the linker (GB9c) and the cluster + /// goes wide the moment a second width-bearing codepoint joins + /// (the ghostty width rule). + #[test] + fn mode_2027_indic_conjunct_joins() { + use crate::performer::handler::Handler; + let mut cw = term_2027(10, 2); + for c in ['\u{915}', '\u{94D}', '\u{937}'] { + cw.input(c); + } + assert_eq!(cw.grid[Line(0)][Column(0)].c(), '\u{915}'); + assert_eq!(cw.grid[Line(0)][Column(0)].wide(), Wide::Wide); + assert_eq!(extras_of(&cw, 0, 0), ['\u{94D}', '\u{937}']); + assert_eq!(cw.grid.cursor.pos.col, Column(2)); + } + + /// A valid emoji variation sequence widens within the cluster; an + /// invalid selector is ignored outright (ghostty `.ignore`) rather + /// than attached like the legacy path does. + #[test] + fn mode_2027_variation_selectors() { + use crate::performer::handler::Handler; + let mut cw = term_2027(10, 2); + cw.input('\u{1F39F}'); // text-presentation default, narrow + assert_eq!(cw.grid[Line(0)][Column(0)].wide(), Wide::Narrow); + cw.input('\u{FE0F}'); + assert_eq!(cw.grid[Line(0)][Column(0)].wide(), Wide::Wide); + assert_eq!(extras_of(&cw, 0, 0), ['\u{FE0F}']); + + let mut cw = term_2027(10, 2); + cw.input('a'); + cw.input('\u{FE0F}'); // 'a' is no emoji base: dropped entirely + assert_eq!(extras_of(&cw, 0, 0), Vec::::new()); + assert!(!cw.grid[Line(0)][Column(0)].has_grapheme()); + assert_eq!(cw.grid.cursor.pos.col, Column(1)); + } + + /// A skin-tone modifier (width-bearing Extend) joins its emoji + /// without growing the already-wide cluster. + #[test] + fn mode_2027_skin_tone_joins() { + use crate::performer::handler::Handler; + let mut cw = term_2027(10, 2); + cw.input('\u{1F44B}'); + cw.input('\u{1F3FB}'); + assert_eq!(cw.grid[Line(0)][Column(0)].wide(), Wide::Wide); + assert_eq!(extras_of(&cw, 0, 0), ['\u{1F3FB}']); + assert_eq!(cw.grid.cursor.pos.col, Column(2)); + } + + /// Plain combining marks behave as before: joined, still narrow. + /// Plain text never joins anything. + #[test] + fn mode_2027_narrow_cases() { + use crate::performer::handler::Handler; + let mut cw = term_2027(10, 2); + cw.input('e'); + cw.input('\u{301}'); + cw.input('x'); + assert_eq!(cw.grid[Line(0)][Column(0)].wide(), Wide::Narrow); + assert_eq!(extras_of(&cw, 0, 0), ['\u{301}']); + assert_eq!(cw.grid[Line(0)][Column(1)].c(), 'x'); + } + + /// A width-bearing continuation arriving when the narrow cluster + /// sits at the last column widens it through the wrap dance: + /// LeadingSpacer stays behind, the cluster moves to the next row. + #[test] + fn mode_2027_cluster_widens_across_row_edge() { + use crate::performer::handler::Handler; + let mut cw = term_2027(3, 3); + cw.input('a'); + cw.input('a'); + cw.input('\u{1F1E7}'); // narrow RI at the last column + cw.input('\u{1F1F7}'); // pairs: cluster must go wide → wraps + assert_eq!(cw.grid[Line(0)][Column(2)].wide(), Wide::LeadingSpacer); + assert_eq!(cw.grid[Line(1)][Column(0)].c(), '\u{1F1E7}'); + assert_eq!(cw.grid[Line(1)][Column(0)].wide(), Wide::Wide); + assert_eq!(extras_of(&cw, 1, 0), ['\u{1F1F7}']); + } + + /// The bulk decode path must route through the cluster machine: + /// a ZWJ sequence arriving as one codepoint batch clusters the + /// same as scalar input. + #[test] + fn mode_2027_bulk_codepoints_cluster() { + use crate::performer::handler::Handler; + let mut cw = term_2027(10, 2); + cw.input_codepoints(&[0x1F9D1, 0x200D, 0x1F33E, 0x61]); + assert_eq!(cw.grid[Line(0)][Column(0)].wide(), Wide::Wide); + assert_eq!(extras_of(&cw, 0, 0), ['\u{200D}', '\u{1F33E}']); + assert_eq!(cw.grid[Line(0)][Column(2)].c(), 'a'); + assert_eq!(cw.grid.cursor.pos.col, Column(3)); + } + + /// Reflow moves a cluster atomically: the wide pair and its + /// attached codepoints survive shrink-rewrap and grow-unwrap. + #[test] + fn mode_2027_cluster_survives_reflow() { + use crate::performer::handler::Handler; + let mut cw = term_2027(6, 3); + cw.input('a'); + cw.input('b'); + for c in ['\u{1F9D1}', '\u{200D}', '\u{1F33E}'] { + cw.input(c); + } + + // Shrink: the cluster no longer fits after "ab" and rewraps. + cw.resize(CrosswordsSize::new(3, 3)); + let mut found = None; + for line in cw.grid.topmost_line().0..3 { + for col in 0..3 { + let cell = cw.grid[Line(line)][Column(col)]; + if cell.c() == '\u{1F9D1}' { + found = Some((line, col)); + } + } + } + let (line, col) = found.expect("cluster base survives shrink"); + assert_eq!(cw.grid[Line(line)][Column(col)].wide(), Wide::Wide); + assert_eq!(extras_of(&cw, line, col), ['\u{200D}', '\u{1F33E}']); + + // Grow back: still one intact cluster. + cw.resize(CrosswordsSize::new(6, 3)); + let mut found = None; + for line in cw.grid.topmost_line().0..3 { + for col in 0..6 { + let cell = cw.grid[Line(line)][Column(col)]; + if cell.c() == '\u{1F9D1}' { + found = Some((line, col)); + } + } + } + let (line, col) = found.expect("cluster base survives grow"); + assert_eq!(cw.grid[Line(line)][Column(col)].wide(), Wide::Wide); + assert_eq!(extras_of(&cw, line, col), ['\u{200D}', '\u{1F33E}']); + + // The moved cells' rows must keep the extras hint, or reclaim + // would sweep the live slot out from under them. + cw.grid.reclaim_extras(); + assert_eq!(extras_of(&cw, line, col), ['\u{200D}', '\u{1F33E}']); + } + + /// Copy/serialize emits the full cluster text, and replaying that + /// text into a fresh mode-2027 terminal reproduces the same cells. + #[test] + fn mode_2027_cluster_round_trips_through_text() { + use crate::performer::handler::Handler; + let mut cw = term_2027(10, 2); + cw.input('a'); + for c in ['\u{1F9D1}', '\u{200D}', '\u{1F33E}'] { + cw.input(c); + } + cw.input('b'); + + let text = cw + .bounds_to_string(Pos::new(Line(0), Column(0)), Pos::new(Line(0), Column(4))); + assert_eq!(text, "a\u{1F9D1}\u{200D}\u{1F33E}b"); + + let mut replay = term_2027(10, 2); + for c in text.chars() { + replay.input(c); + } + assert_eq!(replay.grid[Line(0)][Column(1)].wide(), Wide::Wide); + assert_eq!(extras_of(&replay, 0, 1), ['\u{200D}', '\u{1F33E}']); + assert_eq!(replay.grid[Line(0)][Column(3)].c(), 'b'); + } + #[test] fn hyperlink_crosses_alt_screen_into_the_alt_table() { use crate::performer::handler::Handler; @@ -7767,6 +8192,84 @@ mod tests { cw.swap_alt(); } + /// Mode 2027: a zero-width codepoint with no base to join (orphan + /// at column 0) is dropped, never legacy-attached: the mode just + /// computed a boundary and wcwidth-attaching would contradict it + /// (ghostty Terminal.zig, print width==0 branch). + #[test] + fn mode_2027_orphan_zero_width_dropped() { + use crate::ansi::mode::PrivateMode; + use crate::performer::handler::Handler; + let mut cw = new_term(6, 3); + cw.set_private_mode(PrivateMode::new(2027)); + cw.input('\u{0301}'); + let cell = cw.grid[Line(0)][Column(0)]; + assert_eq!(cell.c(), '\0'); + assert!(cell.extras_id().is_none()); + assert_eq!(cw.grid.cursor.pos.col, Column(0)); + } + + /// Mode 2027: variation-selector validity is judged against the + /// cluster's *last* codepoint, not its base: a selector modifies + /// the character immediately before it (ghostty checks `prev` in + /// `graphemeWidthEffect`). U+261D is a valid VS16 base but the + /// skin tone that joined after it is not, so the trailing VS16 is + /// ignored outright. + #[test] + fn mode_2027_vs_validated_against_last_codepoint() { + use crate::ansi::mode::PrivateMode; + use crate::performer::handler::Handler; + let mut cw = new_term(8, 3); + cw.set_private_mode(PrivateMode::new(2027)); + cw.input('\u{261D}'); // ☝ narrow, valid VS16 base + cw.input('\u{1F3FB}'); // skin tone: Extend, width 2 → cluster goes wide + let col_after_modifier = cw.grid.cursor.pos.col; + cw.input('\u{FE0F}'); // last cp is the skin tone → invalid → ignored + + let cell = cw.grid[Line(0)][Column(0)]; + assert_eq!(cell.c(), '\u{261D}'); + assert_eq!(cell.wide(), Wide::Wide); + let extras = cw.grid.extras_table.get(cell.extras_id().unwrap()).unwrap(); + assert_eq!(extras.zerowidth, vec!['\u{1F3FB}']); + assert_eq!(cw.grid.cursor.pos.col, col_after_modifier); + } + + /// DEC private mode 2027 (grapheme cluster processing): set, + /// reset, DECRQM visibility, RIS, and cross-screen sharing. The + /// mode is plumbing-complete here; segmentation behavior arrives + /// with the cluster input path. + #[test] + fn mode_2027_plumbing() { + use crate::ansi::mode::{NamedPrivateMode, PrivateMode}; + use crate::performer::handler::Handler; + let mut cw = new_term(6, 3); + + // Default: reset. + assert!(!cw.mode().contains(Mode::GRAPHEME_CLUSTER)); + + // DECSET / DECRST round-trip. + cw.set_private_mode(PrivateMode::new(2027)); + assert!(cw.mode().contains(Mode::GRAPHEME_CLUSTER)); + cw.unset_private_mode(PrivateMode::new(2027)); + assert!(!cw.mode().contains(Mode::GRAPHEME_CLUSTER)); + + // 2027 resolves to the named mode, not Unknown. + assert!(matches!( + PrivateMode::new(2027), + PrivateMode::Named(NamedPrivateMode::GraphemeCluster) + )); + + // Shared across screens: the mode lives on the terminal. + cw.set_private_mode(PrivateMode::new(2027)); + cw.swap_alt(); + assert!(cw.mode().contains(Mode::GRAPHEME_CLUSTER)); + cw.swap_alt(); + + // RIS resets to default (off). + cw.reset_state(); + assert!(!cw.mode().contains(Mode::GRAPHEME_CLUSTER)); + } + /// Identical extras content interns into one slot: the u16 id /// space stops being a per-occurrence budget. #[test] diff --git a/rio-vt/src/crosswords/search.rs b/rio-vt/src/crosswords/search.rs index 82c1039e99..6d0ca6cc53 100644 --- a/rio-vt/src/crosswords/search.rs +++ b/rio-vt/src/crosswords/search.rs @@ -331,7 +331,9 @@ impl Crosswords { let mut cell = iter.square(); self.skip_fullwidth(&mut iter, &mut cell, regex.direction); - let mut c = cell.c(); + let mut cell_chars: Vec = Vec::with_capacity(4); + self.cell_search_chars(cell, regex.direction, &mut cell_chars); + let mut char_idx = 0; let mut last_wrapped = iter.square().wrapline(); let mut point = iter.pos(); @@ -348,58 +350,67 @@ impl Crosswords { } 'outer: loop { - // Convert char to array of bytes. - let mut buf = [0; 4]; - let utf8_len = c.encode_utf8(&mut buf).len(); - - // Pass char to DFA as individual bytes. - for i in 0..utf8_len { - // Inverse byte order when going left. - let byte = match regex.direction { - Direction::Right => buf[i], - Direction::Left => buf[utf8_len - i - 1], - }; - - state = regex.dfa.next_state(&mut regex.cache, state, byte)?; - consumed_bytes += 1; - - if i == 0 && state.is_match() { - // Matches require one additional BYTE of lookahead, so we check the match state - // for the first byte of every new character to determine if the last character - // was a match. - regex_match = Some(last_point); - } else if state.is_dead() { - if consumed_bytes == 2 { - // Reset search if we found an empty match. - // - // With an unanchored search, a dead state only occurs after the end of a - // match has been found. While we want to abort after the first match has - // ended, we don't want empty matches since we cannot highlight them. - // - // So once we encounter an empty match, we reset our parser state and clear - // the match, effectively starting a new search one character farther than - // before. - // - // An empty match requires consuming `2` bytes, since the first byte will - // report the match for the empty string, while the second byte then - // reports the dead state indicating the first character isn't part of the - // match. - reset_state!(); - - // Retry this character if first byte caused failure. - // - // After finding an empty match, we want to advance the search start by one - // character. So if the first character has multiple bytes and the dead - // state isn't reached at `i == 0`, then we continue with the rest of the - // loop to advance the parser by one character. - if i == 0 { - continue 'outer; + // Feed every char the cell carries (the base plus its + // attached cluster codepoints) so a search for composed + // text (a decomposed `é`, a ZWJ emoji) can match. Match + // positions stay cell-granular: the whole cluster is one + // grid position. + 'chars: while char_idx < cell_chars.len() { + let c = cell_chars[char_idx]; + // Convert char to array of bytes. + let mut buf = [0; 4]; + let utf8_len = c.encode_utf8(&mut buf).len(); + + // Pass char to DFA as individual bytes. + for i in 0..utf8_len { + // Inverse byte order when going left. + let byte = match regex.direction { + Direction::Right => buf[i], + Direction::Left => buf[utf8_len - i - 1], + }; + + state = regex.dfa.next_state(&mut regex.cache, state, byte)?; + consumed_bytes += 1; + + if i == 0 && state.is_match() { + // Matches require one additional BYTE of lookahead: the match state + // for the first byte of every new character tells whether the last + // character completed a match. + regex_match = Some(last_point); + } else if state.is_dead() { + if consumed_bytes == 2 { + // Reset search if we found an empty match. + // + // With an unanchored search, a dead state only occurs after the end of a + // match has been found. While we want to abort after the first match has + // ended, we don't want empty matches since we cannot highlight them. + // + // So once we encounter an empty match, we reset our parser state and clear + // the match, effectively starting a new search one character farther than + // before. + // + // An empty match requires consuming `2` bytes, since the first byte will + // report the match for the empty string, while the second byte then + // reports the dead state indicating the first character isn't part of the + // match. + reset_state!(); + + // Retry this character if first byte caused failure. + // + // After finding an empty match, we want to advance the search start by one + // character. So if the first character has multiple bytes and the dead + // state isn't reached at `i == 0`, then we continue with the rest of the + // loop to advance the parser by one character. + if i == 0 { + continue 'chars; + } + } else { + // Abort on dead state. + break 'outer; } - } else { - // Abort on dead state. - break 'outer; } } + char_idx += 1; } // Stop once we've reached the target point. @@ -434,7 +445,8 @@ impl Crosswords { self.skip_fullwidth(&mut iter, &mut cell, regex.direction); - c = cell.c(); + self.cell_search_chars(cell, regex.direction, &mut cell_chars); + char_idx = 0; let wrapped = iter.square().wrapline(); last_point = mem::replace(&mut point, iter.pos()); @@ -468,6 +480,38 @@ impl Crosswords { Ok(regex_match) } + /// The characters a cell contributes to a regex search: its base + /// codepoint plus attached cluster codepoints, reversed for a + /// leftward search (the byte loop reverses within each char). A + /// bg-only cell stores color where the codepoint lives and reads + /// as a blank, not as the color bits decoded into a char. + fn cell_search_chars( + &self, + cell: &Square, + direction: Direction, + out: &mut Vec, + ) { + out.clear(); + if !matches!( + cell.content_tag(), + crate::crosswords::square::ContentTag::Codepoint + ) { + out.push(' '); + return; + } + out.push(cell.c()); + if cell.has_grapheme() { + if let Some(id) = cell.extras_id() { + if let Some(extras) = self.grid.extras_table.get(id) { + out.extend(extras.zerowidth.iter().copied()); + } + } + } + if direction == Direction::Left { + out.reverse(); + } + } + /// Advance a grid iterator over fullwidth characters. fn skip_fullwidth<'a>( &self, @@ -1408,6 +1452,69 @@ mod tests { assert_eq!(end, Pos::new(Line(0), Column(6))); } + /// Attached cluster codepoints feed the DFA, so a search for + /// composed text can match cells built from marks. + #[test] + fn search_matches_attached_marks() { + use crate::performer::handler::Handler; + let window_id = crate::event::WindowId::from(0); + let size = CrosswordsSize::new(8, 2); + let mut term = + Crosswords::new(size, CursorShape::Block, VoidListener {}, window_id, 0, 0); + for c in ['x', 'e', '\u{301}', 'y'] { + term.input(c); + } + + // Decomposed é, matched both ways. + let mut regex = RegexSearch::new("e\u{301}").unwrap(); + let start = Pos::new(Line(0), Column(0)); + let end = Pos::new(Line(0), Column(7)); + assert_eq!( + term.regex_search_right(&mut regex, start, end), + Some(Pos::new(Line(0), Column(1))..=Pos::new(Line(0), Column(1))) + ); + let mut regex = RegexSearch::new("e\u{301}").unwrap(); + assert_eq!( + term.regex_search_left(&mut regex, end, start), + Some(Pos::new(Line(0), Column(1))..=Pos::new(Line(0), Column(1))) + ); + + // Spanning into the next plain char: marks sit between base + // and neighbor in the fed stream. + let mut regex = RegexSearch::new("e\u{301}y").unwrap(); + assert_eq!( + term.regex_search_right(&mut regex, start, end), + Some(Pos::new(Line(0), Column(1))..=Pos::new(Line(0), Column(2))) + ); + } + + /// A mode-2027 ZWJ cluster is searchable by its full sequence and + /// matches as one cell-granular position. + #[test] + fn search_matches_zwj_cluster() { + use crate::ansi::mode::PrivateMode; + use crate::performer::handler::Handler; + let window_id = crate::event::WindowId::from(0); + let size = CrosswordsSize::new(8, 2); + let mut term = + Crosswords::new(size, CursorShape::Block, VoidListener {}, window_id, 0, 0); + term.set_private_mode(PrivateMode::new(2027)); + for c in ['a', '\u{1F9D1}', '\u{200D}', '\u{1F33E}', 'b'] { + term.input(c); + } + + let farmer = "\u{1F9D1}\u{200D}\u{1F33E}"; + let mut regex = RegexSearch::new(farmer).unwrap(); + let start = Pos::new(Line(0), Column(0)); + let end = Pos::new(Line(0), Column(7)); + // The cluster occupies cells 1-2 (wide pair): the match spans + // the pair. + assert_eq!( + term.regex_search_right(&mut regex, start, end), + Some(Pos::new(Line(0), Column(1))..=Pos::new(Line(0), Column(2))) + ); + } + #[test] fn fullwidth_across_lines() { let term = mock_term("a🦇\n🦇b"); diff --git a/rio-vt/src/grapheme_lut.rs b/rio-vt/src/grapheme_lut.rs new file mode 100644 index 0000000000..a5404bc1a0 --- /dev/null +++ b/rio-vt/src/grapheme_lut.rs @@ -0,0 +1,131 @@ +//! Flat lookup tables for mode-2027 grapheme clustering. +//! +//! The ghostty devlog-006 treatment: the segmentation hot path must +//! not evaluate rule chains or binary-search property ranges per +//! codepoint. Two tables replace both: +//! +//! - a flat class table over `U+0000..U+20000` (the same BMP+plane-1 +//! window as [`codepoint_width`](crate::codepoint_width), covering +//! CJK and emoji), falling back to the range search above it; +//! - a transition table folding the whole break decision into one +//! index: `(prev_class, next_class, state) → (break?, next_state)`. +//! +//! Both are built at first use *by driving rio-unicode's reference +//! implementation*, so they are correct by construction and can never +//! drift from the conformance-tested rules. + +use std::sync::OnceLock; +use unicode_width::grapheme::{grapheme_class, is_break, BreakState, GraphemeClass}; + +const TABLE_LEN: usize = 0x2_0000; + +static CLASS_TABLE: OnceLock> = OnceLock::new(); +static TRANSITIONS: OnceLock> = OnceLock::new(); + +const CLASSES: usize = GraphemeClass::COUNT; +const STATES: usize = BreakState::COUNT; +/// Transition entry: bit 7 = break, bits 0..6 = packed next state. +const BREAK_BIT: u8 = 0x80; + +fn class_table() -> &'static [u8] { + CLASS_TABLE.get_or_init(|| { + let mut table = vec![0u8; TABLE_LEN].into_boxed_slice(); + for (cp, slot) in table.iter_mut().enumerate() { + if let Some(c) = char::from_u32(cp as u32) { + *slot = grapheme_class(c) as u8; + } + } + table + }) +} + +fn transitions() -> &'static [u8] { + TRANSITIONS.get_or_init(|| { + let mut table = vec![0u8; CLASSES * CLASSES * STATES].into_boxed_slice(); + for prev in 0..CLASSES { + let prev_class = GraphemeClass::from_u8(prev as u8).unwrap(); + for next in 0..CLASSES { + let next_class = GraphemeClass::from_u8(next as u8).unwrap(); + for state in 0..STATES { + let mut break_state = BreakState::unpack(state as u8).unwrap(); + let breaks = is_break(prev_class, next_class, &mut break_state); + table[(prev * CLASSES + next) * STATES + state] = + break_state.pack() | if breaks { BREAK_BIT } else { 0 }; + } + } + } + table + }) +} + +/// The grapheme class of `c` as a table index, one lookup for the +/// common window. +#[inline] +pub fn class_of(c: char) -> u8 { + let cp = c as usize; + if cp < TABLE_LEN { + class_table()[cp] + } else { + grapheme_class(c) as u8 + } +} + +/// One-index break decision: whether a boundary falls between a +/// codepoint of class `prev` and one of class `next` given the packed +/// `state`, which is advanced past `next` in place. +#[inline] +pub fn is_break_lut(prev: u8, next: u8, state: &mut u8) -> bool { + let entry = transitions() + [(prev as usize * CLASSES + next as usize) * STATES + *state as usize]; + *state = entry & !BREAK_BIT; + entry & BREAK_BIT != 0 +} + +/// Packed state after the first codepoint of a sequence +/// (`BreakState::start` in table form): the advance half of any +/// transition depends only on the incoming class and prior state. +#[inline] +pub fn start_state(first: u8) -> u8 { + let entry = transitions()[(first as usize) * STATES]; // prev=0, state=0 + entry & !BREAK_BIT +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every table entry must agree with the reference implementation + /// it was built from, including the class table's fallback seam. + #[test] + fn tables_match_reference() { + for cp in (0..TABLE_LEN as u32 + 0x100).step_by(7) { + if let Some(c) = char::from_u32(cp) { + assert_eq!(class_of(c), grapheme_class(c) as u8, "class of U+{cp:04X}"); + } + } + for prev in 0..CLASSES as u8 { + for next in 0..CLASSES as u8 { + for state in 0..STATES as u8 { + let mut reference = BreakState::unpack(state).unwrap(); + let expected = is_break( + GraphemeClass::from_u8(prev).unwrap(), + GraphemeClass::from_u8(next).unwrap(), + &mut reference, + ); + let mut packed = state; + let got = is_break_lut(prev, next, &mut packed); + assert_eq!(got, expected, "break ({prev},{next},{state})"); + assert_eq!(packed, reference.pack(), "state ({prev},{next},{state})"); + } + } + } + } + + #[test] + fn start_state_matches_reference() { + for class in 0..CLASSES as u8 { + let reference = BreakState::start(GraphemeClass::from_u8(class).unwrap()); + assert_eq!(start_state(class), reference.pack(), "start({class})"); + } + } +} diff --git a/rio-vt/src/lib.rs b/rio-vt/src/lib.rs index 589347735f..64a2856a01 100644 --- a/rio-vt/src/lib.rs +++ b/rio-vt/src/lib.rs @@ -20,6 +20,7 @@ pub mod config; pub mod crosswords; pub mod error; pub mod event; +pub mod grapheme_lut; pub mod performer; pub mod selection; pub mod simd_base64;