Grapheme cluster processing (DEC private mode 2027)

librio-wasm/src/lib.rs

@@ -69,9 +69,16 @@ pub const COLOR_INDEXED: u32 = 1;
 pub const COLOR_RGB: u32 = 2;
 
 /// u32 words per cell in [`RioTerm::write_cells`]:
-/// `[codepoint | wide << 21, fg, bg, style_flags]`.
+/// `[codepoint | wide << 21 | flags, fg, bg, style_flags]`.
 pub const CELL_WORDS: usize = 4;
 
+/// Word-0 flag: the cell carries attached cluster codepoints
+/// (combining marks, or a mode-2027 grapheme cluster tail) beyond the
+/// base codepoint in bits 0..21. Fetch the full text with
+/// [`RioTerm::cluster_text`] and draw that instead of the base char;
+/// a renderer that ignores the bit simply keeps drawing bases.
+pub const CELL_HAS_CLUSTER: u32 = 1 << 23;
+
 enum Event {
     Output(Vec<u8>),
     Wakeup,
@@ -658,6 +665,16 @@ impl RioTerm {
     pub fn kitty_image_rgba(&self, image_id: u32, out: &mut [u8]) -> usize {
         self.state.kitty_image_rgba(image_id, out)
     }
+
+    /// The full text of a cell flagged [`CELL_HAS_CLUSTER`]: the base
+    /// codepoint followed by its attached cluster codepoints
+    /// (combining marks, or a mode-2027 grapheme cluster). Draw this
+    /// string in place of the base char so a ZWJ emoji or a
+    /// decomposed accent renders as the glyph the sequence means.
+    /// `undefined` for cells without attachments.
+    pub fn cluster_text(&self, line: usize, column: usize) -> Option<String> {
+        self.state.cell_cluster_text(line, column)
+    }
 }
 
 impl RioTerm {
@@ -667,8 +684,14 @@ impl RioTerm {
             match self.state.square(line, col) {
                 Some(square) => {
                     let style = self.state.style_of(square);
-                    out[base] =
-                        (square.c() as u32 & 0x1F_FFFF) | ((square.wide() as u32) << 21);
+                    let cluster = if self.state.cluster_of(square).is_some() {
+                        CELL_HAS_CLUSTER
+                    } else {
+                        0
+                    };
+                    out[base] = (square.c() as u32 & 0x1F_FFFF)
+                        | ((square.wide() as u32) << 21)
+                        | cluster;
                     out[base + 1] = pack_color(style.fg);
                     out[base + 2] = pack_color(style.bg);
                     out[base + 3] = style.flags.bits() as u32;

librio/include/librio.h

@@ -255,6 +255,21 @@ uint16_t rio_render_state_lines(const rio_render_state_t *state);
 uint16_t rio_render_state_columns(const rio_render_state_t *state);
 bool rio_render_state_row_dirty(const rio_render_state_t *state, uint16_t line);
 void rio_render_state_reset_dirty(rio_render_state_t *state);
+/* Set in rio_cell_s.style_flags when the cell carries attached cluster
+ * codepoints (combining marks, or a DEC-2027 grapheme cluster) beyond
+ * `codepoint`. Fetch the full text with rio_render_state_cell_cluster
+ * and draw that instead of the base char. StyleFlags proper occupies
+ * bits 0..10; this is bit 15. */
+#define RIO_CELL_HAS_CLUSTER (1u << 15)
+
+/* Write the full text of a cell (base codepoint plus attached cluster
+ * codepoints) as UTF-32 into `out` (capacity `cap` code units).
+ * Returns the total codepoint count, which may exceed `cap` (call
+ * again with a larger buffer); 0 for plain cells. */
+size_t rio_render_state_cell_cluster(const rio_render_state_t *state,
+                                     uint16_t line, uint16_t column,
+                                     uint32_t *out, size_t cap);
+
 rio_cell_s rio_render_state_cell(const rio_render_state_t *state, uint16_t line,
                                  uint16_t column);
 rio_cursor_s rio_render_state_cursor(const rio_render_state_t *state);

librio/src/capi.rs

@@ -941,11 +941,16 @@ pub unsafe extern "C" fn rio_render_state_cell(
             return empty;
         };
         let style = state.style_of(square);
+        let cluster = if state.cluster_of(square).is_some() {
+            RIO_CELL_HAS_CLUSTER
+        } else {
+            0
+        };
         rio_cell_s {
             codepoint: square.c() as u32,
             fg: color_to_c(style.fg),
             bg: color_to_c(style.bg),
-            style_flags: style.flags.bits(),
+            style_flags: style.flags.bits() | cluster,
         }
     }))
     .unwrap_or(rio_cell_s {
@@ -956,6 +961,51 @@ pub unsafe extern "C" fn rio_render_state_cell(
     })
 }
 
+/// Set in `rio_cell_s.style_flags` when the cell carries attached
+/// cluster codepoints (combining marks, or a mode-2027 grapheme
+/// cluster) beyond `codepoint`. Fetch the full text with
+/// [`rio_render_state_cell_cluster`] and draw that instead of the
+/// base char. StyleFlags proper occupies bits 0..10; this is bit 15.
+pub const RIO_CELL_HAS_CLUSTER: u16 = 1 << 15;
+
+/// Write the full text of a cell (base codepoint plus attached
+/// cluster codepoints) as UTF-32 into `out` (capacity `cap` code
+/// units). Returns the total codepoint count, which may exceed `cap`
+/// (call again with a larger buffer); 0 for plain cells, so callers
+/// can treat 0 as "draw `codepoint` as usual".
+#[no_mangle]
+pub unsafe extern "C" fn rio_render_state_cell_cluster(
+    state: *const RenderState,
+    line: u16,
+    column: u16,
+    out: *mut u32,
+    cap: usize,
+) -> usize {
+    catch_unwind(AssertUnwindSafe(|| {
+        if state.is_null() {
+            return 0;
+        }
+        let state = unsafe { &*state };
+        let Some(square) = state.square(line as usize, column as usize) else {
+            return 0;
+        };
+        let Some(cluster) = state.cluster_of(square) else {
+            return 0;
+        };
+        let total = 1 + cluster.len();
+        if !out.is_null() {
+            let write = total.min(cap);
+            let dst = unsafe { core::slice::from_raw_parts_mut(out, write) };
+            let mut src = core::iter::once(square.c()).chain(cluster.iter().copied());
+            for slot in dst.iter_mut() {
+                *slot = src.next().unwrap_or('\0') as u32;
+            }
+        }
+        total
+    }))
+    .unwrap_or(0)
+}
+
 /// Lines the view is scrolled up into history; 0 means the live screen.
 /// Renderers use this to hide the cursor while scrolled.
 #[no_mangle]

librio/src/lib.rs

@@ -1236,6 +1236,33 @@ mod tests {
         assert!(state.display_offset() > 0, "the view should have moved");
     }
 
+    // The wasm renderer draws a cell's full cluster text (base +
+    // attached codepoints) when the wire flag says one exists; the
+    // accessor is the source of that text.
+    #[test]
+    fn render_state_exposes_cluster_text() {
+        let engine = Engine::new(Arc::new(CountingDelegate {
+            wakeups: AtomicUsize::new(0),
+        }));
+        let surface = engine
+            .create_surface(&SurfaceDesc::default())
+            .expect("spawn shell");
+        let mut state = RenderState::new(&surface);
+
+        // Mode 2027 on, then a ZWJ emoji and a decomposed accent.
+        surface
+            .inject_output("\x1b[?2027h\u{1F9D1}\u{200D}\u{1F33E}e\u{301}x".as_bytes());
+        state.update();
+
+        assert_eq!(
+            state.cell_cluster_text(0, 0).as_deref(),
+            Some("\u{1F9D1}\u{200D}\u{1F33E}")
+        );
+        assert_eq!(state.cell_cluster_text(0, 2).as_deref(), Some("e\u{301}"));
+        // Plain cells report nothing.
+        assert_eq!(state.cell_cluster_text(0, 3), None);
+    }
+
     // SGR is the modern form; the X10 fallback offsets by 32.
     #[test]
     fn mouse_reports_encode_both_forms() {

librio/src/render_state.rs

@@ -206,6 +206,38 @@ impl RenderState {
         }
     }
 
+    /// The codepoints attached to a square's grapheme cluster
+    /// (combining marks, or a full mode-2027 cluster tail) when it
+    /// carries any. Bg-only squares reuse the extras-id bits for
+    /// color and never report a cluster.
+    pub fn cluster_of(&self, square: &Square) -> Option<&[char]> {
+        if !square.has_grapheme()
+            || !matches!(
+                square.content_tag(),
+                rio_vt::crosswords::square::ContentTag::Codepoint
+            )
+        {
+            return None;
+        }
+        let extras = square.extras_id().and_then(|eid| self.extras.get(&eid))?;
+        if extras.zerowidth.is_empty() {
+            None
+        } else {
+            Some(&extras.zerowidth)
+        }
+    }
+
+    /// The cell's full text (base codepoint plus attached cluster
+    /// codepoints), or `None` for cells with no attachments.
+    pub fn cell_cluster_text(&self, line: usize, column: usize) -> Option<String> {
+        let square = self.square(line, column)?;
+        let cluster = self.cluster_of(square)?;
+        let mut text = String::with_capacity(4 * (1 + cluster.len()));
+        text.push(square.c());
+        text.extend(cluster.iter());
+        Some(text)
+    }
+
     pub fn square(&self, line: usize, column: usize) -> Option<&Square> {
         let row = self.rows.get(line)?;
         if column >= self.columns {

rio-unicode/src/grapheme.rs

@@ -49,6 +49,38 @@ pub enum GraphemeClass {
     Consonant,
 }
 
+impl GraphemeClass {
+    /// Number of classes; `as u8` values are contiguous in
+    /// `0..COUNT`, so classes can index precomputed tables.
+    pub const COUNT: usize = 18;
+
+    /// Inverse of `as u8`, for table builders enumerating classes.
+    pub fn from_u8(value: u8) -> Option<Self> {
+        use GraphemeClass as G;
+        Some(match value {
+            0 => G::Other,
+            1 => G::CR,
+            2 => G::LF,
+            3 => G::Control,
+            4 => G::Extend,
+            5 => G::ExtendIncb,
+            6 => G::Linker,
+            7 => G::Zwj,
+            8 => G::RegionalIndicator,
+            9 => G::Prepend,
+            10 => G::SpacingMark,
+            11 => G::L,
+            12 => G::V,
+            13 => G::T,
+            14 => G::LV,
+            15 => G::LVT,
+            16 => G::ExtPic,
+            17 => G::Consonant,
+            _ => return None,
+        })
+    }
+}
+
 /// Class lookup: binary search over the generated ranges.
 pub fn grapheme_class(c: char) -> GraphemeClass {
     let cp = c as u32;
@@ -101,6 +133,45 @@ pub struct BreakState {
 }
 
 impl BreakState {
+    /// Number of distinct states; `pack` values are contiguous in
+    /// `0..COUNT`, so states can index precomputed transition tables.
+    pub const COUNT: usize = 18;
+
+    /// Pack into `0..COUNT` for table indexing.
+    pub fn pack(self) -> u8 {
+        let emoji = match self.emoji {
+            EmojiSeq::None => 0u8,
+            EmojiSeq::Emoji => 1,
+            EmojiSeq::EmojiZwj => 2,
+        };
+        let incb = match self.incb {
+            Incb::None => 0u8,
+            Incb::Consonant => 1,
+            Incb::LinkerSeen => 2,
+        };
+        emoji * 6 + incb * 2 + self.ri_odd as u8
+    }
+
+    /// Inverse of [`pack`](Self::pack).
+    pub fn unpack(value: u8) -> Option<Self> {
+        if value as usize >= Self::COUNT {
+            return None;
+        }
+        Some(Self {
+            emoji: match value / 6 {
+                0 => EmojiSeq::None,
+                1 => EmojiSeq::Emoji,
+                _ => EmojiSeq::EmojiZwj,
+            },
+            incb: match (value % 6) / 2 {
+                0 => Incb::None,
+                1 => Incb::Consonant,
+                _ => Incb::LinkerSeen,
+            },
+            ri_odd: value % 2 == 1,
+        })
+    }
+
     /// State after the first codepoint of a sequence.
     pub fn start(first: GraphemeClass) -> Self {
         let mut state = Self::default();

rio-vt/Cargo.toml

@@ -24,6 +24,8 @@ parking_lot = { workspace = true }
 serde = { workspace = true }
 rio-graphics = { workspace = true }
 rio-grapheme-width = { workspace = true }
+# The workspace aliases this to the rio-unicode crate, which also
+# carries the UAX #29 grapheme segmentation used by mode 2027.
 unicode-width = { workspace = true }
 regex-automata = "0.4.16"
 cursor-icon = { version = "1.1.0", default-features = false }

rio-vt/src/ansi/mode.rs

@@ -70,6 +70,7 @@ impl PrivateMode {
             1049 => Self::Named(NamedPrivateMode::SwapScreenAndSetRestoreCursor),
             2004 => Self::Named(NamedPrivateMode::BracketedPaste),
             2026 => Self::Named(NamedPrivateMode::SyncUpdate),
+            2027 => Self::Named(NamedPrivateMode::GraphemeCluster),
             _ => Self::Unknown(mode),
         }
     }
@@ -127,6 +128,14 @@ pub enum NamedPrivateMode {
     BracketedPaste = 2004,
     /// The mode is handled automatically by [`Processor`].
     SyncUpdate = 2026,
+    /// Grapheme cluster processing (contour's terminal-unicode-core).
+    ///
+    /// When set, incoming text is segmented into UAX #29 extended
+    /// grapheme clusters: a multi-codepoint cluster (a ZWJ emoji, a
+    /// conjunct) occupies one cell slot instead of one slot per
+    /// width-bearing codepoint. Queryable via DECRQM, which is how
+    /// applications detect support. Default: reset.
+    GraphemeCluster = 2027,
 }
 
 /// Mode for clearing line.