From cc95c63c011b83a21bbe8f047c806fe3298ea9ff Mon Sep 17 00:00:00 2001 From: Vyncint Ng <115854244+vyncint@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:26:10 +0700 Subject: [PATCH 1/2] fix(emu): measure text in grid columns, group SGR 58, honour ?7h - A new width module lays text out as vt100 0.18 does: each character its own width, a zero-width one joining the cell before it, and the presentation selector widening an emoji. Insert mode, Screen::locate, the diff frame and the text reader use it; each measured a run as a string, which a ZWJ family, lam-alef and a skin-toned emoji are not (#580). The reader falls back to one column per selector cell when a row only fits that way, which is how earlier releases wrote it. - The grid's colour normalizer turns a semicolon-form underline colour (58;5;n, 58;2;r;g;b) into one colon parameter, so vt100 skips it whole instead of taking its arguments for attributes (#579). - CSI ? 7 h asks for the only behaviour the emulator has and is no longer recorded as unsupported; CSI ? 7 l still is (#581). Signed-off-by: Vyncint Ng <115854244+vyncint@users.noreply.github.com> --- crates/termlens/src/emu/shadow.rs | 76 ++++++++++++++-- crates/termlens/src/emu/unhandled.rs | 20 ++++- crates/termlens/src/emu/vt100.rs | 126 +++++++++++++++++++++++++-- crates/termlens/src/lib.rs | 1 + crates/termlens/src/screen.rs | 7 +- crates/termlens/src/screen/diff.rs | 58 ++++++++++-- crates/termlens/src/screen/parse.rs | 74 ++++++++++++++-- crates/termlens/src/width.rs | 112 ++++++++++++++++++++++++ 8 files changed, 441 insertions(+), 33 deletions(-) create mode 100644 crates/termlens/src/width.rs diff --git a/crates/termlens/src/emu/shadow.rs b/crates/termlens/src/emu/shadow.rs index db355f3..144a1ec 100644 --- a/crates/termlens/src/emu/shadow.rs +++ b/crates/termlens/src/emu/shadow.rs @@ -58,9 +58,11 @@ enum State { Csi, } -/// Rewrites colon-form extended colours into the semicolon form understood by -/// `vt100`. Like [`AttrShadow`], it only holds a complete plain SGR and emits -/// every other sequence unchanged. +/// Rewrites the SGR colour forms `vt100` misreads into ones it reads right: +/// colon-form extended colours into the semicolon form it understands, and +/// a semicolon-form underline colour into one colon parameter it skips. +/// Like [`AttrShadow`], it only holds a complete plain SGR and emits every +/// other sequence unchanged. pub(super) struct ColorNormalizer { state: State, pending: Vec, @@ -79,7 +81,7 @@ impl ColorNormalizer { &mut self.state, &mut self.pending, bytes, - normalize_colon_colours, + normalize_for_grid, ) } } @@ -207,7 +209,18 @@ fn rewrite_stream( out } -fn normalize_colon_colours(seq: &[u8]) -> Option> { +/// One plain SGR in the form `vt100` reads right, or `None` to pass it +/// through unchanged. +/// +/// Two forms need it. Colon-form extended colours (`38:2::r:g:b`) become +/// the semicolon form. And an underline colour (`58`), which vt100 does not +/// model, in the semicolon form (`58;5;n`, `58;2;r;g;b`) becomes one colon +/// parameter (`58:5:n`): vt100 reads parameters one at a time, so it took +/// the arguments for attributes — `58;5;1` was blink and bold, and +/// `58;5;0` a reset. As one parameter it is skipped whole, and the +/// sequence is still recorded as one nobody honoured (#579). crossterm +/// writes the semicolon form for `SetUnderlineColor`. +fn normalize_for_grid(seq: &[u8]) -> Option> { let params = seq.get(2..seq.len().checked_sub(1)?)?; if params .iter() @@ -215,14 +228,37 @@ fn normalize_colon_colours(seq: &[u8]) -> Option> { { return None; } - if !params.contains(&b':') { + // The common case, checked without allocating: neither form is here. + if !params.contains(&b':') && !params.split(|&b| b == b';').any(|group| group == b"58") { return None; } let groups: Vec<&[u8]> = params.split(|&b| b == b';').collect(); let mut normalized = Vec::with_capacity(groups.len()); let mut changed = false; - for group in groups { + let mut next = 0; + while next < groups.len() { + let group = groups[next]; + next += 1; + if group == b"58" { + let arguments = match groups.get(next).copied() { + Some(b"5") => 2, + Some(b"2") => 4, + _ => 0, + }; + if arguments > 0 { + let end = (next + arguments).min(groups.len()); + let mut joined = group.to_vec(); + for argument in &groups[next..end] { + joined.push(b':'); + joined.extend_from_slice(argument); + } + normalized.push(joined); + next = end; + changed = true; + continue; + } + } let parts: Vec<&[u8]> = group.split(|&b| b == b':').collect(); let replacement = match parts.as_slice() { [first @ (b"38" | b"48"), b"2", b"", red, green, blue] @@ -396,7 +432,7 @@ mod tests { fn normalized(bytes: &[u8]) -> String { let mut state = State::Ground; let mut pending = Vec::new(); - let mut out = rewrite_stream(&mut state, &mut pending, bytes, normalize_colon_colours); + let mut out = rewrite_stream(&mut state, &mut pending, bytes, normalize_for_grid); // Anything still held at the end of the stream is not a complete SGR. out.append(&mut pending); String::from_utf8_lossy(&out).replace('\x1b', "E") @@ -526,6 +562,30 @@ mod tests { } } + #[test] + fn a_semicolon_underline_colour_becomes_one_colon_parameter() { + // vt100 reads parameters one at a time and has no SGR 58, so its + // arguments must not reach it as parameters of their own (#579). + assert_eq!(normalized(b"\x1b[58;5;1m"), "E[58:5:1m"); + assert_eq!(normalized(b"\x1b[4;58;2;0;1;7;31m"), "E[4;58:2:0:1:7;31m"); + // Truncated: what is there is still one parameter. + assert_eq!(normalized(b"\x1b[58;5m"), "E[58:5m"); + // Already one parameter, a 58 with no colour form after it, and a + // reset of the underline colour: nothing to group. + for seq in [ + &b"\x1b[58:5:1m"[..], + b"\x1b[58m", + b"\x1b[58;3m", + b"\x1b[59m", + b"\x1b[1;5m", + ] { + assert_eq!( + normalized(seq), + String::from_utf8_lossy(seq).replace('\x1b', "E") + ); + } + } + #[test] fn colon_colour_normalization_preserves_unrecognized_bytes() { assert_eq!(normalized(b"\x1b[38:2::10:20:30m"), "E[38;2;10;20;30m"); diff --git a/crates/termlens/src/emu/unhandled.rs b/crates/termlens/src/emu/unhandled.rs index 0937772..13b6ec7 100644 --- a/crates/termlens/src/emu/unhandled.rs +++ b/crates/termlens/src/emu/unhandled.rs @@ -102,6 +102,11 @@ fn params_text(params: &[&[u16]]) -> String { /// console's handshake in every Windows snapshot's list (#149). const TRACKED_PRIVATE_MODES: &[u16] = &[9, 1000, 1002, 1003, 1005, 1006, 1015, 1004, 2026, 9001]; +/// DECAWM, autowrap. Setting it asks for the only behaviour the emulator +/// has, so `CSI ? 7 h` — which ncurses sends at every start (`smam`) — is +/// honoured; resetting it is not modelled, and stays on record (#581). +const AUTOWRAP: u16 = 7; + /// The SGR parameters [`shadow`](super::shadow) recovers: blink, conceal, /// strikethrough and their three "off" forms. /// @@ -193,7 +198,7 @@ impl Callbacks for Unhandled { (Some(b'?'), None, 'h' | 'l') => params .iter() .flat_map(|p| p.iter()) - .all(|m| TRACKED_PRIVATE_MODES.contains(m)), + .all(|m| TRACKED_PRIVATE_MODES.contains(m) || (c == 'h' && *m == AUTOWRAP)), // An SGR every one of whose parameters the shadow recovers is // implemented, and naming it here would contradict the cell // (#320). One parameter outside the set and the whole sequence @@ -278,6 +283,19 @@ mod tests { .collect() } + #[test] + fn autowrap_on_is_honoured_and_autowrap_off_is_on_record() { + // Every ncurses program sends `CSI ? 7 h` at startup, and it asks + // for the only behaviour there is: naming it made an empty + // `unsupported()` impossible for any of them (#581). Turning wrap + // off is not modelled, so that still is named. + assert_eq!(shapes(b"\x1b[?7hready"), [] as [String; 0]); + assert_eq!(shapes(b"\x1b[?7l"), ["^[[?7l"]); + // In a list, the set is honoured with the others it is set with. + assert_eq!(shapes(b"\x1b[?7;1000h"), [] as [String; 0]); + assert_eq!(shapes(b"\x1b[?7;12h"), ["^[[?7;12h"]); + } + #[test] fn a_sequence_nobody_honours_is_recorded_once_in_its_written_form() { assert_eq!( diff --git a/crates/termlens/src/emu/vt100.rs b/crates/termlens/src/emu/vt100.rs index ebcbcb3..6164d06 100644 --- a/crates/termlens/src/emu/vt100.rs +++ b/crates/termlens/src/emu/vt100.rs @@ -7,8 +7,6 @@ use std::num::NonZeroU16; use std::sync::Arc; use std::time::{Duration, Instant}; -use unicode_width::UnicodeWidthStr; - use super::seq::{SeqEvent, SeqTracker, TabOp}; use super::shadow::{AttrShadow, ColorNormalizer}; use super::unhandled::Unhandled; @@ -381,10 +379,11 @@ impl Emulator for Vt100Emulator { for (i, &byte) in bytes.iter().enumerate() { // Insert mode: a printable run pushes the rest of the row right // instead of overwriting it. vt100 does not model IRM, so the - // room is reserved with the `ICH` it does dispatch, sized in - // columns — a wide character counts twice — and never past the - // right margin (#261). The bytes owed to the grid go first, so - // the cursor column the reservation is made at is current. + // room is reserved with the `ICH` it does dispatch, sized in the + // columns the grid will give the run — its own layout, which no + // `unicode-width` measure is (#580) — and never past the right + // margin (#261). The bytes owed to the grid go first, so the + // cursor column the reservation is made at is current. if i >= run_until && self.tracker.insert_mode() && !self.tracker.mid_sequence() @@ -395,7 +394,7 @@ impl Emulator for Vt100Emulator { .position(|&b| !is_printable(b)) .map_or(bytes.len(), |p| i + p); run_until = end; - let width = String::from_utf8_lossy(&bytes[i..end]).width(); + let width = crate::width::columns(&String::from_utf8_lossy(&bytes[i..end])); self.feed_staged(&bytes[fed..i]); fed = i; let screen = self.parser.screen(); @@ -760,6 +759,12 @@ mod tests { } } + fn emu_with_size(rows: u16, cols: u16, bytes: &[u8]) -> Vt100Emulator { + let mut emu = Vt100Emulator::new(rows, cols, 0, crate::graphics::DEFAULT_CAPTURE, false); + feed_all(&mut emu, bytes); + emu + } + fn emu_with(bytes: &[u8]) -> Vt100Emulator { let mut emu = Vt100Emulator::new(4, 10, 0, crate::graphics::DEFAULT_CAPTURE, false); feed_all(&mut emu, bytes); @@ -1363,6 +1368,113 @@ mod tests { assert_eq!(emu.snapshot().row_text(0), "\u{6f22}\u{6f22}\u{6f22} "); } + #[test] + fn the_width_model_is_the_grid_s_own_layout() { + // `width::columns` predicts how far the grid moves the cursor for a + // run of text; insert mode, `locate` and the diff all rely on that + // prediction. Held against vt100 itself, one sample at a time. + for sample in [ + "abc", + "\u{6f22}\u{5b57}x", + "e\u{301}x", + "\u{644}\u{627}", + "\u{1f468}\u{200d}\u{1f469}x", + "\u{26a0}\u{fe0f}x", + "\u{2714}\u{fe0f}\u{31}\u{fe0f}\u{20e3}x", + "\u{1f44d}\u{1f3fd}x", + "a\u{fe0f}b", + "\u{1f1fa}\u{1f1f8}x", + ] { + let emu = emu_with_size(2, 30, sample.as_bytes()); + let (_, col, _) = emu.snapshot().cursor(); + assert_eq!( + usize::from(col), + crate::width::columns(sample), + "{sample:?}" + ); + } + } + + #[test] + fn an_underline_colour_is_one_parameter_not_three_attributes() { + // vt100 does not model SGR 58 and read `58;5;n` a parameter at a + // time: `5` blink, then `n` as an attribute — `0` reset the cell, `1` + // made it bold (#579). crossterm writes exactly this form. + let emu = emu_with(b"\x1b[1;31mA\x1b[58;5;0mB\x1b[0m \x1b[58;5;1mC\x1b[0m"); + let s = emu.snapshot(); + let style = |col| *s.cell(0, col).unwrap().style(); + for col in [0, 1] { + assert_eq!(style(col).fg, Color::Indexed(1), "column {col}"); + assert!(style(col).bold, "column {col}"); + assert!(!style(col).blink, "column {col}"); + } + assert_eq!(style(3), Style::default(), "C asked only for a colour"); + // The truecolour form's arguments are 0, 1 and 7: a reset, bold and + // reverse if read as attributes. + let emu = emu_with(b"\x1b[4mx\x1b[58;2;0;1;7my"); + let y = *emu.snapshot().cell(0, 1).unwrap().style(); + assert!(y.underline && !y.bold && !y.reverse && !y.dim, "{y:?}"); + // Still a sequence nobody honoured, so it is still on record. + let shapes: Vec = emu + .snapshot() + .unsupported() + .iter() + .map(String::from) + .collect(); + assert_eq!(shapes, ["^[[58:2:0:1:7m"]); + } + + #[test] + fn insert_mode_reserves_the_columns_the_grid_gives_the_run() { + // The reservation was the run's string width, and the grid lays a + // ZWJ family out as two wide cells and lam-alef as two letters: + // the difference was overwritten instead of pushed right (#580). + for (run, expected) in [ + ( + "\u{1f468}\u{200d}\u{1f469}", + "\u{1f468}\u{200d}\u{1f469}ABCDEF", + ), + ("\u{644}\u{627}", "\u{644}\u{627}ABCDEFGH"), + ("\u{26a0}\u{fe0f}", "\u{26a0}\u{fe0f}ABCDEFGH"), + ] { + let stream = format!("ABCDEFGHIJ\r\x1b[4h{run}"); + let emu = emu_with_size(2, 10, stream.as_bytes()); + assert_eq!(emu.snapshot().row_text(0), expected, "{run:?}"); + } + } + + #[test] + fn locate_reports_the_column_the_grid_gave_a_history_row() { + // History is text; `locate` turns the text before a match into a + // column, and a column `scrollback_cell` can be asked about must be + // the grid's (#580). + let mut emu = Vt100Emulator::new(2, 20, 100, crate::graphics::DEFAULT_CAPTURE, true); + feed_all( + &mut emu, + "\u{1f468}\u{200d}\u{1f469} SECRET\r\nb\r\nc\r\n".as_bytes(), + ); + let s = emu.snapshot(); + assert_eq!( + s.locate("SECRET"), + Some(crate::screen::Location::History { row: 0, col: 5 }) + ); + assert_eq!(s.scrollback_cell(0, 5).map(Cell::contents), Some("S")); + } + + #[test] + fn a_selector_widened_emoji_survives_the_text_format() { + // The grid gives `⚠️` two columns; the saved text reads back with + // the same cells, so `find` agrees before and after (#580). + let emu = emu_with_size(2, 10, "\u{26a0}\u{fe0f}X \u{2714}\u{fe0f}Y".as_bytes()); + let live = emu.snapshot(); + let back = Screen::parse(&live.to_string()).unwrap(); + for col in 0..10 { + assert_eq!(back.cell(0, col), live.cell(0, col), "column {col}"); + } + assert_eq!(live.find("Y"), Some((0, 6))); + assert_eq!(back.find("Y"), Some((0, 6))); + } + #[test] fn history_rows_keep_the_width_they_were_captured_at() { // Documented decision (`Terminal::resize`): history is not reflowed diff --git a/crates/termlens/src/lib.rs b/crates/termlens/src/lib.rs index d2ae601..eb69cca 100644 --- a/crates/termlens/src/lib.rs +++ b/crates/termlens/src/lib.rs @@ -116,6 +116,7 @@ mod spawn; mod terminal; mod utf8; mod wait; +mod width; pub use error::{Error, Result}; #[cfg(feature = "decode")] diff --git a/crates/termlens/src/screen.rs b/crates/termlens/src/screen.rs index 50a7503..b13bda8 100644 --- a/crates/termlens/src/screen.rs +++ b/crates/termlens/src/screen.rs @@ -1194,6 +1194,10 @@ impl Screen { /// outside the screen. /// /// This is the fact [`logical_text`](Self::logical_text) joins rows on. + /// The backend records a wrap only where the last column held + /// something, so a wide character that moved to the next row because it + /// did not fit in an empty last column leaves the row `false` + /// (`docs/LIMITATIONS.md`). #[must_use] pub fn row_wrapped(&self, row: u16) -> bool { self.state @@ -1427,7 +1431,8 @@ impl Screen { let Some(byte_off) = folded.find(needle.as_str()) else { continue; }; - let col = unicode_width::UnicodeWidthStr::width(&folded[..byte_off]); + // The columns the grid gave the text before the match (#580). + let col = crate::width::columns(&folded[..byte_off]); return Some(Location::History { row, col: u16::try_from(col).unwrap_or(u16::MAX), diff --git a/crates/termlens/src/screen/diff.rs b/crates/termlens/src/screen/diff.rs index 9d75563..35b5f91 100644 --- a/crates/termlens/src/screen/diff.rs +++ b/crates/termlens/src/screen/diff.rs @@ -8,8 +8,6 @@ use std::fmt; -use unicode_width::UnicodeWidthStr; - use super::{same_cursor, Cell, Screen, Style}; /// The difference between two screens. Built by [`Screen::diff`]; render it @@ -255,12 +253,13 @@ fn cursor_text((row, col, visible): (u16, u16, bool)) -> String { } } -/// `text` followed by blanks out to `width` display columns, so the `│` -/// closing the row line lands in the same column as the one closing its -/// marker line. The `{: String { - let missing = width.saturating_sub(text.width()); + let missing = width.saturating_sub(crate::width::columns(text)); let mut out = String::with_capacity(text.len() + missing); out.push_str(text); out.extend(std::iter::repeat_n(' ', missing)); @@ -387,4 +386,49 @@ mod tests { ); } } + + /// The pad measured a row as a string, where a ZWJ family is one + /// two-column cluster and lam-alef one column — four and two on the + /// grid, which is where the markers are placed (#580). + #[test] + fn rows_the_string_measure_misjudges_keep_the_frame_on_grid_columns() { + let screen = |last: &str| { + let blank = || Cell::new(String::new(), Style::default(), false, false); + let cells = vec![ + Cell::new("\u{1f468}\u{200d}".into(), Style::default(), true, false), + Cell::new(String::new(), Style::default(), false, true), + Cell::new("\u{1f469}".into(), Style::default(), true, false), + Cell::new(String::new(), Style::default(), false, true), + Cell::new("\u{644}".into(), Style::default(), false, false), + Cell::new("\u{627}".into(), Style::default(), false, false), + Cell::new(last.into(), Style::default(), false, false), + blank(), + blank(), + blank(), + ]; + Screen::from_parts(10, 1, 0, 0, true, cells, TermState::default()) + }; + let rendered = screen("x").diff(&screen("y")).to_string(); + let mut lines = rendered.lines().skip(1); + let text = lines.next().expect("a row line"); + let marks = lines.next().expect("its marker line"); + let grid_columns = |line: &str| { + let mut columns = Vec::new(); + for (at, _) in line.match_indices('│') { + columns.push(crate::width::columns(&line[..at])); + } + columns + }; + assert_eq!( + grid_columns(text), + grid_columns(marks), + "the │ separators sit at different grid columns:\n{rendered}" + ); + assert_eq!( + marks + .find('^') + .map(|at| crate::width::columns(&marks[..at])), + Some(11) + ); + } } diff --git a/crates/termlens/src/screen/parse.rs b/crates/termlens/src/screen/parse.rs index 55f9cc3..05116aa 100644 --- a/crates/termlens/src/screen/parse.rs +++ b/crates/termlens/src/screen/parse.rs @@ -170,7 +170,28 @@ fn parse_header(line: &str) -> Result<(u16, u16, (u16, u16, bool))> { /// One grid line into `row`, a slice of exactly `cols` cells. A zero-width /// combining mark joins the cell before it; a wide character takes two; a /// control character is an error. +/// +/// An emoji-presentation selector (U+FE0F) makes the emoji it follows two +/// columns wide, as the emulator lays it out since 0.11.7 (vt100 0.18), so +/// a row written with one reads back with the same columns. Earlier +/// releases gave such a cell one column, and the text does not say which +/// layout wrote it; a row that only fits at one column per such cell can +/// only be one of those, and is read that way, so no file an earlier +/// release wrote stops parsing. fn parse_row(line: &str, number: usize, cols: u16, row: &mut [Cell]) -> Result<()> { + lay_out_row(line, number, cols, row, true).or_else(|_| { + row.fill(Cell::new(String::new(), Style::default(), false, false)); + lay_out_row(line, number, cols, row, false) + }) +} + +/// [`parse_row`] for one layout: `widen` is whether the selector widens. +fn lay_out_row(line: &str, number: usize, cols: u16, row: &mut [Cell], widen: bool) -> Result<()> { + let too_wide = || { + Error::Parse(format!( + "line {number}: the row is wider than the {cols} columns the header declares" + )) + }; let mut col = 0usize; for ch in line.chars() { // `unicode_width` answers `None` for a control character and @@ -194,20 +215,34 @@ fn parse_row(line: &str, number: usize, cols: u16, row: &mut [Cell]) -> Result<( c } }); - match owner.map(|c| &mut row[c]) { - Some(cell) if !cell.contents.is_empty() => cell.contents.push(ch), - _ => { - return Err(Error::Parse(format!( - "line {number}: {ch:?} has no character to combine with" - ))); + let Some(owner) = owner.filter(|&c| !row[c].contents.is_empty()) else { + return Err(Error::Parse(format!( + "line {number}: {ch:?} has no character to combine with" + ))); + }; + row[owner].contents.push(ch); + // The selector widens the cell it joins when the cluster is + // wide — only a narrow cell just written can be widened, since + // the column after it is the one it grows into. + if widen + && ch == crate::width::VS16 + && owner + 1 == col + && !row[owner].wide + && crate::width::cell_is_wide(&row[owner].contents) + { + if col >= usize::from(cols) { + return Err(too_wide()); } + row[owner].wide = true; + row[col] = Cell::new(String::new(), Style::default(), false, true); + col += 1; } continue; } + // At most two columns, as the emulator draws them. + let width = width.min(2); if col + width > usize::from(cols) { - return Err(Error::Parse(format!( - "line {number}: the row is wider than the {cols} columns the header declares" - ))); + return Err(too_wide()); } row[col] = Cell::new(ch.to_string(), Style::default(), width == 2, false); if width == 2 { @@ -427,6 +462,27 @@ mod tests { assert!(many.cell(0, 3).unwrap().is_wide_continuation()); } + /// The emulator gives `⚠️` two columns since vt100 0.18, so a row with + /// one reads back with the same columns; a row that only fits at one + /// column per such cell is what earlier releases wrote, and still reads + /// (#580). + #[test] + fn an_emoji_presentation_selector_widens_its_cell_unless_the_row_cannot_hold_it() { + let screen = Screen::parse("size: 4x1 cursor: 0,0\n\u{26a0}\u{fe0f}X").unwrap(); + assert_eq!(screen.cell(0, 0).unwrap().contents(), "\u{26a0}\u{fe0f}"); + assert!(screen.cell(0, 0).unwrap().is_wide()); + assert!(screen.cell(0, 1).unwrap().is_wide_continuation()); + assert_eq!(screen.cell(0, 2).unwrap().contents(), "X"); + // Four columns of text at one column per selector cell: a 0.11.6 + // row, read the way 0.11.6 laid it out. + let earlier = Screen::parse("size: 4x1 cursor: 0,0\n\u{26a0}\u{fe0f}XYZ").unwrap(); + assert!(!earlier.cell(0, 0).unwrap().is_wide()); + assert_eq!(earlier.cell(0, 1).unwrap().contents(), "X"); + assert_eq!(earlier.cell(0, 3).unwrap().contents(), "Z"); + // A row too wide either way is still refused. + assert!(Screen::parse("size: 4x1 cursor: 0,0\n\u{26a0}\u{fe0f}XYZW").is_err()); + } + /// `unicode_width` answers `None` for a control character and `Some(0)` /// for a combining mark, and only the second joins a cell (#376). #[test] diff --git a/crates/termlens/src/width.rs b/crates/termlens/src/width.rs new file mode 100644 index 0000000..13867ad --- /dev/null +++ b/crates/termlens/src/width.rs @@ -0,0 +1,112 @@ +//! How many columns the grid gives a run of text. +//! +//! Neither `unicode-width` measure is the grid's. The emulator (vt100 0.18) +//! lays text out a character at a time: each takes its own width, at most +//! two columns; a zero-width character joins the cell before it; and an +//! emoji-presentation selector (U+FE0F) makes that cell two columns wide +//! when the text it completes is. So a ZWJ family is one cluster to +//! `UnicodeWidthStr` but two wide cells here, and `⚠️` is one column to a +//! sum of character widths but two here. Every place that turns text into +//! columns asks this instead (#580). + +use unicode_width::{UnicodeWidthChar, UnicodeWidthStr}; + +/// The emoji-presentation selector, the one zero-width character that can +/// change the width of the cell it joins. +pub(crate) const VS16: char = '\u{FE0F}'; + +/// How many columns `text` takes when drawn on one row from the cursor, +/// laid out exactly as the emulator lays it out. +/// +/// A zero-width character at the very start joins a cell before the run, +/// which is not this text's to count. +pub(crate) fn columns(text: &str) -> usize { + let mut total = 0; + // The last cell this text started, and its width, so a selector after + // it can widen it. + let mut cell = String::new(); + let mut cell_width = 0; + for c in text.chars() { + let width = match c.width() { + Some(width) => width.min(2), + // vt100 draws no control character, and treats any other + // character without a width as one column. + None if u32::from(c) < 256 => continue, + None => 1, + }; + if width > 0 { + total += width; + cell.clear(); + cell.push(c); + cell_width = width; + } else if !cell.is_empty() { + cell.push(c); + if c == VS16 { + let widened = if cell.width() > 1 { 2 } else { 1 }; + total = total - cell_width + widened; + cell_width = widened; + } + } + } + total +} + +/// Whether a cell holding `contents` is two columns wide: its first +/// character is, or an emoji-presentation selector makes the cluster so. +pub(crate) fn cell_is_wide(contents: &str) -> bool { + columns(contents) == 2 +} + +#[cfg(test)] +mod tests { + use super::{cell_is_wide, columns}; + + #[test] + fn narrow_and_wide_characters_count_their_own_width() { + assert_eq!(columns("abc"), 3); + assert_eq!(columns("\u{6f22}\u{5b57}"), 4); + assert_eq!(columns(""), 0); + } + + #[test] + fn a_zero_width_character_joins_the_cell_before_it() { + // e + combining acute. + assert_eq!(columns("e\u{301}x"), 2); + // Nothing before it in the run: it joins a cell this text did not + // start, and costs nothing here. + assert_eq!(columns("\u{301}x"), 1); + } + + #[test] + fn a_ligature_is_still_one_cell_per_letter() { + // Lam-alef is one column to the string measure, two letters here. + assert_eq!(unicode_width::UnicodeWidthStr::width("\u{644}\u{627}"), 1); + assert_eq!(columns("\u{644}\u{627}"), 2); + } + + #[test] + fn a_zwj_family_is_two_wide_cells_not_one_cluster() { + // 👨 ZWJ 👩: the joiner joins the man's cell, the woman starts one. + let family = "\u{1f468}\u{200d}\u{1f469}"; + assert_eq!(unicode_width::UnicodeWidthStr::width(family), 2); + assert_eq!(columns(family), 4); + } + + #[test] + fn the_presentation_selector_widens_the_emoji_it_follows() { + // ⚠ (narrow) + VS16 is two columns; a skin tone modifier is a wide + // character of its own. + assert_eq!(columns("\u{26a0}\u{fe0f}X"), 3); + assert_eq!(columns("\u{1f44d}\u{1f3fd}"), 4); + // The selector on a letter changes nothing. + assert_eq!(columns("a\u{fe0f}"), 1); + assert!(cell_is_wide("\u{26a0}\u{fe0f}")); + assert!(!cell_is_wide("\u{26a0}")); + assert!(cell_is_wide("\u{6f22}")); + } + + #[test] + fn control_characters_take_no_column() { + assert_eq!(columns("a\u{7}b\u{85}c"), 3); + } +} From 93df99a9dce08282296b871b7ab978d0c57a2f58 Mon Sep 17 00:00:00 2001 From: Vyncint Ng <115854244+vyncint@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:26:11 +0700 Subject: [PATCH 2/2] docs: log #579-#581, and document the limits #580, #581, #585 leave LIMITATIONS.md names the insert run that wraps at the margin, autowrap off, and the wide character that wraps unrecorded over an empty last column (#585); DESIGN.md section 3 gives the reader's width rule. Signed-off-by: Vyncint Ng <115854244+vyncint@users.noreply.github.com> --- CHANGELOG.md | 24 ++++++++++++++++++++++++ docs/DESIGN.md | 8 +++++++- docs/LIMITATIONS.md | 15 ++++++++++++--- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 698e438..944b7bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,30 @@ reads that marker. a row of `=` drew as a single character. Under insert mode the repeated characters still overwrite rather than insert, as `docs/LIMITATIONS.md` says. +- An underline colour in the form crossterm writes it (`CSI 58;5;n m`, + `CSI 58;2;r;g;b m`) no longer reads as attributes (#579). The emulator + took `5` for blink and `n` for an attribute of its own, so a cell lost its + style (`58;5;0` reset it) or turned bold (`58;5;1`); ratatui writes + underline colours by default. The sequence is still listed by + `unsupported()`, now as one parameter (`^[[58:5:1m`). +- Text is measured in the columns the grid gives it (#580). Insert mode + reserved room for a run by its string width, so a ZWJ family, lam-alef or + a skin-toned emoji overwrote the characters after it instead of pushing + them right; `Screen::locate` reported a history column that + `scrollback_cell` did not address; and the `termlens diff` frame sat a + column or two off on such rows. A saved screen holding an emoji the + presentation selector widened (`⚠️`) reads back with the columns it was + written at, and a row an earlier release wrote with one column for it + still reads. +- `CSI ? 7 h`, which every ncurses program sends at startup, is no longer + listed by `unsupported()` (#581). It asks for autowrap, the only + behaviour the emulator has, so no ncurses application could pass + `unsupported().is_empty()`. `CSI ? 7 l` is still listed: autowrap cannot + be turned off, as `docs/LIMITATIONS.md` now says. +- `Screen::row_wrapped` and `docs/LIMITATIONS.md` say that a wide character + which moved to the next row because it did not fit in an empty last + column is not recorded as a wrap, so `logical_text()` splits the line + there (#585). - `termlens` no longer panics with exit code 101 (#594). A closed or full stderr (`2>&1 | true`, `2>/dev/full`) made every diagnostic and the `inspect` trailer panic; the write is now checked, and a trailer that cannot diff --git a/docs/DESIGN.md b/docs/DESIGN.md index be4dd6d..d42e737 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -621,7 +621,13 @@ Rules: grid, so `cell()`, `row_text()`, and `find()` coordinates are unaffected. 4. Wide characters occupy their real columns: the character appears once; its continuation cell contributes nothing to the text but does count - for column arithmetic (`find` returns true terminal columns). + for column arithmetic (`find` returns true terminal columns). The + columns are the emulator's, character by character — a ZWJ family is + two wide cells, not one cluster — and an emoji-presentation selector + (U+FE0F) makes the emoji it follows two columns, as vt100 0.18 lays it + out. Releases before 0.11.7 gave such an emoji one column and the text + does not say which wrote it, so the reader widens it unless the row + then would not fit, and reads that row the earlier way. 5. Styles are text-invisible by default; `Screen::with_styles()` opts in. Its rendering is the plain `Display` output followed by a blank line and a `styles:` block: diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index b6c2557..08ed2e8 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -44,8 +44,10 @@ CHANGELOG entry. - **Insert mode (`IRM`, `CSI 4 h`) pushes the rest of the row right**, as ncurses's `insch` expects on a terminal advertising `smir`; `RIS` and `DECSTR` clear it, and `Screen::insert_mode()` reports an application - that left it on. One exception: the characters REP (`CSI n b`) repeats - overwrite the row even in insert mode, where xterm inserts them. Other + that left it on. Two exceptions: the characters REP (`CSI n b`) repeats + overwrite the row even in insert mode, where xterm inserts them; and an + inserted run that reaches the right margin wraps onto the next row over + its text, where xterm pushes that row right as well. Other ANSI modes the backend drops (`LNM` and the rest) are not modelled — and *say so*: `Screen::unsupported()` lists every sequence the emulator did not implement, in the form `^[[20h`, so a test can tell a @@ -54,7 +56,14 @@ CHANGELOG entry. row by row and do not span the wrap; `Screen::row_wrapped(row)` reports the backend's record of where a line wrapped, and `Screen::logical_text()` joins wrapped rows back together for the - assertion that spans one. + assertion that spans one. The backend records a wrap only where the last + column held something: a wide character that does not fit in an empty + last column moves to the next row unrecorded, so on six columns + `abcde漢fg` reads as `abcde` and `漢fg` there. +- **Autowrap cannot be turned off.** `CSI ? 7 l` (DECAWM reset) is not + modelled: text past the right margin still wraps, and `unsupported()` + names the sequence. `CSI ? 7 h`, which every ncurses program sends, asks + for what the emulator already does and is not named. - **Tab stops are the application's to set.** `HTS` (`ESC H`), `TBC` (`CSI g`, `CSI 3 g`), `CHT` (`CSI I`) and `CBT` (`CSI Z`) all work, and a plain `\t` honours whatever stops are set rather than a fixed eight.