From fb85278d6fcd4af49fc0f642f3504ec9ceefc599 Mon Sep 17 00:00:00 2001 From: Vincent Bernardoff Date: Sat, 17 Oct 2026 18:28:49 +0200 Subject: [PATCH] feat: render Discord Lottie stickers with tlottie Render cached Lottie JSON natively into the existing animation cache, including sticker-picker previews. Keep still-frame fallback, playback settings, frame budgets, bounded parsing, and transparent BGRA output. Pin the GUI-only CPU renderer without adding daemon dependencies. Add synthetic rendering and cache regressions, extend the Discord picker fixture, and document resource limits. The reported cached sticker renders correctly, including its first-frame fallback. Co-Authored-By: GPT-6 --- Cargo.lock | 6 + crates/chatd/src/adapters/discord/content.rs | 2 +- crates/chatd/src/adapters/discord/model.rs | 3 +- crates/chatd/tests/discord.rs | 7 +- crates/tinyclient/Cargo.toml | 5 +- crates/tinyclient/src/frames.rs | 14 +- .../tinyclient/src/gui/conversation/images.rs | 24 ++- crates/tinyclient/src/lottie.rs | 192 ++++++++++++++++++ crates/tinyclient/src/main.rs | 2 + .../tinyclient/tests/fixtures/moving.lottie | 63 ++++++ docs/design/polish/06-composer.md | 13 ++ 11 files changed, 319 insertions(+), 12 deletions(-) create mode 100644 crates/tinyclient/src/lottie.rs create mode 100644 crates/tinyclient/tests/fixtures/moving.lottie diff --git a/Cargo.lock b/Cargo.lock index 915ea87..eaa9d4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9086,6 +9086,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "smallvec", + "tlottie", "unicode-segmentation", "ureq", "url", @@ -9108,6 +9109,11 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" +[[package]] +name = "tlottie" +version = "1.0.6" +source = "git+https://github.com/dkaraush/tlottie?rev=92df98dc209bc39b1e567ec74a8c86a0af5239de#92df98dc209bc39b1e567ec74a8c86a0af5239de" + [[package]] name = "tls_codec" version = "0.4.2" diff --git a/crates/chatd/src/adapters/discord/content.rs b/crates/chatd/src/adapters/discord/content.rs index 8303acf..fd85751 100644 --- a/crates/chatd/src/adapters/discord/content.rs +++ b/crates/chatd/src/adapters/discord/content.rs @@ -519,7 +519,7 @@ fn attachment(rich: &mut Rich, attached: &Attached) -> Attachment { } } -/// A sticker is drawn like an attached picture. Lottie ones are a file nobody can draw here. +/// A sticker is drawn like an attached picture; the shell renders Lottie JSON with tlottie. fn sticker(rich: &mut Rich, sticker: &model::Sticker) -> Attachment { let (kind, url, mime) = match sticker.format_type { 3 => (MediaKind::File, format!("{}/stickers/{}.json", model::cdn(), sticker.id), "application/json"), diff --git a/crates/chatd/src/adapters/discord/model.rs b/crates/chatd/src/adapters/discord/model.rs index 65619af..d2c0814 100644 --- a/crates/chatd/src/adapters/discord/model.rs +++ b/crates/chatd/src/adapters/discord/model.rs @@ -457,11 +457,12 @@ fn nullable_yes<'de, D: Deserializer<'de>>(de: D) -> Result { } /// A sticker's picture from the media proxy, registered as a message shows it (`content.rs`), so -/// the picker and a sent sticker share one file. `None` for a Lottie one, which has no still. +/// the picker and a sent sticker share one file. Lottie JSON is rendered by the shell. pub fn sticker(id: &str, format: u8) -> Option<(MediaFile, String)> { let (kind, url) = match format { 1 => (MediaKind::Image, format!("{}/stickers/{id}.png?size=160", media())), 2 => (MediaKind::Image, apng_sticker(id)), + 3 => (MediaKind::File, format!("{}/stickers/{id}.json", cdn())), 4 => (MediaKind::Gif, format!("{}/stickers/{id}.gif", media())), _ => return None, }; diff --git a/crates/chatd/tests/discord.rs b/crates/chatd/tests/discord.rs index f3c21c1..81df53b 100644 --- a/crates/chatd/tests/discord.rs +++ b/crates/chatd/tests/discord.rs @@ -2742,14 +2742,15 @@ fn files_go_up_from_their_paths_with_the_senders_options_and_a_cancel_stops_them assert_eq!(posted[1].body["content"], "https://tenor.com/view/cat-typing-1"); assert_eq!(posted[1].body["allowed_mentions"]["replied_user"], true); // The server's own emoji and stickers, from READY: what lost its boost, an animated one - // (Nitro's), one only another role may use and the Lottie sticker are not offered, and a - // DM offers none without Nitro. Nothing is asked. + // (Nitro's), and one only another role may use are not offered. Lottie stickers + // have a native renderer; a DM still offers no server emoji without Nitro. Nothing is asked. let before = world.discord.asked().len(); let emoji = catalog(protocol::CatalogKind::Emoji, "", Some("4000")); assert_eq!(emoji.iter().map(|item| (item.id.as_str(), item.name.as_str())).collect::>(), [("emoji:5101", ":ferris:")]); assert!(catalog(protocol::CatalogKind::Emoji, "", None).is_empty()); let stickers = catalog(protocol::CatalogKind::Stickers, "", Some("4000")); - assert_eq!(stickers.iter().map(|item| item.id.as_str()).collect::>(), ["5201"]); + assert_eq!(stickers.iter().map(|item| item.id.as_str()).collect::>(), ["5201", "5202"]); + assert_eq!(stickers[1].file.as_ref().unwrap().kind, protocol::MediaKind::File, "Lottie JSON is offered to the native renderer"); assert_eq!(world.discord.asked().len(), before); let pick = protocol::Pick { kind: protocol::CatalogKind::Stickers, id: "5201".into() }; send_with(&world, "4101", "77005", "", None, protocol::Outgoing { pick: Some(pick), ..Default::default() }); diff --git a/crates/tinyclient/Cargo.toml b/crates/tinyclient/Cargo.toml index 87fa989..7e96609 100644 --- a/crates/tinyclient/Cargo.toml +++ b/crates/tinyclient/Cargo.toml @@ -26,12 +26,15 @@ serde_json = { workspace = true, optional = true } ureq = { version = "3", default-features = false, features = ["native-tls"], optional = true } url = { version = "2", optional = true } +# Native CPU rendering of Discord Lottie stickers, pinned for reproducible builds. +tlottie = { git = "https://github.com/dkaraush/tlottie", rev = "92df98dc209bc39b1e567ec74a8c86a0af5239de", optional = true } + [dev-dependencies] # The frame decoder's tests run without the window. image = { version = "0.25", default-features = false, features = ["png", "jpeg", "gif", "webp", "bmp"] } [features] -gui = ["dep:ureq", "dep:url", "dep:gpui", "dep:gpui_platform", "dep:unicode-segmentation", "dep:libc", "dep:image", "dep:smallvec", "dep:serde_json", "dep:windows-sys"] +gui = ["dep:tlottie", "dep:ureq", "dep:url", "dep:gpui", "dep:gpui_platform", "dep:unicode-segmentation", "dep:libc", "dep:image", "dep:smallvec", "dep:serde_json", "dep:windows-sys"] # Adds `--shot out.png`: render the window to a file and quit, for design review without # Screen Recording permission. Pulls GPUI's test support, so it is not part of `gui`. gui-shot = ["gui", "gpui/test-support", "gpui_platform/test-support"] diff --git a/crates/tinyclient/src/frames.rs b/crates/tinyclient/src/frames.rs index cfafe00..8328f06 100644 --- a/crates/tinyclient/src/frames.rs +++ b/crates/tinyclient/src/frames.rs @@ -1,4 +1,4 @@ -//! Moving pictures: animated GIF, animated WebP and APNG, decoded to frames at the size they +//! Moving pictures: animated GIF, WebP, APNG and (with the GUI) Lottie, decoded at the size they //! are drawn. A file's header is read first, without decoding, for its canvas, frame count and //! how many times it plays; that decides the size and how many frames are kept so one //! animation never passes its [`Limits`]. Frames are scaled down one at a time as they are @@ -42,7 +42,7 @@ pub struct Sniff { } /// Reads a file's header for whether it moves. `None` for a still picture, one frame, or a -/// file that is not one of the three formats. +/// file that is not a supported animation format. pub fn sniff(bytes: &[u8]) -> Option { let found = if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { gif(bytes) @@ -51,7 +51,10 @@ pub fn sniff(bytes: &[u8]) -> Option { } else if bytes.starts_with(b"\x89PNG\r\n\x1a\n") { apng(bytes) } else { - None + #[cfg(feature = "gui")] + { crate::lottie::sniff(bytes) } + #[cfg(not(feature = "gui"))] + { None } }?; (found.frames > 1 && found.width > 0 && found.height > 0).then_some(found) } @@ -163,7 +166,7 @@ pub struct Plan { /// The pixels of a `canvas` scaled to `edge` on its longer side, then cut to `crop` times its /// width from the top. -fn scaled(canvas: (u32, u32), edge: u32, crop: Option) -> (u32, u32) { +pub(crate) fn scaled(canvas: (u32, u32), edge: u32, crop: Option) -> (u32, u32) { let (w, h) = (u64::from(canvas.0.max(1)), u64::from(canvas.1.max(1))); let edge = u64::from(edge.max(1)); let (width, height) = if w >= h { (edge, (h * edge / w).max(1)) } else { ((w * edge / h).max(1), edge) }; @@ -230,6 +233,9 @@ fn shown_for(delay: image::Delay) -> Duration { /// Decodes the frames of a moving picture per its plan. `None` when it does not move, does not /// fit, or will not decode. pub fn decode(bytes: &[u8], edge: u32, crop: Option, limits: Limits) -> Option { + #[cfg(feature = "gui")] + if crate::lottie::candidate(bytes) { return crate::lottie::decode(bytes, edge, crop, Some(limits)); } + let sniff = sniff(bytes)?; let plan = plan(&sniff, edge, crop, limits)?; let source = || Cursor::new(bytes); diff --git a/crates/tinyclient/src/gui/conversation/images.rs b/crates/tinyclient/src/gui/conversation/images.rs index ed32737..bf9245b 100644 --- a/crates/tinyclient/src/gui/conversation/images.rs +++ b/crates/tinyclient/src/gui/conversation/images.rs @@ -128,6 +128,10 @@ pub struct Decoded { /// whole. `crop` keeps only the top of a very tall picture: that many times its width. /// `None` when the file is missing or is not a picture. pub fn decode(path: &Path, edge: u32, crop: Option) -> Option { + if let Some(bytes) = crate::lottie::read(path) { + let mut image = crate::lottie::decode(&bytes, edge, crop, None)?; + return Some(Decoded { width: image.width, height: image.height, bgra: image.frames.pop()?.0 }); + } if !looks_like_a_picture(path) { return None; } @@ -288,13 +292,15 @@ pub fn load(path: &Path, edge: u32, crop: Option, frames: Option) - } /// Whether the start of a file says it may move: a GIF, a WebP marked animated, a PNG with an -/// animation chunk before its pixels. Only those are read whole. +/// animation chunk before its pixels, or a possible Lottie JSON object. Only those +/// are read whole; Lottie parsing applies its own stricter input limit. fn might_move(path: &Path) -> bool { use std::io::Read; let mut head = vec![0u8; 4096]; let read = std::fs::File::open(path).and_then(|mut file| file.read(&mut head)).unwrap_or(0); let head = &head[..read]; - head.starts_with(b"GIF8") + crate::lottie::candidate(head) + || head.starts_with(b"GIF8") || (head.starts_with(b"RIFF") && head.get(8..16) == Some(b"WEBPVP8X") && head.get(20).is_some_and(|flags| flags & 0x02 != 0)) || (head.starts_with(b"\x89PNG") && head.windows(4).take_while(|chunk| *chunk != b"IDAT").any(|chunk| chunk == b"acTL")) } @@ -706,6 +712,20 @@ mod tests { } } + #[test] + fn a_cached_lottie_sticker_animates_or_keeps_its_first_frame() { + let path = std::env::temp_dir().join(format!("tc-lottie-{}", std::process::id())); + std::fs::write(&path, include_bytes!("../../../tests/fixtures/moving.lottie")).unwrap(); + let (playing, moves) = load(&path, 16, None, Some(frames::SMALL)).unwrap(); + assert_eq!((playing.frame_count(), moves), (4, Moves::Playing)); + anim::MESSAGES.give(anim::bytes_of(&playing)); + for limits in [None, Some(Limits { bytes: 0, frames: 2 })] { + let (still, moves) = load(&path, 16, None, limits).unwrap(); + assert_eq!((still.frame_count(), moves), (1, Moves::Still)); + } + std::fs::remove_file(path).unwrap(); + } + #[test] fn a_moving_picture_is_read_for_frames_only_when_they_are_wanted() { // Only these are read whole: a GIF, a WebP marked animated, a PNG with an animation chunk. diff --git a/crates/tinyclient/src/lottie.rs b/crates/tinyclient/src/lottie.rs new file mode 100644 index 0000000..001b4d4 --- /dev/null +++ b/crates/tinyclient/src/lottie.rs @@ -0,0 +1,192 @@ +//! Native Lottie stickers, rendered off the UI thread into the existing frame cache. +//! tlottie never resolves external resources. Input, canvas and retained frames are bounded. + +use crate::frames::{self, Animation, Limits, Sniff}; +use std::io::Read; +use std::path::Path; +use std::time::{Duration, Instant}; + +const MAX_INPUT: usize = 2 << 20; +const MAX_EDGE: u32 = 512; + +pub fn candidate(bytes: &[u8]) -> bool { + bytes.iter().copied().find(|b| !b.is_ascii_whitespace()) == Some(b'{') +} + +pub fn read(path: &Path) -> Option> { + let mut file = std::fs::File::open(path).ok()?; + let mut head = [0; 64]; + let n = file.read(&mut head).ok()?; + if !candidate(&head[..n]) { + return None; + } + use std::io::Seek; + file.rewind().ok()?; + let mut bytes = Vec::new(); + file.take((MAX_INPUT + 1) as u64) + .read_to_end(&mut bytes) + .ok()?; + (bytes.len() <= MAX_INPUT).then_some(bytes) +} + +pub fn sniff(bytes: &[u8]) -> Option { + if bytes.len() > MAX_INPUT || !candidate(bytes) { + return None; + } + let header: serde_json::Value = serde_json::from_slice(bytes).ok()?; + let width = u32::try_from(header["w"].as_u64()?).ok()?; + let height = u32::try_from(header["h"].as_u64()?).ok()?; + let rate = header["fr"].as_f64()?; + let count = (header["op"].as_f64()? - header["ip"].as_f64()?).ceil(); + header["layers"].as_array()?; + if !(1..=4096).contains(&width) + || !(1..=4096).contains(&height) + || !(0.1..=240.).contains(&rate) + || !(1.0..=1000.0).contains(&count) + { + return None; + } + Some(Sniff { + width, + height, + frames: count as usize, + plays: None, + }) +} + +pub fn decode( + bytes: &[u8], + edge: u32, + crop: Option, + limits: Option, +) -> Option { + let sniff = sniff(bytes)?; + let parse_limits = tlottie::Limits { + max_input_bytes: MAX_INPUT, + max_dimension: 4096, + max_parse_bytes: 64 << 20, + ..Default::default() + }; + let composition = tlottie::Composition::parse(bytes, &parse_limits).ok()?; + let moving = limits.is_some() && !composition.is_static() && sniff.frames > 1; + let edge = edge.clamp(1, MAX_EDGE).min(sniff.width.max(sniff.height)); + let (width, height, step) = if moving { + let plan = frames::plan(&sniff, edge, crop, limits?)?; + (plan.width, plan.height, plan.step) + } else { + let (width, height) = frames::scaled((sniff.width, sniff.height), edge, crop); + (width, height, 1) + }; + // Render the full aspect ratio, then keep the top rows for CoverTop. + let full_height = ((u64::from(sniff.height) * u64::from(width) / u64::from(sniff.width)).max(1) + as u32) + .max(height); + let count = if moving { sniff.frames } else { 1 }; + let rate = f64::from(composition.frame_rate); + let first = composition.in_point; + let mut renderer = tlottie::CPURenderer::new(composition); + let mut pixels = vec![0u32; width as usize * full_height as usize]; + let mut kept = Vec::new(); + let started = Instant::now(); + for at in (0..count).step_by(step) { + if started.elapsed() > Duration::from_secs(2) { + return None; + } + renderer + .render( + first + at as f32, + &mut pixels, + width, + full_height, + tlottie::RenderOptions::default(), + ) + .ok()?; + let mut bgra = Vec::with_capacity(width as usize * height as usize * 4); + for &pixel in pixels.iter().take(width as usize * height as usize) { + let alpha = pixel >> 24; + let straight = |channel: u32| { + if alpha == 0 { + 0 + } else { + ((channel * 255 + alpha / 2) / alpha).min(255) as u8 + } + }; + bgra.extend([ + straight((pixel >> 16) & 255), + straight((pixel >> 8) & 255), + straight(pixel & 255), + alpha as u8, + ]); + } + let held = Duration::from_secs_f64((step.min(count - at)) as f64 / rate); + kept.push((bgra, held)); + } + Some(Animation { + width, + height, + frames: kept, + plays: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + fn fixture() -> Vec { + include_bytes!("../tests/fixtures/moving.lottie").to_vec() + } + #[test] + fn native_frames_move_and_have_straight_bgra_and_correct_timing() { + let bytes = fixture(); + let image = decode(&bytes, 16, None, Some(frames::SMALL)).unwrap(); + assert_eq!((image.width, image.height, image.frames.len()), (16, 16, 4)); + assert_ne!(image.frames[0].0, image.frames[3].0); + assert!( + image.frames[0] + .0 + .chunks_exact(4) + .any(|p| p[0] == 0 && p[2] == 255 && (120..=130).contains(&p[3])) + ); + assert!( + image + .frames + .iter() + .all(|(_, delay)| *delay == Duration::from_millis(100)) + ); + assert!(image.plays.is_none()); + let still = decode(&bytes, 16, None, None).unwrap(); + assert_eq!(still.frames.len(), 1); + assert_eq!(still.frames[0].0, image.frames[0].0); + } + #[test] + fn cropped_portrait_stickers_keep_the_original_aspect_ratio() { + let mut value: serde_json::Value = serde_json::from_slice(&fixture()).unwrap(); + value["h"] = 32.into(); + let bytes = serde_json::to_vec(&value).unwrap(); + let full = decode(&bytes, 32, None, None).unwrap(); + let cropped = decode(&bytes, 32, Some(0.5), None).unwrap(); + assert_eq!((full.width, full.height), (16, 32)); + assert_eq!((cropped.width, cropped.height), (16, 8)); + assert_eq!(cropped.frames[0].0, full.frames[0].0[..16 * 8 * 4]); + } + + #[test] + fn invalid_and_oversized_inputs_are_rejected_and_frame_budgets_hold() { + for bytes in [b"{}".as_slice(), b"{not json}", b"[]"] { + assert!(decode(bytes, 16, None, None).is_none()); + } + let mut value: serde_json::Value = serde_json::from_slice(&fixture()).unwrap(); + value["w"] = 100000.into(); + assert!(sniff(&serde_json::to_vec(&value).unwrap()).is_none()); + let limits = Limits { + bytes: 16 * 16 * 4 * 2, + frames: 2, + }; + let image = decode(&fixture(), 16, None, Some(limits)).unwrap(); + assert!(image.bytes() <= limits.bytes && image.frames.len() <= limits.frames); + assert_eq!( + image.frames.iter().map(|(_, d)| *d).sum::(), + Duration::from_millis(400) + ); + } +} diff --git a/crates/tinyclient/src/main.rs b/crates/tinyclient/src/main.rs index 107ad3a..28d9dfe 100644 --- a/crates/tinyclient/src/main.rs +++ b/crates/tinyclient/src/main.rs @@ -10,6 +10,8 @@ mod update; mod frames; #[cfg(feature = "gui")] mod gui; +#[cfg(feature = "gui")] +mod lottie; mod view; use client::ask; diff --git a/crates/tinyclient/tests/fixtures/moving.lottie b/crates/tinyclient/tests/fixtures/moving.lottie new file mode 100644 index 0000000..933dd95 --- /dev/null +++ b/crates/tinyclient/tests/fixtures/moving.lottie @@ -0,0 +1,63 @@ +{ + "v": "5.6.2", + "w": 16, + "h": 16, + "fr": 10, + "ip": 0, + "op": 4, + "layers": [ + { + "ty": 1, + "ind": 1, + "sw": 8, + "sh": 8, + "sc": "#ff0000", + "ip": 0, + "op": 4, + "st": 0, + "ks": { + "p": { + "a": 1, + "k": [ + { + "t": 0, + "s": [ + 0, + 0 + ], + "e": [ + 8, + 0 + ] + }, + { + "t": 4, + "s": [ + 8, + 0 + ] + } + ] + }, + "a": { + "a": 0, + "k": [ + 0, + 0 + ] + }, + "s": { + "a": 0, + "k": [ + 100, + 100 + ] + }, + "o": { + "a": 0, + "k": 50 + } + } + } + ] +} diff --git a/docs/design/polish/06-composer.md b/docs/design/polish/06-composer.md index 721a6bc..63b3557 100644 --- a/docs/design/polish/06-composer.md +++ b/docs/design/polish/06-composer.md @@ -185,6 +185,19 @@ removing a file (Discord) and replacing a photo (Telegram) are P2. - **Still-image decode**: media cache filenames have no extension. On Linux and Windows, detect the image format from its bytes, as on macOS. Thumbnail decoding shrinks large sources but never enlarges a smaller source to the requested edge. + +- **Lottie stickers**: tlottie's CPU renderer parses cached Discord JSON and + supplies frames to the same image cache as GIF/APNG. The GUI-only dependency + is pinned in Cargo.lock; daemons and non-GUI builds do not link it. The picker + registers Lottie JSON with the same sticker media id used by received messages. + Decode runs off the UI thread, respects autoplay/Reduce Motion and existing + frame budgets, and retains frame zero when animation is disabled or cannot fit. + Inputs are at most 2 MiB, source dimensions 4096, duration 1000 source frames, + and rate 0.1–240 fps. Rendering is capped at a 512-pixel edge, retains at most + the cache's frame/byte allowance, and stops between frames after two seconds. + Transparent premultiplied RGBA output becomes straight BGRA for GPUI. External + files/fonts named in animation JSON are never loaded. Telegram TGS support is + unchanged; this path handles JSON Lottie stickers. - **GIF decode rule**: every cell is a first-frame still (`images.rs` reads index 0 only). The one hovered cell swaps to `img()` of the provider's smallest GIF rendition (Tenor `tinygif`, ~220 px): this rev's `img()` steps frames itself and stops under `reduce_motion()`. Played only if width × height × 4 × frames ≤ 8 MB, else it stays still; dropped