diff --git a/Cargo.lock b/Cargo.lock index 915ea87..eaa9d4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9086,6 +9086,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "smallvec", + "tlottie", "unicode-segmentation", "ureq", "url", @@ -9108,6 +9109,11 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" +[[package]] +name = "tlottie" +version = "1.0.6" +source = "git+https://github.com/dkaraush/tlottie?rev=92df98dc209bc39b1e567ec74a8c86a0af5239de#92df98dc209bc39b1e567ec74a8c86a0af5239de" + [[package]] name = "tls_codec" version = "0.4.2" diff --git a/crates/chatd/src/adapters/discord/content.rs b/crates/chatd/src/adapters/discord/content.rs index 8303acf..fd85751 100644 --- a/crates/chatd/src/adapters/discord/content.rs +++ b/crates/chatd/src/adapters/discord/content.rs @@ -519,7 +519,7 @@ fn attachment(rich: &mut Rich, attached: &Attached) -> Attachment { } } -/// A sticker is drawn like an attached picture. Lottie ones are a file nobody can draw here. +/// A sticker is drawn like an attached picture; the shell renders Lottie JSON with tlottie. fn sticker(rich: &mut Rich, sticker: &model::Sticker) -> Attachment { let (kind, url, mime) = match sticker.format_type { 3 => (MediaKind::File, format!("{}/stickers/{}.json", model::cdn(), sticker.id), "application/json"), diff --git a/crates/chatd/src/adapters/discord/model.rs b/crates/chatd/src/adapters/discord/model.rs index 65619af..d2c0814 100644 --- a/crates/chatd/src/adapters/discord/model.rs +++ b/crates/chatd/src/adapters/discord/model.rs @@ -457,11 +457,12 @@ fn nullable_yes<'de, D: Deserializer<'de>>(de: D) -> Result { } /// A sticker's picture from the media proxy, registered as a message shows it (`content.rs`), so -/// the picker and a sent sticker share one file. `None` for a Lottie one, which has no still. +/// the picker and a sent sticker share one file. Lottie JSON is rendered by the shell. pub fn sticker(id: &str, format: u8) -> Option<(MediaFile, String)> { let (kind, url) = match format { 1 => (MediaKind::Image, format!("{}/stickers/{id}.png?size=160", media())), 2 => (MediaKind::Image, apng_sticker(id)), + 3 => (MediaKind::File, format!("{}/stickers/{id}.json", cdn())), 4 => (MediaKind::Gif, format!("{}/stickers/{id}.gif", media())), _ => return None, }; diff --git a/crates/chatd/tests/discord.rs b/crates/chatd/tests/discord.rs index f3c21c1..81df53b 100644 --- a/crates/chatd/tests/discord.rs +++ b/crates/chatd/tests/discord.rs @@ -2742,14 +2742,15 @@ fn files_go_up_from_their_paths_with_the_senders_options_and_a_cancel_stops_them assert_eq!(posted[1].body["content"], "https://tenor.com/view/cat-typing-1"); assert_eq!(posted[1].body["allowed_mentions"]["replied_user"], true); // The server's own emoji and stickers, from READY: what lost its boost, an animated one - // (Nitro's), one only another role may use and the Lottie sticker are not offered, and a - // DM offers none without Nitro. Nothing is asked. + // (Nitro's), and one only another role may use are not offered. Lottie stickers + // have a native renderer; a DM still offers no server emoji without Nitro. Nothing is asked. let before = world.discord.asked().len(); let emoji = catalog(protocol::CatalogKind::Emoji, "", Some("4000")); assert_eq!(emoji.iter().map(|item| (item.id.as_str(), item.name.as_str())).collect::>(), [("emoji:5101", ":ferris:")]); assert!(catalog(protocol::CatalogKind::Emoji, "", None).is_empty()); let stickers = catalog(protocol::CatalogKind::Stickers, "", Some("4000")); - assert_eq!(stickers.iter().map(|item| item.id.as_str()).collect::>(), ["5201"]); + assert_eq!(stickers.iter().map(|item| item.id.as_str()).collect::>(), ["5201", "5202"]); + assert_eq!(stickers[1].file.as_ref().unwrap().kind, protocol::MediaKind::File, "Lottie JSON is offered to the native renderer"); assert_eq!(world.discord.asked().len(), before); let pick = protocol::Pick { kind: protocol::CatalogKind::Stickers, id: "5201".into() }; send_with(&world, "4101", "77005", "", None, protocol::Outgoing { pick: Some(pick), ..Default::default() }); diff --git a/crates/tinyclient/Cargo.toml b/crates/tinyclient/Cargo.toml index 87fa989..7e96609 100644 --- a/crates/tinyclient/Cargo.toml +++ b/crates/tinyclient/Cargo.toml @@ -26,12 +26,15 @@ serde_json = { workspace = true, optional = true } ureq = { version = "3", default-features = false, features = ["native-tls"], optional = true } url = { version = "2", optional = true } +# Native CPU rendering of Discord Lottie stickers, pinned for reproducible builds. +tlottie = { git = "https://github.com/dkaraush/tlottie", rev = "92df98dc209bc39b1e567ec74a8c86a0af5239de", optional = true } + [dev-dependencies] # The frame decoder's tests run without the window. image = { version = "0.25", default-features = false, features = ["png", "jpeg", "gif", "webp", "bmp"] } [features] -gui = ["dep:ureq", "dep:url", "dep:gpui", "dep:gpui_platform", "dep:unicode-segmentation", "dep:libc", "dep:image", "dep:smallvec", "dep:serde_json", "dep:windows-sys"] +gui = ["dep:tlottie", "dep:ureq", "dep:url", "dep:gpui", "dep:gpui_platform", "dep:unicode-segmentation", "dep:libc", "dep:image", "dep:smallvec", "dep:serde_json", "dep:windows-sys"] # Adds `--shot out.png`: render the window to a file and quit, for design review without # Screen Recording permission. Pulls GPUI's test support, so it is not part of `gui`. gui-shot = ["gui", "gpui/test-support", "gpui_platform/test-support"] diff --git a/crates/tinyclient/src/frames.rs b/crates/tinyclient/src/frames.rs index cfafe00..8328f06 100644 --- a/crates/tinyclient/src/frames.rs +++ b/crates/tinyclient/src/frames.rs @@ -1,4 +1,4 @@ -//! Moving pictures: animated GIF, animated WebP and APNG, decoded to frames at the size they +//! Moving pictures: animated GIF, WebP, APNG and (with the GUI) Lottie, decoded at the size they //! are drawn. A file's header is read first, without decoding, for its canvas, frame count and //! how many times it plays; that decides the size and how many frames are kept so one //! animation never passes its [`Limits`]. Frames are scaled down one at a time as they are @@ -42,7 +42,7 @@ pub struct Sniff { } /// Reads a file's header for whether it moves. `None` for a still picture, one frame, or a -/// file that is not one of the three formats. +/// file that is not a supported animation format. pub fn sniff(bytes: &[u8]) -> Option { let found = if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { gif(bytes) @@ -51,7 +51,10 @@ pub fn sniff(bytes: &[u8]) -> Option { } else if bytes.starts_with(b"\x89PNG\r\n\x1a\n") { apng(bytes) } else { - None + #[cfg(feature = "gui")] + { crate::lottie::sniff(bytes) } + #[cfg(not(feature = "gui"))] + { None } }?; (found.frames > 1 && found.width > 0 && found.height > 0).then_some(found) } @@ -163,7 +166,7 @@ pub struct Plan { /// The pixels of a `canvas` scaled to `edge` on its longer side, then cut to `crop` times its /// width from the top. -fn scaled(canvas: (u32, u32), edge: u32, crop: Option) -> (u32, u32) { +pub(crate) fn scaled(canvas: (u32, u32), edge: u32, crop: Option) -> (u32, u32) { let (w, h) = (u64::from(canvas.0.max(1)), u64::from(canvas.1.max(1))); let edge = u64::from(edge.max(1)); let (width, height) = if w >= h { (edge, (h * edge / w).max(1)) } else { ((w * edge / h).max(1), edge) }; @@ -230,6 +233,9 @@ fn shown_for(delay: image::Delay) -> Duration { /// Decodes the frames of a moving picture per its plan. `None` when it does not move, does not /// fit, or will not decode. pub fn decode(bytes: &[u8], edge: u32, crop: Option, limits: Limits) -> Option { + #[cfg(feature = "gui")] + if crate::lottie::candidate(bytes) { return crate::lottie::decode(bytes, edge, crop, Some(limits)); } + let sniff = sniff(bytes)?; let plan = plan(&sniff, edge, crop, limits)?; let source = || Cursor::new(bytes); diff --git a/crates/tinyclient/src/gui/conversation/images.rs b/crates/tinyclient/src/gui/conversation/images.rs index ed32737..bf9245b 100644 --- a/crates/tinyclient/src/gui/conversation/images.rs +++ b/crates/tinyclient/src/gui/conversation/images.rs @@ -128,6 +128,10 @@ pub struct Decoded { /// whole. `crop` keeps only the top of a very tall picture: that many times its width. /// `None` when the file is missing or is not a picture. pub fn decode(path: &Path, edge: u32, crop: Option) -> Option { + if let Some(bytes) = crate::lottie::read(path) { + let mut image = crate::lottie::decode(&bytes, edge, crop, None)?; + return Some(Decoded { width: image.width, height: image.height, bgra: image.frames.pop()?.0 }); + } if !looks_like_a_picture(path) { return None; } @@ -288,13 +292,15 @@ pub fn load(path: &Path, edge: u32, crop: Option, frames: Option) - } /// Whether the start of a file says it may move: a GIF, a WebP marked animated, a PNG with an -/// animation chunk before its pixels. Only those are read whole. +/// animation chunk before its pixels, or a possible Lottie JSON object. Only those +/// are read whole; Lottie parsing applies its own stricter input limit. fn might_move(path: &Path) -> bool { use std::io::Read; let mut head = vec![0u8; 4096]; let read = std::fs::File::open(path).and_then(|mut file| file.read(&mut head)).unwrap_or(0); let head = &head[..read]; - head.starts_with(b"GIF8") + crate::lottie::candidate(head) + || head.starts_with(b"GIF8") || (head.starts_with(b"RIFF") && head.get(8..16) == Some(b"WEBPVP8X") && head.get(20).is_some_and(|flags| flags & 0x02 != 0)) || (head.starts_with(b"\x89PNG") && head.windows(4).take_while(|chunk| *chunk != b"IDAT").any(|chunk| chunk == b"acTL")) } @@ -706,6 +712,20 @@ mod tests { } } + #[test] + fn a_cached_lottie_sticker_animates_or_keeps_its_first_frame() { + let path = std::env::temp_dir().join(format!("tc-lottie-{}", std::process::id())); + std::fs::write(&path, include_bytes!("../../../tests/fixtures/moving.lottie")).unwrap(); + let (playing, moves) = load(&path, 16, None, Some(frames::SMALL)).unwrap(); + assert_eq!((playing.frame_count(), moves), (4, Moves::Playing)); + anim::MESSAGES.give(anim::bytes_of(&playing)); + for limits in [None, Some(Limits { bytes: 0, frames: 2 })] { + let (still, moves) = load(&path, 16, None, limits).unwrap(); + assert_eq!((still.frame_count(), moves), (1, Moves::Still)); + } + std::fs::remove_file(path).unwrap(); + } + #[test] fn a_moving_picture_is_read_for_frames_only_when_they_are_wanted() { // Only these are read whole: a GIF, a WebP marked animated, a PNG with an animation chunk. diff --git a/crates/tinyclient/src/lottie.rs b/crates/tinyclient/src/lottie.rs new file mode 100644 index 0000000..001b4d4 --- /dev/null +++ b/crates/tinyclient/src/lottie.rs @@ -0,0 +1,192 @@ +//! Native Lottie stickers, rendered off the UI thread into the existing frame cache. +//! tlottie never resolves external resources. Input, canvas and retained frames are bounded. + +use crate::frames::{self, Animation, Limits, Sniff}; +use std::io::Read; +use std::path::Path; +use std::time::{Duration, Instant}; + +const MAX_INPUT: usize = 2 << 20; +const MAX_EDGE: u32 = 512; + +pub fn candidate(bytes: &[u8]) -> bool { + bytes.iter().copied().find(|b| !b.is_ascii_whitespace()) == Some(b'{') +} + +pub fn read(path: &Path) -> Option> { + let mut file = std::fs::File::open(path).ok()?; + let mut head = [0; 64]; + let n = file.read(&mut head).ok()?; + if !candidate(&head[..n]) { + return None; + } + use std::io::Seek; + file.rewind().ok()?; + let mut bytes = Vec::new(); + file.take((MAX_INPUT + 1) as u64) + .read_to_end(&mut bytes) + .ok()?; + (bytes.len() <= MAX_INPUT).then_some(bytes) +} + +pub fn sniff(bytes: &[u8]) -> Option { + if bytes.len() > MAX_INPUT || !candidate(bytes) { + return None; + } + let header: serde_json::Value = serde_json::from_slice(bytes).ok()?; + let width = u32::try_from(header["w"].as_u64()?).ok()?; + let height = u32::try_from(header["h"].as_u64()?).ok()?; + let rate = header["fr"].as_f64()?; + let count = (header["op"].as_f64()? - header["ip"].as_f64()?).ceil(); + header["layers"].as_array()?; + if !(1..=4096).contains(&width) + || !(1..=4096).contains(&height) + || !(0.1..=240.).contains(&rate) + || !(1.0..=1000.0).contains(&count) + { + return None; + } + Some(Sniff { + width, + height, + frames: count as usize, + plays: None, + }) +} + +pub fn decode( + bytes: &[u8], + edge: u32, + crop: Option, + limits: Option, +) -> Option { + let sniff = sniff(bytes)?; + let parse_limits = tlottie::Limits { + max_input_bytes: MAX_INPUT, + max_dimension: 4096, + max_parse_bytes: 64 << 20, + ..Default::default() + }; + let composition = tlottie::Composition::parse(bytes, &parse_limits).ok()?; + let moving = limits.is_some() && !composition.is_static() && sniff.frames > 1; + let edge = edge.clamp(1, MAX_EDGE).min(sniff.width.max(sniff.height)); + let (width, height, step) = if moving { + let plan = frames::plan(&sniff, edge, crop, limits?)?; + (plan.width, plan.height, plan.step) + } else { + let (width, height) = frames::scaled((sniff.width, sniff.height), edge, crop); + (width, height, 1) + }; + // Render the full aspect ratio, then keep the top rows for CoverTop. + let full_height = ((u64::from(sniff.height) * u64::from(width) / u64::from(sniff.width)).max(1) + as u32) + .max(height); + let count = if moving { sniff.frames } else { 1 }; + let rate = f64::from(composition.frame_rate); + let first = composition.in_point; + let mut renderer = tlottie::CPURenderer::new(composition); + let mut pixels = vec![0u32; width as usize * full_height as usize]; + let mut kept = Vec::new(); + let started = Instant::now(); + for at in (0..count).step_by(step) { + if started.elapsed() > Duration::from_secs(2) { + return None; + } + renderer + .render( + first + at as f32, + &mut pixels, + width, + full_height, + tlottie::RenderOptions::default(), + ) + .ok()?; + let mut bgra = Vec::with_capacity(width as usize * height as usize * 4); + for &pixel in pixels.iter().take(width as usize * height as usize) { + let alpha = pixel >> 24; + let straight = |channel: u32| { + if alpha == 0 { + 0 + } else { + ((channel * 255 + alpha / 2) / alpha).min(255) as u8 + } + }; + bgra.extend([ + straight((pixel >> 16) & 255), + straight((pixel >> 8) & 255), + straight(pixel & 255), + alpha as u8, + ]); + } + let held = Duration::from_secs_f64((step.min(count - at)) as f64 / rate); + kept.push((bgra, held)); + } + Some(Animation { + width, + height, + frames: kept, + plays: None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + fn fixture() -> Vec { + include_bytes!("../tests/fixtures/moving.lottie").to_vec() + } + #[test] + fn native_frames_move_and_have_straight_bgra_and_correct_timing() { + let bytes = fixture(); + let image = decode(&bytes, 16, None, Some(frames::SMALL)).unwrap(); + assert_eq!((image.width, image.height, image.frames.len()), (16, 16, 4)); + assert_ne!(image.frames[0].0, image.frames[3].0); + assert!( + image.frames[0] + .0 + .chunks_exact(4) + .any(|p| p[0] == 0 && p[2] == 255 && (120..=130).contains(&p[3])) + ); + assert!( + image + .frames + .iter() + .all(|(_, delay)| *delay == Duration::from_millis(100)) + ); + assert!(image.plays.is_none()); + let still = decode(&bytes, 16, None, None).unwrap(); + assert_eq!(still.frames.len(), 1); + assert_eq!(still.frames[0].0, image.frames[0].0); + } + #[test] + fn cropped_portrait_stickers_keep_the_original_aspect_ratio() { + let mut value: serde_json::Value = serde_json::from_slice(&fixture()).unwrap(); + value["h"] = 32.into(); + let bytes = serde_json::to_vec(&value).unwrap(); + let full = decode(&bytes, 32, None, None).unwrap(); + let cropped = decode(&bytes, 32, Some(0.5), None).unwrap(); + assert_eq!((full.width, full.height), (16, 32)); + assert_eq!((cropped.width, cropped.height), (16, 8)); + assert_eq!(cropped.frames[0].0, full.frames[0].0[..16 * 8 * 4]); + } + + #[test] + fn invalid_and_oversized_inputs_are_rejected_and_frame_budgets_hold() { + for bytes in [b"{}".as_slice(), b"{not json}", b"[]"] { + assert!(decode(bytes, 16, None, None).is_none()); + } + let mut value: serde_json::Value = serde_json::from_slice(&fixture()).unwrap(); + value["w"] = 100000.into(); + assert!(sniff(&serde_json::to_vec(&value).unwrap()).is_none()); + let limits = Limits { + bytes: 16 * 16 * 4 * 2, + frames: 2, + }; + let image = decode(&fixture(), 16, None, Some(limits)).unwrap(); + assert!(image.bytes() <= limits.bytes && image.frames.len() <= limits.frames); + assert_eq!( + image.frames.iter().map(|(_, d)| *d).sum::(), + Duration::from_millis(400) + ); + } +} diff --git a/crates/tinyclient/src/main.rs b/crates/tinyclient/src/main.rs index 107ad3a..28d9dfe 100644 --- a/crates/tinyclient/src/main.rs +++ b/crates/tinyclient/src/main.rs @@ -10,6 +10,8 @@ mod update; mod frames; #[cfg(feature = "gui")] mod gui; +#[cfg(feature = "gui")] +mod lottie; mod view; use client::ask; diff --git a/crates/tinyclient/tests/fixtures/moving.lottie b/crates/tinyclient/tests/fixtures/moving.lottie new file mode 100644 index 0000000..933dd95 --- /dev/null +++ b/crates/tinyclient/tests/fixtures/moving.lottie @@ -0,0 +1,63 @@ +{ + "v": "5.6.2", + "w": 16, + "h": 16, + "fr": 10, + "ip": 0, + "op": 4, + "layers": [ + { + "ty": 1, + "ind": 1, + "sw": 8, + "sh": 8, + "sc": "#ff0000", + "ip": 0, + "op": 4, + "st": 0, + "ks": { + "p": { + "a": 1, + "k": [ + { + "t": 0, + "s": [ + 0, + 0 + ], + "e": [ + 8, + 0 + ] + }, + { + "t": 4, + "s": [ + 8, + 0 + ] + } + ] + }, + "a": { + "a": 0, + "k": [ + 0, + 0 + ] + }, + "s": { + "a": 0, + "k": [ + 100, + 100 + ] + }, + "o": { + "a": 0, + "k": 50 + } + } + } + ] +} diff --git a/docs/design/polish/06-composer.md b/docs/design/polish/06-composer.md index 721a6bc..63b3557 100644 --- a/docs/design/polish/06-composer.md +++ b/docs/design/polish/06-composer.md @@ -185,6 +185,19 @@ removing a file (Discord) and replacing a photo (Telegram) are P2. - **Still-image decode**: media cache filenames have no extension. On Linux and Windows, detect the image format from its bytes, as on macOS. Thumbnail decoding shrinks large sources but never enlarges a smaller source to the requested edge. + +- **Lottie stickers**: tlottie's CPU renderer parses cached Discord JSON and + supplies frames to the same image cache as GIF/APNG. The GUI-only dependency + is pinned in Cargo.lock; daemons and non-GUI builds do not link it. The picker + registers Lottie JSON with the same sticker media id used by received messages. + Decode runs off the UI thread, respects autoplay/Reduce Motion and existing + frame budgets, and retains frame zero when animation is disabled or cannot fit. + Inputs are at most 2 MiB, source dimensions 4096, duration 1000 source frames, + and rate 0.1–240 fps. Rendering is capped at a 512-pixel edge, retains at most + the cache's frame/byte allowance, and stops between frames after two seconds. + Transparent premultiplied RGBA output becomes straight BGRA for GPUI. External + files/fonts named in animation JSON are never loaded. Telegram TGS support is + unchanged; this path handles JSON Lottie stickers. - **GIF decode rule**: every cell is a first-frame still (`images.rs` reads index 0 only). The one hovered cell swaps to `img()` of the provider's smallest GIF rendition (Tenor `tinygif`, ~220 px): this rev's `img()` steps frames itself and stops under `reduce_motion()`. Played only if width × height × 4 × frames ≤ 8 MB, else it stays still; dropped