Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion crates/chatd/src/adapters/discord/content.rs
Original file line number Diff line number Diff line change
Expand Up @@ -519,7 +519,7 @@ fn attachment(rich: &mut Rich, attached: &Attached) -> Attachment {
}
}

/// A sticker is drawn like an attached picture. Lottie ones are a file nobody can draw here.
/// A sticker is drawn like an attached picture; the shell renders Lottie JSON with tlottie.
fn sticker(rich: &mut Rich, sticker: &model::Sticker) -> Attachment {
let (kind, url, mime) = match sticker.format_type {
3 => (MediaKind::File, format!("{}/stickers/{}.json", model::cdn(), sticker.id), "application/json"),
Expand Down
3 changes: 2 additions & 1 deletion crates/chatd/src/adapters/discord/model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -457,11 +457,12 @@ fn nullable_yes<'de, D: Deserializer<'de>>(de: D) -> Result<bool, D::Error> {
}

/// A sticker's picture from the media proxy, registered as a message shows it (`content.rs`), so
/// the picker and a sent sticker share one file. `None` for a Lottie one, which has no still.
/// the picker and a sent sticker share one file. Lottie JSON is rendered by the shell.
pub fn sticker(id: &str, format: u8) -> Option<(MediaFile, String)> {
let (kind, url) = match format {
1 => (MediaKind::Image, format!("{}/stickers/{id}.png?size=160", media())),
2 => (MediaKind::Image, apng_sticker(id)),
3 => (MediaKind::File, format!("{}/stickers/{id}.json", cdn())),
4 => (MediaKind::Gif, format!("{}/stickers/{id}.gif", media())),
_ => return None,
};
Expand Down
7 changes: 4 additions & 3 deletions crates/chatd/tests/discord.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2742,14 +2742,15 @@ fn files_go_up_from_their_paths_with_the_senders_options_and_a_cancel_stops_them
assert_eq!(posted[1].body["content"], "https://tenor.com/view/cat-typing-1");
assert_eq!(posted[1].body["allowed_mentions"]["replied_user"], true);
// The server's own emoji and stickers, from READY: what lost its boost, an animated one
// (Nitro's), one only another role may use and the Lottie sticker are not offered, and a
// DM offers none without Nitro. Nothing is asked.
// (Nitro's), and one only another role may use are not offered. Lottie stickers
// have a native renderer; a DM still offers no server emoji without Nitro. Nothing is asked.
let before = world.discord.asked().len();
let emoji = catalog(protocol::CatalogKind::Emoji, "", Some("4000"));
assert_eq!(emoji.iter().map(|item| (item.id.as_str(), item.name.as_str())).collect::<Vec<_>>(), [("emoji:5101", ":ferris:")]);
assert!(catalog(protocol::CatalogKind::Emoji, "", None).is_empty());
let stickers = catalog(protocol::CatalogKind::Stickers, "", Some("4000"));
assert_eq!(stickers.iter().map(|item| item.id.as_str()).collect::<Vec<_>>(), ["5201"]);
assert_eq!(stickers.iter().map(|item| item.id.as_str()).collect::<Vec<_>>(), ["5201", "5202"]);
assert_eq!(stickers[1].file.as_ref().unwrap().kind, protocol::MediaKind::File, "Lottie JSON is offered to the native renderer");
assert_eq!(world.discord.asked().len(), before);
let pick = protocol::Pick { kind: protocol::CatalogKind::Stickers, id: "5201".into() };
send_with(&world, "4101", "77005", "", None, protocol::Outgoing { pick: Some(pick), ..Default::default() });
Expand Down
5 changes: 4 additions & 1 deletion crates/tinyclient/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,15 @@ serde_json = { workspace = true, optional = true }
ureq = { version = "3", default-features = false, features = ["native-tls"], optional = true }
url = { version = "2", optional = true }

# Native CPU rendering of Discord Lottie stickers, pinned for reproducible builds.
tlottie = { git = "https://github.com/dkaraush/tlottie", rev = "92df98dc209bc39b1e567ec74a8c86a0af5239de", optional = true }

[dev-dependencies]
# The frame decoder's tests run without the window.
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "gif", "webp", "bmp"] }

[features]
gui = ["dep:ureq", "dep:url", "dep:gpui", "dep:gpui_platform", "dep:unicode-segmentation", "dep:libc", "dep:image", "dep:smallvec", "dep:serde_json", "dep:windows-sys"]
gui = ["dep:tlottie", "dep:ureq", "dep:url", "dep:gpui", "dep:gpui_platform", "dep:unicode-segmentation", "dep:libc", "dep:image", "dep:smallvec", "dep:serde_json", "dep:windows-sys"]
# Adds `--shot out.png`: render the window to a file and quit, for design review without
# Screen Recording permission. Pulls GPUI's test support, so it is not part of `gui`.
gui-shot = ["gui", "gpui/test-support", "gpui_platform/test-support"]
Expand Down
14 changes: 10 additions & 4 deletions crates/tinyclient/src/frames.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! Moving pictures: animated GIF, animated WebP and APNG, decoded to frames at the size they
//! Moving pictures: animated GIF, WebP, APNG and (with the GUI) Lottie, decoded at the size they
//! are drawn. A file's header is read first, without decoding, for its canvas, frame count and
//! how many times it plays; that decides the size and how many frames are kept so one
//! animation never passes its [`Limits`]. Frames are scaled down one at a time as they are
Expand Down Expand Up @@ -42,7 +42,7 @@ pub struct Sniff {
}

/// Reads a file's header for whether it moves. `None` for a still picture, one frame, or a
/// file that is not one of the three formats.
/// file that is not a supported animation format.
pub fn sniff(bytes: &[u8]) -> Option<Sniff> {
let found = if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") {
gif(bytes)
Expand All @@ -51,7 +51,10 @@ pub fn sniff(bytes: &[u8]) -> Option<Sniff> {
} else if bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
apng(bytes)
} else {
None
#[cfg(feature = "gui")]
{ crate::lottie::sniff(bytes) }
#[cfg(not(feature = "gui"))]
{ None }
}?;
(found.frames > 1 && found.width > 0 && found.height > 0).then_some(found)
}
Expand Down Expand Up @@ -163,7 +166,7 @@ pub struct Plan {

/// The pixels of a `canvas` scaled to `edge` on its longer side, then cut to `crop` times its
/// width from the top.
fn scaled(canvas: (u32, u32), edge: u32, crop: Option<f32>) -> (u32, u32) {
pub(crate) fn scaled(canvas: (u32, u32), edge: u32, crop: Option<f32>) -> (u32, u32) {
let (w, h) = (u64::from(canvas.0.max(1)), u64::from(canvas.1.max(1)));
let edge = u64::from(edge.max(1));
let (width, height) = if w >= h { (edge, (h * edge / w).max(1)) } else { ((w * edge / h).max(1), edge) };
Expand Down Expand Up @@ -230,6 +233,9 @@ fn shown_for(delay: image::Delay) -> Duration {
/// Decodes the frames of a moving picture per its plan. `None` when it does not move, does not
/// fit, or will not decode.
pub fn decode(bytes: &[u8], edge: u32, crop: Option<f32>, limits: Limits) -> Option<Animation> {
#[cfg(feature = "gui")]
if crate::lottie::candidate(bytes) { return crate::lottie::decode(bytes, edge, crop, Some(limits)); }

let sniff = sniff(bytes)?;
let plan = plan(&sniff, edge, crop, limits)?;
let source = || Cursor::new(bytes);
Expand Down
24 changes: 22 additions & 2 deletions crates/tinyclient/src/gui/conversation/images.rs
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,10 @@ pub struct Decoded {
/// whole. `crop` keeps only the top of a very tall picture: that many times its width.
/// `None` when the file is missing or is not a picture.
pub fn decode(path: &Path, edge: u32, crop: Option<f32>) -> Option<Decoded> {
if let Some(bytes) = crate::lottie::read(path) {
let mut image = crate::lottie::decode(&bytes, edge, crop, None)?;
return Some(Decoded { width: image.width, height: image.height, bgra: image.frames.pop()?.0 });
}
if !looks_like_a_picture(path) {
return None;
}
Expand Down Expand Up @@ -288,13 +292,15 @@ pub fn load(path: &Path, edge: u32, crop: Option<f32>, frames: Option<Limits>) -
}

/// Whether the start of a file says it may move: a GIF, a WebP marked animated, a PNG with an
/// animation chunk before its pixels. Only those are read whole.
/// animation chunk before its pixels, or a possible Lottie JSON object. Only those
/// are read whole; Lottie parsing applies its own stricter input limit.
fn might_move(path: &Path) -> bool {
use std::io::Read;
let mut head = vec![0u8; 4096];
let read = std::fs::File::open(path).and_then(|mut file| file.read(&mut head)).unwrap_or(0);
let head = &head[..read];
head.starts_with(b"GIF8")
crate::lottie::candidate(head)
|| head.starts_with(b"GIF8")
|| (head.starts_with(b"RIFF") && head.get(8..16) == Some(b"WEBPVP8X") && head.get(20).is_some_and(|flags| flags & 0x02 != 0))
|| (head.starts_with(b"\x89PNG") && head.windows(4).take_while(|chunk| *chunk != b"IDAT").any(|chunk| chunk == b"acTL"))
}
Expand Down Expand Up @@ -706,6 +712,20 @@ mod tests {
}
}

#[test]
fn a_cached_lottie_sticker_animates_or_keeps_its_first_frame() {
let path = std::env::temp_dir().join(format!("tc-lottie-{}", std::process::id()));
std::fs::write(&path, include_bytes!("../../../tests/fixtures/moving.lottie")).unwrap();
let (playing, moves) = load(&path, 16, None, Some(frames::SMALL)).unwrap();
assert_eq!((playing.frame_count(), moves), (4, Moves::Playing));
anim::MESSAGES.give(anim::bytes_of(&playing));
for limits in [None, Some(Limits { bytes: 0, frames: 2 })] {
let (still, moves) = load(&path, 16, None, limits).unwrap();
assert_eq!((still.frame_count(), moves), (1, Moves::Still));
}
std::fs::remove_file(path).unwrap();
}

#[test]
fn a_moving_picture_is_read_for_frames_only_when_they_are_wanted() {
// Only these are read whole: a GIF, a WebP marked animated, a PNG with an animation chunk.
Expand Down
192 changes: 192 additions & 0 deletions crates/tinyclient/src/lottie.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
//! Native Lottie stickers, rendered off the UI thread into the existing frame cache.
//! tlottie never resolves external resources. Input, canvas and retained frames are bounded.

use crate::frames::{self, Animation, Limits, Sniff};
use std::io::Read;
use std::path::Path;
use std::time::{Duration, Instant};

const MAX_INPUT: usize = 2 << 20;
const MAX_EDGE: u32 = 512;

pub fn candidate(bytes: &[u8]) -> bool {
bytes.iter().copied().find(|b| !b.is_ascii_whitespace()) == Some(b'{')
}

pub fn read(path: &Path) -> Option<Vec<u8>> {
let mut file = std::fs::File::open(path).ok()?;
let mut head = [0; 64];
let n = file.read(&mut head).ok()?;
if !candidate(&head[..n]) {
return None;
}
use std::io::Seek;
file.rewind().ok()?;
let mut bytes = Vec::new();
file.take((MAX_INPUT + 1) as u64)
.read_to_end(&mut bytes)
.ok()?;
(bytes.len() <= MAX_INPUT).then_some(bytes)
}

pub fn sniff(bytes: &[u8]) -> Option<Sniff> {
if bytes.len() > MAX_INPUT || !candidate(bytes) {
return None;
}
let header: serde_json::Value = serde_json::from_slice(bytes).ok()?;
let width = u32::try_from(header["w"].as_u64()?).ok()?;
let height = u32::try_from(header["h"].as_u64()?).ok()?;
let rate = header["fr"].as_f64()?;
let count = (header["op"].as_f64()? - header["ip"].as_f64()?).ceil();
header["layers"].as_array()?;
if !(1..=4096).contains(&width)
|| !(1..=4096).contains(&height)
|| !(0.1..=240.).contains(&rate)
|| !(1.0..=1000.0).contains(&count)
{
return None;
}
Some(Sniff {
width,
height,
frames: count as usize,
plays: None,
})
}

pub fn decode(
bytes: &[u8],
edge: u32,
crop: Option<f32>,
limits: Option<Limits>,
) -> Option<Animation> {
let sniff = sniff(bytes)?;
let parse_limits = tlottie::Limits {
max_input_bytes: MAX_INPUT,
max_dimension: 4096,
max_parse_bytes: 64 << 20,
..Default::default()
};
let composition = tlottie::Composition::parse(bytes, &parse_limits).ok()?;
let moving = limits.is_some() && !composition.is_static() && sniff.frames > 1;
let edge = edge.clamp(1, MAX_EDGE).min(sniff.width.max(sniff.height));
let (width, height, step) = if moving {
let plan = frames::plan(&sniff, edge, crop, limits?)?;
(plan.width, plan.height, plan.step)
} else {
let (width, height) = frames::scaled((sniff.width, sniff.height), edge, crop);
(width, height, 1)
};
// Render the full aspect ratio, then keep the top rows for CoverTop.
let full_height = ((u64::from(sniff.height) * u64::from(width) / u64::from(sniff.width)).max(1)
as u32)
.max(height);
let count = if moving { sniff.frames } else { 1 };
let rate = f64::from(composition.frame_rate);
let first = composition.in_point;
let mut renderer = tlottie::CPURenderer::new(composition);
let mut pixels = vec![0u32; width as usize * full_height as usize];
let mut kept = Vec::new();
let started = Instant::now();
for at in (0..count).step_by(step) {
if started.elapsed() > Duration::from_secs(2) {
return None;
}
renderer
.render(
first + at as f32,
&mut pixels,
width,
full_height,
tlottie::RenderOptions::default(),
)
.ok()?;
let mut bgra = Vec::with_capacity(width as usize * height as usize * 4);
for &pixel in pixels.iter().take(width as usize * height as usize) {
let alpha = pixel >> 24;
let straight = |channel: u32| {
if alpha == 0 {
0
} else {
((channel * 255 + alpha / 2) / alpha).min(255) as u8
}
};
bgra.extend([
straight((pixel >> 16) & 255),
straight((pixel >> 8) & 255),
straight(pixel & 255),
alpha as u8,
]);
}
let held = Duration::from_secs_f64((step.min(count - at)) as f64 / rate);
kept.push((bgra, held));
}
Some(Animation {
width,
height,
frames: kept,
plays: None,
})
}

#[cfg(test)]
mod tests {
use super::*;
fn fixture() -> Vec<u8> {
include_bytes!("../tests/fixtures/moving.lottie").to_vec()
}
#[test]
fn native_frames_move_and_have_straight_bgra_and_correct_timing() {
let bytes = fixture();
let image = decode(&bytes, 16, None, Some(frames::SMALL)).unwrap();
assert_eq!((image.width, image.height, image.frames.len()), (16, 16, 4));
assert_ne!(image.frames[0].0, image.frames[3].0);
assert!(
image.frames[0]
.0
.chunks_exact(4)
.any(|p| p[0] == 0 && p[2] == 255 && (120..=130).contains(&p[3]))
);
assert!(
image
.frames
.iter()
.all(|(_, delay)| *delay == Duration::from_millis(100))
);
assert!(image.plays.is_none());
let still = decode(&bytes, 16, None, None).unwrap();
assert_eq!(still.frames.len(), 1);
assert_eq!(still.frames[0].0, image.frames[0].0);
}
#[test]
fn cropped_portrait_stickers_keep_the_original_aspect_ratio() {
let mut value: serde_json::Value = serde_json::from_slice(&fixture()).unwrap();
value["h"] = 32.into();
let bytes = serde_json::to_vec(&value).unwrap();
let full = decode(&bytes, 32, None, None).unwrap();
let cropped = decode(&bytes, 32, Some(0.5), None).unwrap();
assert_eq!((full.width, full.height), (16, 32));
assert_eq!((cropped.width, cropped.height), (16, 8));
assert_eq!(cropped.frames[0].0, full.frames[0].0[..16 * 8 * 4]);
}

#[test]
fn invalid_and_oversized_inputs_are_rejected_and_frame_budgets_hold() {
for bytes in [b"{}".as_slice(), b"{not json}", b"[]"] {
assert!(decode(bytes, 16, None, None).is_none());
}
let mut value: serde_json::Value = serde_json::from_slice(&fixture()).unwrap();
value["w"] = 100000.into();
assert!(sniff(&serde_json::to_vec(&value).unwrap()).is_none());
let limits = Limits {
bytes: 16 * 16 * 4 * 2,
frames: 2,
};
let image = decode(&fixture(), 16, None, Some(limits)).unwrap();
assert!(image.bytes() <= limits.bytes && image.frames.len() <= limits.frames);
assert_eq!(
image.frames.iter().map(|(_, d)| *d).sum::<Duration>(),
Duration::from_millis(400)
);
}
}
2 changes: 2 additions & 0 deletions crates/tinyclient/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ mod update;
mod frames;
#[cfg(feature = "gui")]
mod gui;
#[cfg(feature = "gui")]
mod lottie;
mod view;

use client::ask;
Expand Down
Loading
Loading