Skip to content

Commit 5ef4f86

Browse files
TyriarTyriar
authored
Merge pull request #5846 from xtermjs/qoi_decoder
QOI support for IIP
2 parents 874e767 + 4dc934c commit 5ef4f86

6 files changed

Lines changed: 108 additions & 36 deletions

File tree

‎addons/addon-image/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,6 @@
2525
},
2626
"devDependencies": {
2727
"sixel": "^0.16.0",
28-
"xterm-wasm-parts": "^0.3.0"
28+
"xterm-wasm-parts": "^0.4.1"
2929
}
3030
}

‎addons/addon-image/src/IIPHandler.ts‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import { ImageRenderer } from './ImageRenderer';
77
import { IIPImageStorage } from './IIPImageStorage';
88
import { CELL_SIZE_DEFAULT } from './ImageStorage';
99
import Base64Decoder from 'xterm-wasm-parts/lib/base64/Base64Decoder.wasm';
10+
import QoiDecoder from 'xterm-wasm-parts/lib/qoi/QoiDecoder.wasm';
1011
import { HeaderParser, IHeaderFields, HeaderState } from './IIPHeaderParser';
1112
import { imageType, UNSUPPORTED_TYPE } from './IIPMetrics';
1213

@@ -36,6 +37,7 @@ export class IIPHandler implements IOscHandler, IResetHandler {
3637
private _hp = new HeaderParser();
3738
private _header: IHeaderFields = DEFAULT_HEADER;
3839
private _dec: Base64Decoder;
40+
private _qoiDec: QoiDecoder;
3941
private _metrics = UNSUPPORTED_TYPE;
4042

4143
constructor(
@@ -47,6 +49,7 @@ export class IIPHandler implements IOscHandler, IResetHandler {
4749
const maxEncodedBytes = Math.ceil(this._opts.iipSizeLimit * 4 / 3);
4850
const initialBytes = Math.min(DecoderConst.INITIAL_DATA, maxEncodedBytes);
4951
this._dec = new Base64Decoder(DecoderConst.KEEP_DATA, maxEncodedBytes, initialBytes);
52+
this._qoiDec = new QoiDecoder(DecoderConst.KEEP_DATA);
5053
}
5154

5255
public reset(): void {}
@@ -115,27 +118,27 @@ export class IIPHandler implements IOscHandler, IResetHandler {
115118
return true;
116119
}
117120

118-
// HACK: The types on Blob are too restrictive, this is a Uint8Array so the browser accepts it
119-
const blob = new Blob([this._dec.data8 as Uint8Array<ArrayBuffer>], { type: this._metrics.mime });
120-
this._dec.release();
121-
122-
if (!window.createImageBitmap) {
123-
const url = URL.createObjectURL(blob);
124-
const img = new Image();
125-
return new Promise<boolean>(r => {
126-
img.addEventListener('load', () => {
127-
URL.revokeObjectURL(url);
128-
const canvas = ImageRenderer.createCanvas(window.document, w, h);
129-
canvas.getContext('2d')?.drawImage(img, 0, 0, w, h);
130-
this._storage.addImage(canvas);
131-
r(true);
132-
});
133-
img.src = url;
134-
// sanity measure to avoid terminal blocking from dangling promise
135-
// happens from corrupt data (onload never gets fired)
136-
setTimeout(() => r(true), 1000);
137-
});
121+
let blob: Blob | ImageData;
122+
if (this._metrics.mime === 'image/qoi') {
123+
const data = this._qoiDec.decode(this._dec.data8);
124+
blob = new ImageData(
125+
new Uint8ClampedArray(data.buffer, data.byteOffset, data.byteLength),
126+
this._qoiDec.width,
127+
this._qoiDec.height
128+
);
129+
this._qoiDec.release();
130+
if (w === this._qoiDec.width && h === this._qoiDec.height) {
131+
// use fast-path if we don't need to rescale
132+
this._dec.release();
133+
const canvas = ImageRenderer.createCanvas(undefined, this._qoiDec.width, this._qoiDec.height);
134+
canvas.getContext('2d')?.putImageData(blob, 0, 0);
135+
this._storage.addImage(canvas);
136+
return true;
137+
}
138+
} else {
139+
blob = new Blob([this._dec.data8], { type: this._metrics.mime });
138140
}
141+
this._dec.release();
139142
return createImageBitmap(blob, { resizeWidth: w, resizeHeight: h })
140143
.then(bm => {
141144
this._storage.addImage(bm);

‎addons/addon-image/src/IIPMetrics.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
*/
55

66

7-
export type ImageType = 'image/png' | 'image/jpeg' | 'image/gif' | 'unsupported' | '';
7+
export type ImageType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/qoi' | 'unsupported' | '';
88

99
export interface IMetrics {
1010
mime: ImageType;
@@ -45,6 +45,14 @@ export function imageType(d: Uint8Array): IMetrics {
4545
height: d[9] << 8 | d[8]
4646
};
4747
}
48+
// QOI: qoif
49+
if (d32[0] === 0x66696F71) {
50+
return {
51+
mime: 'image/qoi',
52+
width: d[4] << 24 | d[5] << 16 | d[6] << 8 | d[7],
53+
height: d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11]
54+
};
55+
}
4856
return UNSUPPORTED_TYPE;
4957
}
5058

‎addons/addon-image/test/ImageAddon.test.ts‎

Lines changed: 71 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { readFileSync } from 'fs';
88
import { FINALIZER, introducer, sixelEncode } from 'sixel';
99
import { ITestContext, createTestContext, openTerminal, pollFor, timeout } from '../../../test/playwright/TestUtils';
1010
import { deepStrictEqual, ok, strictEqual } from 'assert';
11+
import QoiEncoder from 'xterm-wasm-parts/lib/qoi/QoiEncoder.wasm';
1112

1213
/**
1314
* Plugin ctor options.
@@ -74,6 +75,10 @@ const TESTDATA_IIP: [string, [number, number]][] = [
7475
[readFileSync('./addons/addon-image/fixture/iip/w3c_jpg.iip', { encoding: 'utf-8' }), [72, 48]],
7576
[readFileSync('./addons/addon-image/fixture/iip/w3c_png.iip', { encoding: 'utf-8' }), [72, 48]]
7677
];
78+
const PALETTE_PNG_BASE64 = readFileSync('./addons/addon-image/fixture/palette.png').toString('base64');
79+
const qoiEnc = new QoiEncoder(1024*1024);
80+
const qoiData = qoiEnc.encode(TESTDATA.bytes, 640, 80);
81+
const PALETTE_QOI_BASE64 = Buffer.from(qoiData).toString('base64');
7782

7883
let ctx: ITestContext;
7984
test.beforeAll(async ({ browser }) => {
@@ -85,11 +90,6 @@ test.afterAll(async () => await ctx.page.close());
8590
test.describe('ImageAddon', () => {
8691

8792
test.beforeEach(async ({}, testInfo) => {
88-
// DEBT: This test never worked on webkit
89-
if (ctx.browser.browserType().name() === 'webkit') {
90-
testInfo.skip();
91-
return;
92-
}
9393
await ctx.page.evaluate(`
9494
window.term.reset()
9595
window.imageAddon?.dispose();
@@ -312,6 +312,68 @@ test.describe('ImageAddon', () => {
312312
deepStrictEqual(await getOrigSize(1), TESTDATA_IIP[4][1]);
313313
});
314314
});
315+
316+
test.describe('IIP - QOI support', () => {
317+
test('palette should yield same bytes from PNG and QOI', async () => {
318+
await ctx.proxy.write(`\x1b]1337;File=inline=1;size=525:${PALETTE_PNG_BASE64}\x07`);
319+
deepStrictEqual(await getOrigSize(1), [640, 80]);
320+
await ctx.proxy.write(`\x1b[10H\x1b]1337;File=inline=1;size=${qoiData.length}:${PALETTE_QOI_BASE64}\x07`);
321+
deepStrictEqual(await getOrigSize(2), [640, 80]);
322+
const pngScrape = await getImageAtBufferCell(0, 0);
323+
const qoiScrape = await getImageAtBufferCell(0, 11);
324+
deepStrictEqual(qoiScrape, pngScrape);
325+
});
326+
});
327+
328+
test.describe('IIP - resizing', () => {
329+
/**
330+
* The correct resize behavior is wonky and needs to be tested against iTerm2,
331+
* especially for missing params to derive correct default behavior.
332+
* We document the current behavior here w'o claiming to be fully in line with iTerm2.
333+
* ref: https://iterm2.com/documentation-images.html
334+
* imgcat: https://iterm2.com/utilities/imgcat
335+
*
336+
* NOTE: QOI has a slightly different parse path, thus we test resizing explicitly
337+
*/
338+
const images = [
339+
['palette.png', 525, PALETTE_PNG_BASE64],
340+
['palette.qoi', qoiData.length, PALETTE_QOI_BASE64]
341+
];
342+
for (const [name, size, payload] of images) {
343+
test(name + ': N --> width=20 height=5 preserveAspectRatio=0', async () => {
344+
// cell based resize
345+
const header = 'width=20;height=5;preserveAspectRatio=0';
346+
await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header}:${payload}\x07`);
347+
const dim = await getDimensions();
348+
deepStrictEqual(await getOrigSize(1), [dim.cellWidth * 20, dim.cellHeight * 5]);
349+
});
350+
test(name + ': Npx --> width=320px height=160px preserveAspectRatio=0', async () => {
351+
// pixel based resize
352+
const header = 'width=320px;height=160px;preserveAspectRatio=0';
353+
await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header}:${payload}\x07`);
354+
deepStrictEqual(await getOrigSize(1), [320, 160]);
355+
});
356+
test(name + ': N% --> width=50% height=30% preserveAspectRatio=0', async () => {
357+
// % of viewport resize
358+
const header = 'width=50%;height=30%;preserveAspectRatio=0';
359+
await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header}:${payload}\x07`);
360+
const dim = await getDimensions();
361+
deepStrictEqual(await getOrigSize(1), [Math.floor(dim.width * 0.5), Math.floor(dim.height * 0.3)]);
362+
});
363+
test(name + ': ommitted dimension assumes preserveAspectRatio=1', async () => {
364+
// width provided in percent
365+
const header = 'width=50%';
366+
await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header}:${payload}\x07`);
367+
const dim = await getDimensions();
368+
const width = Math.floor(dim.width * 0.5);
369+
deepStrictEqual(await getOrigSize(1), [width, Math.floor(width * 80 / 640)]);
370+
// height provided in pixel
371+
const header2 = 'height=200px';
372+
await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header2}:${payload}\x07`);
373+
deepStrictEqual(await getOrigSize(2), [Math.floor(200 * 640 / 80), 200]);
374+
});
375+
}
376+
});
315377
});
316378

317379
/**
@@ -345,3 +407,7 @@ async function getOrigSize(id: number): Promise<[number, number]> {
345407
window.imageAddon._storage._images.get(${id}).orig.height
346408
]`);
347409
}
410+
411+
async function getImageAtBufferCell(x: number, y: number): Promise<string | undefined> {
412+
return ctx.page.evaluate<any>(`window.imageAddon.getImageAtBufferCell(${x}, ${y})?.toDataURL('image/png')`);
413+
}

‎addons/addon-image/test/KittyGraphics.test.ts‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,11 +109,6 @@ test.describe('Kitty Graphics Protocol', () => {
109109
// TODO: Distinguish lowercase delete selectors (placement only) from uppercase (placement + free data)
110110

111111
test.beforeEach(async ({}, testInfo) => {
112-
// DEBT: This test never worked on webkit
113-
if (ctx.browser.browserType().name() === 'webkit') {
114-
testInfo.skip();
115-
return;
116-
}
117112
await ctx.page.evaluate(`
118113
window.term.reset()
119114
window.imageAddon?.dispose();

‎package-lock.json‎

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)