From eb8204d69b7dab94cb1caf6c26664a16dcc843dc Mon Sep 17 00:00:00 2001 From: Surma Date: Tue, 18 May 2021 19:53:39 +0100 Subject: [PATCH 1/9] Add a call-out to Web Codecs API --- src/client/lazy-app/Compress/index.tsx | 7 ++++- src/client/lazy-app/util/web-codecs/index.ts | 26 +++++++++++++++++ .../util/web-codecs/missing-types.d.ts | 28 +++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 src/client/lazy-app/util/web-codecs/index.ts create mode 100644 src/client/lazy-app/util/web-codecs/missing-types.d.ts diff --git a/src/client/lazy-app/Compress/index.tsx b/src/client/lazy-app/Compress/index.tsx index 19e30fcb..cb5b0ee8 100644 --- a/src/client/lazy-app/Compress/index.tsx +++ b/src/client/lazy-app/Compress/index.tsx @@ -34,6 +34,7 @@ import { resize } from 'features/processors/resize/client'; import type SnackBarElement from 'shared/custom-els/snack-bar'; import { Arrow, ExpandIcon } from '../icons'; import { generateCliInvocation } from '../util/cli'; +import * as WebCodecs from '../util/web-codecs'; export type OutputType = EncoderType | 'identity'; @@ -109,8 +110,12 @@ async function decodeImage( if (mimeType === 'image/webp2') { return await workerBridge.wp2Decode(signal, blob); } - // If it's not one of those types, fall through and try built-in decoding for a laugh. } + // If none of those work, let’s see if Web Codecs API is available + if (await WebCodecs.isTypeSupported(mimeType)) { + return await abortable(signal, WebCodecs.decode(blob, mimeType)); + } + // Otherwise fall through and try built-in decoding for a laugh. return await abortable(signal, builtinDecode(blob)); } catch (err) { if (err.name === 'AbortError') throw err; diff --git a/src/client/lazy-app/util/web-codecs/index.ts b/src/client/lazy-app/util/web-codecs/index.ts new file mode 100644 index 00000000..1d946a1f --- /dev/null +++ b/src/client/lazy-app/util/web-codecs/index.ts @@ -0,0 +1,26 @@ +import { drawableToImageData } from 'client/lazy-app/util'; + +const hasImageDecoder = typeof ImageDecoder !== 'undefined'; +export async function isTypeSupported(mimeType: string): Promise { + if (!hasImageDecoder) { + return false; + } + return ImageDecoder.isTypeSupported(mimeType); +} +export async function decode( + blob: Blob | File, + mimeType: string, +): Promise { + if (!hasImageDecoder) { + throw Error( + `This browser does not support ImageDecoder. This function should not have been called.`, + ); + } + const decoder = new ImageDecoder({ + type: mimeType, + // Non-obvious way to turn an Blob into a ReadableStream + data: new Response(blob).body!, + }); + const result = await decoder.decode(); + return drawableToImageData(result.image); +} diff --git a/src/client/lazy-app/util/web-codecs/missing-types.d.ts b/src/client/lazy-app/util/web-codecs/missing-types.d.ts new file mode 100644 index 00000000..e5e72cc1 --- /dev/null +++ b/src/client/lazy-app/util/web-codecs/missing-types.d.ts @@ -0,0 +1,28 @@ +interface ImageDecoderInit { + type: string; + data: BufferSource | ReadableStream; + premultiplyAlpha?: PremultiplyAlpha; + colorSpaceConversion?: ColorSpaceConversion; + desiredWidth?: number; + desiredHeight?: number; + preferAnimation?: boolean; +} + +interface ImageDecodeOptions { + frameIndex: number; + completeFramesOnly: boolean; +} + +interface ImageDecodeResult { + image: VideoFrame; + complete: boolean; +} + +// Absolutely not correct, but it’s all we need and I’m lazy. +type VideoFrame = ImageBitmap; + +declare class ImageDecoder { + static isTypeSupported(type: string): Promise; + constructor(desc: ImageDecoderInit); + decode(opts?: Partial): Promise; +} From c417bd0a7ad034b9f9545f18678fb2259c96aabf Mon Sep 17 00:00:00 2001 From: Surma Date: Tue, 18 May 2021 20:02:43 +0100 Subject: [PATCH 2/9] Its working now --- src/client/lazy-app/util/index.ts | 28 +++++++++--- src/client/lazy-app/util/web-codecs/index.ts | 4 +- .../util/web-codecs/missing-types.d.ts | 43 ++++++++++++++++++- 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/src/client/lazy-app/util/index.ts b/src/client/lazy-app/util/index.ts index 3c6dd6bc..85ed4c4b 100644 --- a/src/client/lazy-app/util/index.ts +++ b/src/client/lazy-app/util/index.ts @@ -192,17 +192,35 @@ interface DrawableToImageDataOptions { sh?: number; } +function getWidth( + drawable: ImageBitmap | HTMLImageElement | VideoFrame, +): number { + if ('displayWidth' in drawable) { + return drawable.displayWidth; + } + return drawable.width; +} + +function getHeight( + drawable: ImageBitmap | HTMLImageElement | VideoFrame, +): number { + if ('displayHeight' in drawable) { + return drawable.displayHeight; + } + return drawable.height; +} + export function drawableToImageData( - drawable: ImageBitmap | HTMLImageElement, + drawable: ImageBitmap | HTMLImageElement | VideoFrame, opts: DrawableToImageDataOptions = {}, ): ImageData { const { - width = drawable.width, - height = drawable.height, + width = getWidth(drawable), + height = getHeight(drawable), sx = 0, sy = 0, - sw = drawable.width, - sh = drawable.height, + sw = getWidth(drawable), + sh = getHeight(drawable), } = opts; // Make canvas same size as image diff --git a/src/client/lazy-app/util/web-codecs/index.ts b/src/client/lazy-app/util/web-codecs/index.ts index 1d946a1f..c0c39291 100644 --- a/src/client/lazy-app/util/web-codecs/index.ts +++ b/src/client/lazy-app/util/web-codecs/index.ts @@ -21,6 +21,6 @@ export async function decode( // Non-obvious way to turn an Blob into a ReadableStream data: new Response(blob).body!, }); - const result = await decoder.decode(); - return drawableToImageData(result.image); + const { image } = await decoder.decode(); + return drawableToImageData(image); } diff --git a/src/client/lazy-app/util/web-codecs/missing-types.d.ts b/src/client/lazy-app/util/web-codecs/missing-types.d.ts index e5e72cc1..30246cfe 100644 --- a/src/client/lazy-app/util/web-codecs/missing-types.d.ts +++ b/src/client/lazy-app/util/web-codecs/missing-types.d.ts @@ -18,8 +18,47 @@ interface ImageDecodeResult { complete: boolean; } -// Absolutely not correct, but it’s all we need and I’m lazy. -type VideoFrame = ImageBitmap; +// I didn’t do all the types because the class is kinda complex. +// I focused on what we need. +declare class VideoFrame { + codedWidth: number; + codedHeight: number; + cropLeft: number; + cropTop: number; + cropWidth: number; + cropHeight: number; + displayWidth: number; + displayHeight: number; + + clone(): VideoFrame; + close(): void; +} + +interface CanvasDrawImage { + drawImage( + image: CanvasImageSource | VideoFrame, + dx: number, + dy: number, + ): void; + drawImage( + image: CanvasImageSource | VideoFrame, + dx: number, + dy: number, + dw: number, + dh: number, + ): void; + drawImage( + image: CanvasImageSource | VideoFrame, + sx: number, + sy: number, + sw: number, + sh: number, + dx: number, + dy: number, + dw: number, + dh: number, + ): void; +} declare class ImageDecoder { static isTypeSupported(type: string): Promise; From 8bcaeb2f78ea8a0cdd2bfbdf849cb02232e996b1 Mon Sep 17 00:00:00 2001 From: Surma Date: Tue, 18 May 2021 20:04:04 +0100 Subject: [PATCH 3/9] Simplify types a bit --- .../lazy-app/util/web-codecs/missing-types.d.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/client/lazy-app/util/web-codecs/missing-types.d.ts b/src/client/lazy-app/util/web-codecs/missing-types.d.ts index 30246cfe..6e832db8 100644 --- a/src/client/lazy-app/util/web-codecs/missing-types.d.ts +++ b/src/client/lazy-app/util/web-codecs/missing-types.d.ts @@ -20,20 +20,13 @@ interface ImageDecodeResult { // I didn’t do all the types because the class is kinda complex. // I focused on what we need. +// See https://w3c.github.io/webcodecs/#videoframe declare class VideoFrame { - codedWidth: number; - codedHeight: number; - cropLeft: number; - cropTop: number; - cropWidth: number; - cropHeight: number; displayWidth: number; displayHeight: number; - - clone(): VideoFrame; - close(): void; } +// Add VideoFrame to canvas’ drawImage() interface CanvasDrawImage { drawImage( image: CanvasImageSource | VideoFrame, From 118885cd2645184de2e07f5852c42cb6241f3e55 Mon Sep 17 00:00:00 2001 From: Surma Date: Fri, 21 May 2021 15:13:00 +0100 Subject: [PATCH 4/9] Fall through to built-in decoding --- src/client/lazy-app/Compress/index.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/client/lazy-app/Compress/index.tsx b/src/client/lazy-app/Compress/index.tsx index cb5b0ee8..6d4a9c8b 100644 --- a/src/client/lazy-app/Compress/index.tsx +++ b/src/client/lazy-app/Compress/index.tsx @@ -113,7 +113,11 @@ async function decodeImage( } // If none of those work, let’s see if Web Codecs API is available if (await WebCodecs.isTypeSupported(mimeType)) { - return await abortable(signal, WebCodecs.decode(blob, mimeType)); + let result; + try { + result = await abortable(signal, WebCodecs.decode(blob, mimeType)); + return result; + } catch (e) {} } // Otherwise fall through and try built-in decoding for a laugh. return await abortable(signal, builtinDecode(blob)); From 2f00fe2b1b7b436b0441cc1a564fd8ab2cf9372d Mon Sep 17 00:00:00 2001 From: Surma Date: Fri, 21 May 2021 15:14:22 +0100 Subject: [PATCH 5/9] =?UTF-8?q?I=E2=80=99m=20an=20idiot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/client/lazy-app/Compress/index.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/client/lazy-app/Compress/index.tsx b/src/client/lazy-app/Compress/index.tsx index 6d4a9c8b..bfb8a032 100644 --- a/src/client/lazy-app/Compress/index.tsx +++ b/src/client/lazy-app/Compress/index.tsx @@ -113,10 +113,8 @@ async function decodeImage( } // If none of those work, let’s see if Web Codecs API is available if (await WebCodecs.isTypeSupported(mimeType)) { - let result; try { - result = await abortable(signal, WebCodecs.decode(blob, mimeType)); - return result; + return await abortable(signal, WebCodecs.decode(blob, mimeType)); } catch (e) {} } // Otherwise fall through and try built-in decoding for a laugh. From 4033d1c96501381d85250bc04f7506f8b2494e07 Mon Sep 17 00:00:00 2001 From: Surma Date: Mon, 24 May 2021 15:39:38 +0100 Subject: [PATCH 6/9] Move logic to builtinDecode --- src/client/lazy-app/Compress/index.tsx | 9 +-------- src/client/lazy-app/util/index.ts | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/client/lazy-app/Compress/index.tsx b/src/client/lazy-app/Compress/index.tsx index bfb8a032..10fc4926 100644 --- a/src/client/lazy-app/Compress/index.tsx +++ b/src/client/lazy-app/Compress/index.tsx @@ -34,7 +34,6 @@ import { resize } from 'features/processors/resize/client'; import type SnackBarElement from 'shared/custom-els/snack-bar'; import { Arrow, ExpandIcon } from '../icons'; import { generateCliInvocation } from '../util/cli'; -import * as WebCodecs from '../util/web-codecs'; export type OutputType = EncoderType | 'identity'; @@ -111,14 +110,8 @@ async function decodeImage( return await workerBridge.wp2Decode(signal, blob); } } - // If none of those work, let’s see if Web Codecs API is available - if (await WebCodecs.isTypeSupported(mimeType)) { - try { - return await abortable(signal, WebCodecs.decode(blob, mimeType)); - } catch (e) {} - } // Otherwise fall through and try built-in decoding for a laugh. - return await abortable(signal, builtinDecode(blob)); + return await builtinDecode(signal, blob, mimeType); } catch (err) { if (err.name === 'AbortError') throw err; console.log(err); diff --git a/src/client/lazy-app/util/index.ts b/src/client/lazy-app/util/index.ts index 85ed4c4b..fe573804 100644 --- a/src/client/lazy-app/util/index.ts +++ b/src/client/lazy-app/util/index.ts @@ -10,6 +10,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import * as WebCodecs from '../util/web-codecs'; + /** * Compare two objects, returning a boolean indicating if * they have the same properties and strictly equal values. @@ -234,13 +237,27 @@ export function drawableToImageData( return ctx.getImageData(0, 0, width, height); } -export async function builtinDecode(blob: Blob): Promise { +export async function builtinDecode( + signal: AbortSignal, + blob: Blob, + mimeType: string, +): Promise { + // If WebCodecs are supported, use that. + if (await WebCodecs.isTypeSupported(mimeType)) { + if (signal.aborted) return Promise.reject('Aborted'); + try { + return await WebCodecs.decode(blob, mimeType); + } catch (e) {} + } + if (signal.aborted) return Promise.reject('Aborted'); + // Prefer createImageBitmap as it's the off-thread option for Firefox. const drawable = 'createImageBitmap' in self ? await createImageBitmap(blob) : await blobToImg(blob); + if (signal.aborted) return Promise.reject('Aborted'); return drawableToImageData(drawable); } From cbfa503fcbc92f36ee5cd0e05ddf1c02640a06e7 Mon Sep 17 00:00:00 2001 From: Surma Date: Mon, 24 May 2021 16:26:05 +0100 Subject: [PATCH 7/9] Update src/client/lazy-app/util/index.ts Co-authored-by: Jake Archibald --- src/client/lazy-app/util/index.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/client/lazy-app/util/index.ts b/src/client/lazy-app/util/index.ts index fe573804..7f8917d9 100644 --- a/src/client/lazy-app/util/index.ts +++ b/src/client/lazy-app/util/index.ts @@ -249,15 +249,13 @@ export async function builtinDecode( return await WebCodecs.decode(blob, mimeType); } catch (e) {} } - if (signal.aborted) return Promise.reject('Aborted'); - + assertSignal(signal); + // Prefer createImageBitmap as it's the off-thread option for Firefox. - const drawable = - 'createImageBitmap' in self - ? await createImageBitmap(blob) - : await blobToImg(blob); - - if (signal.aborted) return Promise.reject('Aborted'); + const drawable = await abortable( + signal, + 'createImageBitmap' in self ? createImageBitmap(blob) : blobToImg(blob), + ); return drawableToImageData(drawable); } From 58661078e2380cf509f4f668edab4062902c1132 Mon Sep 17 00:00:00 2001 From: Surma Date: Mon, 24 May 2021 16:26:20 +0100 Subject: [PATCH 8/9] Update src/client/lazy-app/util/index.ts Co-authored-by: Jake Archibald --- src/client/lazy-app/util/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/lazy-app/util/index.ts b/src/client/lazy-app/util/index.ts index 7f8917d9..a7e86944 100644 --- a/src/client/lazy-app/util/index.ts +++ b/src/client/lazy-app/util/index.ts @@ -244,9 +244,9 @@ export async function builtinDecode( ): Promise { // If WebCodecs are supported, use that. if (await WebCodecs.isTypeSupported(mimeType)) { - if (signal.aborted) return Promise.reject('Aborted'); + assertSignal(signal); try { - return await WebCodecs.decode(blob, mimeType); + return await abortable(signal, WebCodecs.decode(blob, mimeType)); } catch (e) {} } assertSignal(signal); From 128096afd936b93be94bbebddb902e2fe40a01eb Mon Sep 17 00:00:00 2001 From: Surma Date: Mon, 24 May 2021 16:29:12 +0100 Subject: [PATCH 9/9] Make TS happy --- src/client/lazy-app/util/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/lazy-app/util/index.ts b/src/client/lazy-app/util/index.ts index a7e86944..317dc059 100644 --- a/src/client/lazy-app/util/index.ts +++ b/src/client/lazy-app/util/index.ts @@ -250,9 +250,9 @@ export async function builtinDecode( } catch (e) {} } assertSignal(signal); - + // Prefer createImageBitmap as it's the off-thread option for Firefox. - const drawable = await abortable( + const drawable = await abortable( signal, 'createImageBitmap' in self ? createImageBitmap(blob) : blobToImg(blob), );