diff --git a/src/client/lazy-app/Compress/index.tsx b/src/client/lazy-app/Compress/index.tsx index 97903af4..d8453c41 100644 --- a/src/client/lazy-app/Compress/index.tsx +++ b/src/client/lazy-app/Compress/index.tsx @@ -107,9 +107,9 @@ async function decodeImage( if (mimeType === 'image/webp2') { return await workerBridge.wp2Decode(signal, blob); } - // If it's not one of those types, fall through and try built-in decoding for a laugh. } - return await abortable(signal, builtinDecode(blob)); + // Otherwise fall through and try built-in decoding for a laugh. + return await builtinDecode(signal, blob, mimeType); } catch (err) { if (err.name === 'AbortError') throw err; console.log(err); diff --git a/src/client/lazy-app/util/index.ts b/src/client/lazy-app/util/index.ts index 3c6dd6bc..317dc059 100644 --- a/src/client/lazy-app/util/index.ts +++ b/src/client/lazy-app/util/index.ts @@ -10,6 +10,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import * as WebCodecs from '../util/web-codecs'; + /** * Compare two objects, returning a boolean indicating if * they have the same properties and strictly equal values. @@ -192,17 +195,35 @@ interface DrawableToImageDataOptions { sh?: number; } +function getWidth( + drawable: ImageBitmap | HTMLImageElement | VideoFrame, +): number { + if ('displayWidth' in drawable) { + return drawable.displayWidth; + } + return drawable.width; +} + +function getHeight( + drawable: ImageBitmap | HTMLImageElement | VideoFrame, +): number { + if ('displayHeight' in drawable) { + return drawable.displayHeight; + } + return drawable.height; +} + export function drawableToImageData( - drawable: ImageBitmap | HTMLImageElement, + drawable: ImageBitmap | HTMLImageElement | VideoFrame, opts: DrawableToImageDataOptions = {}, ): ImageData { const { - width = drawable.width, - height = drawable.height, + width = getWidth(drawable), + height = getHeight(drawable), sx = 0, sy = 0, - sw = drawable.width, - sh = drawable.height, + sw = getWidth(drawable), + sh = getHeight(drawable), } = opts; // Make canvas same size as image @@ -216,13 +237,25 @@ export function drawableToImageData( return ctx.getImageData(0, 0, width, height); } -export async function builtinDecode(blob: Blob): Promise { - // Prefer createImageBitmap as it's the off-thread option for Firefox. - const drawable = - 'createImageBitmap' in self - ? await createImageBitmap(blob) - : await blobToImg(blob); +export async function builtinDecode( + signal: AbortSignal, + blob: Blob, + mimeType: string, +): Promise { + // If WebCodecs are supported, use that. + if (await WebCodecs.isTypeSupported(mimeType)) { + assertSignal(signal); + try { + return await abortable(signal, WebCodecs.decode(blob, mimeType)); + } catch (e) {} + } + assertSignal(signal); + // Prefer createImageBitmap as it's the off-thread option for Firefox. + const drawable = await abortable( + signal, + 'createImageBitmap' in self ? createImageBitmap(blob) : blobToImg(blob), + ); return drawableToImageData(drawable); } diff --git a/src/client/lazy-app/util/web-codecs/index.ts b/src/client/lazy-app/util/web-codecs/index.ts new file mode 100644 index 00000000..c0c39291 --- /dev/null +++ b/src/client/lazy-app/util/web-codecs/index.ts @@ -0,0 +1,26 @@ +import { drawableToImageData } from 'client/lazy-app/util'; + +const hasImageDecoder = typeof ImageDecoder !== 'undefined'; +export async function isTypeSupported(mimeType: string): Promise { + if (!hasImageDecoder) { + return false; + } + return ImageDecoder.isTypeSupported(mimeType); +} +export async function decode( + blob: Blob | File, + mimeType: string, +): Promise { + if (!hasImageDecoder) { + throw Error( + `This browser does not support ImageDecoder. This function should not have been called.`, + ); + } + const decoder = new ImageDecoder({ + type: mimeType, + // Non-obvious way to turn an Blob into a ReadableStream + data: new Response(blob).body!, + }); + const { image } = await decoder.decode(); + return drawableToImageData(image); +} diff --git a/src/client/lazy-app/util/web-codecs/missing-types.d.ts b/src/client/lazy-app/util/web-codecs/missing-types.d.ts new file mode 100644 index 00000000..6e832db8 --- /dev/null +++ b/src/client/lazy-app/util/web-codecs/missing-types.d.ts @@ -0,0 +1,60 @@ +interface ImageDecoderInit { + type: string; + data: BufferSource | ReadableStream; + premultiplyAlpha?: PremultiplyAlpha; + colorSpaceConversion?: ColorSpaceConversion; + desiredWidth?: number; + desiredHeight?: number; + preferAnimation?: boolean; +} + +interface ImageDecodeOptions { + frameIndex: number; + completeFramesOnly: boolean; +} + +interface ImageDecodeResult { + image: VideoFrame; + complete: boolean; +} + +// I didn’t do all the types because the class is kinda complex. +// I focused on what we need. +// See https://w3c.github.io/webcodecs/#videoframe +declare class VideoFrame { + displayWidth: number; + displayHeight: number; +} + +// Add VideoFrame to canvas’ drawImage() +interface CanvasDrawImage { + drawImage( + image: CanvasImageSource | VideoFrame, + dx: number, + dy: number, + ): void; + drawImage( + image: CanvasImageSource | VideoFrame, + dx: number, + dy: number, + dw: number, + dh: number, + ): void; + drawImage( + image: CanvasImageSource | VideoFrame, + sx: number, + sy: number, + sw: number, + sh: number, + dx: number, + dy: number, + dw: number, + dh: number, + ): void; +} + +declare class ImageDecoder { + static isTypeSupported(type: string): Promise; + constructor(desc: ImageDecoderInit); + decode(opts?: Partial): Promise; +}