Merge pull request #1018 from GoogleChromeLabs/web-codecs

Add a call-out to Web Codecs API during decoding

src/client/lazy-app/Compress/index.tsx

@@ -107,9 +107,9 @@ async function decodeImage(
       if (mimeType === 'image/webp2') {
         return await workerBridge.wp2Decode(signal, blob);
       }
-      // If it's not one of those types, fall through and try built-in decoding for a laugh.
     }
-    return await abortable(signal, builtinDecode(blob));
+    // Otherwise fall through and try built-in decoding for a laugh.
+    return await builtinDecode(signal, blob, mimeType);
   } catch (err) {
     if (err.name === 'AbortError') throw err;
     console.log(err);

src/client/lazy-app/util/index.ts

@@ -10,6 +10,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+import * as WebCodecs from '../util/web-codecs';
+
 /**
  * Compare two objects, returning a boolean indicating if
  * they have the same properties and strictly equal values.
@@ -192,17 +195,35 @@ interface DrawableToImageDataOptions {
   sh?: number;
 }
 
+function getWidth(
+  drawable: ImageBitmap | HTMLImageElement | VideoFrame,
+): number {
+  if ('displayWidth' in drawable) {
+    return drawable.displayWidth;
+  }
+  return drawable.width;
+}
+
+function getHeight(
+  drawable: ImageBitmap | HTMLImageElement | VideoFrame,
+): number {
+  if ('displayHeight' in drawable) {
+    return drawable.displayHeight;
+  }
+  return drawable.height;
+}
+
 export function drawableToImageData(
-  drawable: ImageBitmap | HTMLImageElement,
+  drawable: ImageBitmap | HTMLImageElement | VideoFrame,
   opts: DrawableToImageDataOptions = {},
 ): ImageData {
   const {
-    width = drawable.width,
-    height = drawable.height,
+    width = getWidth(drawable),
+    height = getHeight(drawable),
     sx = 0,
     sy = 0,
-    sw = drawable.width,
-    sh = drawable.height,
+    sw = getWidth(drawable),
+    sh = getHeight(drawable),
   } = opts;
 
   // Make canvas same size as image
@@ -216,13 +237,25 @@ export function drawableToImageData(
   return ctx.getImageData(0, 0, width, height);
 }
 
-export async function builtinDecode(blob: Blob): Promise<ImageData> {
-  // Prefer createImageBitmap as it's the off-thread option for Firefox.
-  const drawable =
-    'createImageBitmap' in self
-      ? await createImageBitmap(blob)
-      : await blobToImg(blob);
+export async function builtinDecode(
+  signal: AbortSignal,
+  blob: Blob,
+  mimeType: string,
+): Promise<ImageData> {
+  // If WebCodecs are supported, use that.
+  if (await WebCodecs.isTypeSupported(mimeType)) {
+    assertSignal(signal);
+    try {
+      return await abortable(signal, WebCodecs.decode(blob, mimeType));
+    } catch (e) {}
+  }
+  assertSignal(signal);
 
+  // Prefer createImageBitmap as it's the off-thread option for Firefox.
+  const drawable = await abortable<HTMLImageElement | ImageBitmap>(
+    signal,
+    'createImageBitmap' in self ? createImageBitmap(blob) : blobToImg(blob),
+  );
   return drawableToImageData(drawable);
 }
 

src/client/lazy-app/util/web-codecs/index.ts

@@ -0,0 +1,26 @@
+import { drawableToImageData } from 'client/lazy-app/util';
+
+const hasImageDecoder = typeof ImageDecoder !== 'undefined';
+export async function isTypeSupported(mimeType: string): Promise<boolean> {
+  if (!hasImageDecoder) {
+    return false;
+  }
+  return ImageDecoder.isTypeSupported(mimeType);
+}
+export async function decode(
+  blob: Blob | File,
+  mimeType: string,
+): Promise<ImageData> {
+  if (!hasImageDecoder) {
+    throw Error(
+      `This browser does not support ImageDecoder. This function should not have been called.`,
+    );
+  }
+  const decoder = new ImageDecoder({
+    type: mimeType,
+    // Non-obvious way to turn an Blob into a ReadableStream
+    data: new Response(blob).body!,
+  });
+  const { image } = await decoder.decode();
+  return drawableToImageData(image);
+}

src/client/lazy-app/util/web-codecs/missing-types.d.ts

@@ -0,0 +1,60 @@
+interface ImageDecoderInit {
+  type: string;
+  data: BufferSource | ReadableStream;
+  premultiplyAlpha?: PremultiplyAlpha;
+  colorSpaceConversion?: ColorSpaceConversion;
+  desiredWidth?: number;
+  desiredHeight?: number;
+  preferAnimation?: boolean;
+}
+
+interface ImageDecodeOptions {
+  frameIndex: number;
+  completeFramesOnly: boolean;
+}
+
+interface ImageDecodeResult {
+  image: VideoFrame;
+  complete: boolean;
+}
+
+// I didn’t do all the types because the class is kinda complex.
+// I focused on what we need.
+// See https://w3c.github.io/webcodecs/#videoframe
+declare class VideoFrame {
+  displayWidth: number;
+  displayHeight: number;
+}
+
+// Add VideoFrame to canvas’ drawImage()
+interface CanvasDrawImage {
+  drawImage(
+    image: CanvasImageSource | VideoFrame,
+    dx: number,
+    dy: number,
+  ): void;
+  drawImage(
+    image: CanvasImageSource | VideoFrame,
+    dx: number,
+    dy: number,
+    dw: number,
+    dh: number,
+  ): void;
+  drawImage(
+    image: CanvasImageSource | VideoFrame,
+    sx: number,
+    sy: number,
+    sw: number,
+    sh: number,
+    dx: number,
+    dy: number,
+    dw: number,
+    dh: number,
+  ): void;
+}
+
+declare class ImageDecoder {
+  static isTypeSupported(type: string): Promise<boolean>;
+  constructor(desc: ImageDecoderInit);
+  decode(opts?: Partial<ImageDecodeOptions>): Promise<ImageDecodeResult>;
+}