wasm-bindgen-rayon and new OMT plugin (#1007)

* WIP: wasm-bindgen-rayon and new OMT plugin

* Bump package-lock

* Make OMT work again

* Update year

* Restore accidental change

* Prevent minification of `nextDefineUri`

* Delay loading external deps

This gives inline `define` calls a chance to define dependencies for earlier `define` calls on the same page.

* Add comment

codecs/oxipng/pkg-parallel/snippets/wasm-bindgen-rayon-3d2df09ebec17a22/src/workerHelpers.js

@@ -0,0 +1,98 @@
+/**
+ * Copyright 2021 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Note: we use `wasm_bindgen_worker_`-prefixed message types to make sure
+// we can handle bundling into other files, which might happen to have their
+// own `postMessage`/`onmessage` communication channels.
+//
+// If we didn't take that into the account, we could send much simpler signals
+// like just `0` or whatever, but the code would be less resilient.
+
+function waitForMsgType(target, type) {
+  return new Promise(resolve => {
+    target.addEventListener('message', function onMsg({ data }) {
+      if (data == null || data.type !== type) return;
+      target.removeEventListener('message', onMsg);
+      resolve(data);
+    });
+  });
+}
+
+waitForMsgType(self, 'wasm_bindgen_worker_init').then(async data => {
+  // # Note 1
+  // Our JS should have been generated in
+  // `[out-dir]/snippets/wasm-bindgen-rayon-[hash]/workerHelpers.js`,
+  // resolve the main module via `../../..`.
+  //
+  // This might need updating if the generated structure changes on wasm-bindgen
+  // side ever in the future, but works well with bundlers today. The whole
+  // point of this crate, after all, is to abstract away unstable features
+  // and temporary bugs so that you don't need to deal with them in your code.
+  //
+  // # Note 2
+  // This could be a regular import, but then some bundlers complain about
+  // circular deps.
+  //
+  // Dynamic import could be cheap if this file was inlined into the parent,
+  // which would require us just using `../../..` in `new Worker` below,
+  // but that doesn't work because wasm-pack unconditionally adds
+  // "sideEffects":false (see below).
+  //
+  // OTOH, even though it can't be inlined, it should be still reasonably
+  // cheap since the requested file is already in cache (it was loaded by
+  // the main thread).
+  const pkg = await import('../../..');
+  await pkg.default(data.module, data.memory);
+  postMessage({ type: 'wasm_bindgen_worker_ready' });
+  pkg.wbg_rayon_start_worker(data.receiver);
+});
+
+export async function startWorkers(module, memory, builder) {
+  const workerInit = {
+    type: 'wasm_bindgen_worker_init',
+    module,
+    memory,
+    receiver: builder.receiver()
+  };
+
+  try {
+    await Promise.all(
+      Array.from({ length: builder.numThreads() }, () => {
+        // Self-spawn into a new Worker.
+        //
+        // TODO: while `new URL('...', import.meta.url) becomes a semi-standard
+        // way to get asset URLs relative to the module across various bundlers
+        // and browser, ideally we should switch to `import.meta.resolve`
+        // once it becomes a standard.
+        //
+        // Note: we could use `../../..` as the URL here to inline workerHelpers.js
+        // into the parent entry instead of creating another split point -
+        // this would be preferable from optimization perspective -
+        // however, Webpack then eliminates all message handler code
+        // because wasm-pack produces "sideEffects":false in package.json
+        // unconditionally.
+        //
+        // The only way to work around that is to have side effect code
+        // in an entry point such as Worker file itself.
+        const worker = new Worker(new URL('./workerHelpers.js', import.meta.url), {
+          type: 'module'
+        });
+        worker.postMessage(workerInit);
+        return waitForMsgType(worker, 'wasm_bindgen_worker_ready');
+      })
+    );
+    builder.build();
+  } finally {
+    builder.free();
+  }
+}

codecs/oxipng/pkg-parallel/squoosh_oxipng.d.ts

@@ -7,25 +7,43 @@
 */
 export function optimise(data: Uint8Array, level: number): Uint8Array;
 /**
-* @param {number} num
-* @returns {any}
+* @param {number} num_threads
+* @returns {Promise<any>}
 */
-export function worker_initializer(num: number): any;
+export function initThreadPool(num_threads: number): Promise<any>;
+/**
+* @param {number} receiver
+*/
+export function wbg_rayon_start_worker(receiver: number): void;
 /**
 */
-export function start_main_thread(): void;
+export class wbg_rayon_PoolBuilder {
+  free(): void;
+/**
+* @returns {number}
+*/
+  numThreads(): number;
+/**
+* @returns {number}
+*/
+  receiver(): number;
 /**
 */
-export function start_worker_thread(): void;
+  build(): void;
+}
 
 export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
 
 export interface InitOutput {
   readonly optimise: (a: number, b: number, c: number, d: number) => void;
-  readonly worker_initializer: (a: number) => number;
-  readonly start_main_thread: () => void;
-  readonly start_worker_thread: () => void;
+  readonly __wbg_wbg_rayon_poolbuilder_free: (a: number) => void;
+  readonly wbg_rayon_poolbuilder_numThreads: (a: number) => number;
+  readonly wbg_rayon_poolbuilder_receiver: (a: number) => number;
+  readonly wbg_rayon_poolbuilder_build: (a: number) => void;
+  readonly initThreadPool: (a: number) => number;
+  readonly wbg_rayon_start_worker: (a: number) => void;
   readonly __wbindgen_export_0: WebAssembly.Memory;
+  readonly __wbindgen_add_to_stack_pointer: (a: number) => number;
   readonly __wbindgen_malloc: (a: number) => number;
   readonly __wbindgen_free: (a: number, b: number) => void;
   readonly __wbindgen_start: () => void;
@@ -41,4 +59,3 @@ export interface InitOutput {
 * @returns {Promise<InitOutput>}
 */
 export default function init (module_or_path?: InitInput | Promise<InitInput>, maybe_memory?: WebAssembly.Memory): Promise<InitOutput>;
-        

codecs/oxipng/pkg-parallel/squoosh_oxipng.js

@@ -1,21 +1,6 @@
+import { startWorkers } from './snippets/wasm-bindgen-rayon-3d2df09ebec17a22/src/workerHelpers.js';
 
 let wasm;
-let memory;
-
-const heap = new Array(32).fill(undefined);
-
-heap.push(undefined, null, true, false);
-
-let heap_next = heap.length;
-
-function addHeapObject(obj) {
-    if (heap_next === heap.length) heap.push(heap.length + 1);
-    const idx = heap_next;
-    heap_next = heap[idx];
-
-    heap[idx] = obj;
-    return idx;
-}
 
 let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
 
@@ -33,6 +18,21 @@ function getStringFromWasm0(ptr, len) {
     return cachedTextDecoder.decode(getUint8Memory0().slice(ptr, ptr + len));
 }
 
+const heap = new Array(32).fill(undefined);
+
+heap.push(undefined, null, true, false);
+
+let heap_next = heap.length;
+
+function addHeapObject(obj) {
+    if (heap_next === heap.length) heap.push(heap.length + 1);
+    const idx = heap_next;
+    heap_next = heap[idx];
+
+    heap[idx] = obj;
+    return idx;
+}
+
 let WASM_VECTOR_LEN = 0;
 
 function passArray8ToWasm0(arg, malloc) {
@@ -60,8 +60,7 @@ function getArrayU8FromWasm0(ptr, len) {
 */
 export function optimise(data, level) {
     try {
-        const retptr = wasm.__wbindgen_export_1.value - 16;
-        wasm.__wbindgen_export_1.value = retptr;
+        const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
         var ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_malloc);
         var len0 = WASM_VECTOR_LEN;
         wasm.optimise(retptr, ptr0, len0, level);
@@ -71,7 +70,7 @@ export function optimise(data, level) {
         wasm.__wbindgen_free(r0, r1 * 1);
         return v1;
     } finally {
-        wasm.__wbindgen_export_1.value += 16;
+        wasm.__wbindgen_add_to_stack_pointer(16);
     }
 }
 
@@ -89,29 +88,66 @@ function takeObject(idx) {
     return ret;
 }
 /**
-* @param {number} num
-* @returns {any}
+* @param {number} num_threads
+* @returns {Promise<any>}
 */
-export function worker_initializer(num) {
-    var ret = wasm.worker_initializer(num);
+export function initThreadPool(num_threads) {
+    var ret = wasm.initThreadPool(num_threads);
     return takeObject(ret);
 }
 
 /**
+* @param {number} receiver
 */
-export function start_main_thread() {
-    wasm.start_main_thread();
+export function wbg_rayon_start_worker(receiver) {
+    wasm.wbg_rayon_start_worker(receiver);
 }
 
 /**
 */
-export function start_worker_thread() {
-    wasm.start_worker_thread();
+export class wbg_rayon_PoolBuilder {
+
+    static __wrap(ptr) {
+        const obj = Object.create(wbg_rayon_PoolBuilder.prototype);
+        obj.ptr = ptr;
+
+        return obj;
+    }
+
+    __destroy_into_raw() {
+        const ptr = this.ptr;
+        this.ptr = 0;
+
+        return ptr;
+    }
+
+    free() {
+        const ptr = this.__destroy_into_raw();
+        wasm.__wbg_wbg_rayon_poolbuilder_free(ptr);
+    }
+    /**
+    * @returns {number}
+    */
+    numThreads() {
+        var ret = wasm.wbg_rayon_poolbuilder_numThreads(this.ptr);
+        return ret >>> 0;
+    }
+    /**
+    * @returns {number}
+    */
+    receiver() {
+        var ret = wasm.wbg_rayon_poolbuilder_receiver(this.ptr);
+        return ret;
+    }
+    /**
+    */
+    build() {
+        wasm.wbg_rayon_poolbuilder_build(this.ptr);
+    }
 }
 
-async function load(module, imports, maybe_memory) {
+async function load(module, imports) {
     if (typeof Response === 'function' && module instanceof Response) {
-        memory = imports.wbg.memory = new WebAssembly.Memory({initial:17,maximum:16384,shared:true});
         if (typeof WebAssembly.instantiateStreaming === 'function') {
             try {
                 return await WebAssembly.instantiateStreaming(module, imports);
@@ -130,7 +166,6 @@ async function load(module, imports, maybe_memory) {
         return await WebAssembly.instantiate(bytes, imports);
 
     } else {
-        memory = imports.wbg.memory = maybe_memory;
         const instance = await WebAssembly.instantiate(module, imports);
 
         if (instance instanceof WebAssembly.Instance) {
@@ -144,10 +179,13 @@ async function load(module, imports, maybe_memory) {
 
 async function init(input, maybe_memory) {
     if (typeof input === 'undefined') {
-        input = import.meta.url.replace(/\.js$/, '_bg.wasm');
+        input = new URL('squoosh_oxipng_bg.wasm', import.meta.url);
     }
     const imports = {};
     imports.wbg = {};
+    imports.wbg.__wbindgen_throw = function(arg0, arg1) {
+        throw new Error(getStringFromWasm0(arg0, arg1));
+    };
     imports.wbg.__wbindgen_module = function() {
         var ret = init.__wbindgen_wasm_module;
         return addHeapObject(ret);
@@ -156,19 +194,18 @@ async function init(input, maybe_memory) {
         var ret = wasm.__wbindgen_export_0;
         return addHeapObject(ret);
     };
-    imports.wbg.__wbg_of_6510501edc06d65e = function(arg0, arg1) {
-        var ret = Array.of(takeObject(arg0), takeObject(arg1));
+    imports.wbg.__wbg_startWorkers_914655bb4d5bb5e1 = function(arg0, arg1, arg2) {
+        var ret = startWorkers(takeObject(arg0), takeObject(arg1), wbg_rayon_PoolBuilder.__wrap(arg2));
         return addHeapObject(ret);
     };
-    imports.wbg.__wbindgen_throw = function(arg0, arg1) {
-        throw new Error(getStringFromWasm0(arg0, arg1));
-    };
 
     if (typeof input === 'string' || (typeof Request === 'function' && input instanceof Request) || (typeof URL === 'function' && input instanceof URL)) {
         input = fetch(input);
     }
 
-    const { instance, module } = await load(await input, imports, maybe_memory);
+    imports.wbg.memory = maybe_memory || new WebAssembly.Memory({initial:17,maximum:16384,shared:true});
+
+    const { instance, module } = await load(await input, imports);
 
     wasm = instance.exports;
     init.__wbindgen_wasm_module = module;

codecs/oxipng/pkg-parallel/squoosh_oxipng_bg.wasm.d.ts

@@ -1,10 +1,14 @@
 /* tslint:disable */
 /* eslint-disable */
 export function optimise(a: number, b: number, c: number, d: number): void;
-export function worker_initializer(a: number): number;
-export function start_main_thread(): void;
-export function start_worker_thread(): void;
+export function __wbg_wbg_rayon_poolbuilder_free(a: number): void;
+export function wbg_rayon_poolbuilder_numThreads(a: number): number;
+export function wbg_rayon_poolbuilder_receiver(a: number): number;
+export function wbg_rayon_poolbuilder_build(a: number): void;
+export function initThreadPool(a: number): number;
+export function wbg_rayon_start_worker(a: number): void;
 export const __wbindgen_export_0: WebAssembly.Memory;
+export function __wbindgen_add_to_stack_pointer(a: number): number;
 export function __wbindgen_malloc(a: number): number;
 export function __wbindgen_free(a: number, b: number): void;
 export function __wbindgen_start(): void;