Merge pull request #1074 from GoogleChromeLabs/avif-node-mt

codecs/avif/Makefile

@@ -16,10 +16,10 @@ export
 OUT_ENC_JS = enc/avif_enc.js
 OUT_NODE_ENC_JS = enc/avif_node_enc.js
 OUT_ENC_MT_JS = enc/avif_enc_mt.js
+OUT_NODE_ENC_MT_JS = enc/avif_node_enc_mt.js
 OUT_DEC_JS = dec/avif_dec.js
 OUT_NODE_DEC_JS = dec/avif_node_dec.js
 
-OUT_ENC_CPP = enc/avif_enc.cpp
 OUT_ENC_CPP = enc/avif_enc.cpp
 OUT_DEC_CPP = dec/avif_dec.cpp
 ENVIRONMENT = worker
@@ -28,9 +28,9 @@ HELPER_MAKEFLAGS := -f helper.Makefile
 
 .PHONY: all clean
 
-all: $(OUT_ENC_JS) $(OUT_DEC_JS) $(OUT_ENC_MT_JS) $(OUT_NODE_ENC_JS) $(OUT_NODE_DEC_JS)
+all: $(OUT_ENC_JS) $(OUT_DEC_JS) $(OUT_ENC_MT_JS) $(OUT_NODE_ENC_JS) $(OUT_NODE_ENC_MT_JS) $(OUT_NODE_DEC_JS)
 
-$(OUT_NODE_ENC_JS): ENVIRONMENT=node
+$(OUT_NODE_ENC_JS) $(OUT_NODE_ENC_MT_JS): ENVIRONMENT=node
 $(OUT_NODE_ENC_JS) $(OUT_ENC_JS): $(OUT_ENC_CPP) $(CODEC_DIR)/CMakeLists.txt $(LIBAOM_DIR)/CMakeLists.txt
 	$(MAKE) \
 		$(HELPER_MAKEFLAGS) \
@@ -44,7 +44,7 @@ $(OUT_NODE_ENC_JS) $(OUT_ENC_JS): $(OUT_ENC_CPP) $(CODEC_DIR)/CMakeLists.txt $(L
 		ENVIRONMENT=$(ENVIRONMENT) \
 		LIBAVIF_FLAGS="-DAVIF_CODEC_AOM_DECODE=0"
 
-$(OUT_ENC_MT_JS): $(OUT_ENC_CPP) $(CODEC_DIR)/CMakeLists.txt $(LIBAOM_DIR)/CMakeLists.txt
+$(OUT_ENC_MT_JS) $(OUT_NODE_ENC_MT_JS): $(OUT_ENC_CPP) $(CODEC_DIR)/CMakeLists.txt $(LIBAOM_DIR)/CMakeLists.txt
 	$(MAKE) \
 		$(HELPER_MAKEFLAGS) \
 		OUT_JS=$@ \
@@ -89,4 +89,7 @@ $(LIBAOM_DIR)/CMakeLists.txt: $(LIBAOM_PACKAGE)
 clean:
 	$(MAKE) $(HELPER_MAKEFLAGS) OUT_JS=$(OUT_ENC_JS) clean
 	$(MAKE) $(HELPER_MAKEFLAGS) OUT_JS=$(OUT_ENC_MT_JS) clean
+	$(MAKE) $(HELPER_MAKEFLAGS) OUT_JS=$(OUT_ENC_NODE_JS) clean
+	$(MAKE) $(HELPER_MAKEFLAGS) OUT_JS=$(OUT_ENC_NODE_MT_JS) clean
 	$(MAKE) $(HELPER_MAKEFLAGS) OUT_JS=$(OUT_DEC_JS) clean
+	$(MAKE) $(HELPER_MAKEFLAGS) OUT_JS=$(OUT_DEV_NODE_JS) clean

codecs/avif/enc/avif_node_enc_mt.worker.js

@@ -0,0 +1 @@
+"use strict";var Module={};if(typeof process==="object"&&typeof process.versions==="object"&&typeof process.versions.node==="string"){var nodeWorkerThreads=require("worker_threads");var parentPort=nodeWorkerThreads.parentPort;parentPort.on("message",function(data){onmessage({data:data})});var nodeFS=require("fs");Object.assign(global,{self:global,require:require,Module:Module,location:{href:__filename},Worker:nodeWorkerThreads.Worker,importScripts:function(f){(0,eval)(nodeFS.readFileSync(f,"utf8"))},postMessage:function(msg){parentPort.postMessage(msg)},performance:global.performance||{now:function(){return Date.now()}}})}var initializedJS=false;function threadPrintErr(){var text=Array.prototype.slice.call(arguments).join(" ");console.error(text)}function threadAlert(){var text=Array.prototype.slice.call(arguments).join(" ");postMessage({cmd:"alert",text:text,threadId:Module["_pthread_self"]()})}var err=threadPrintErr;self.alert=threadAlert;Module["instantiateWasm"]=function(info,receiveInstance){var instance=new WebAssembly.Instance(Module["wasmModule"],info);receiveInstance(instance);Module["wasmModule"]=null;return instance.exports};function moduleLoaded(){}self.onmessage=function(e){try{if(e.data.cmd==="load"){Module["wasmModule"]=e.data.wasmModule;Module["wasmMemory"]=e.data.wasmMemory;Module["buffer"]=Module["wasmMemory"].buffer;Module["ENVIRONMENT_IS_PTHREAD"]=true;(e.data.urlOrBlob?import(e.data.urlOrBlob):import("./avif_node_enc_mt.js")).then(function(exports){return exports.default(Module)}).then(function(instance){Module=instance;moduleLoaded()})}else if(e.data.cmd==="objectTransfer"){Module["PThread"].receiveObjectTransfer(e.data)}else if(e.data.cmd==="run"){Module["__performance_now_clock_drift"]=performance.now()-e.data.time;Module["__emscripten_thread_init"](e.data.threadInfoStruct,0,0);var max=e.data.stackBase;var top=e.data.stackBase+e.data.stackSize;Module["establishStackSpace"](top,max);Module["PThread"].receiveObjectTransfer(e.data);Module["PThread"].threadInit();if(!initializedJS){Module["___embind_register_native_and_builtin_types"]();initializedJS=true}try{var result=Module["invokeEntryPoint"](e.data.start_routine,e.data.arg);if(Module["keepRuntimeAlive"]()){Module["PThread"].setExitStatus(result)}else{Module["PThread"].threadExit(result)}}catch(ex){if(ex==="Canceled!"){Module["PThread"].threadCancel()}else if(ex!="unwind"){if(ex instanceof Module["ExitStatus"]){if(Module["keepRuntimeAlive"]()){}else{Module["PThread"].threadExit(ex.status)}}else{Module["PThread"].threadExit(-2);throw ex}}}}else if(e.data.cmd==="cancel"){if(Module["_pthread_self"]()){Module["PThread"].threadCancel()}}else if(e.data.target==="setimmediate"){}else if(e.data.cmd==="processThreadQueue"){if(Module["_pthread_self"]()){Module["_emscripten_current_thread_process_queued_calls"]()}}else{err("worker.js received unknown command "+e.data.cmd);err(e.data)}}catch(ex){err("worker.js onmessage() captured an uncaught exception: "+ex);if(ex&&ex.stack)err(ex.stack);throw ex}};

libsquoosh/lib/chunk-plugin.js

@@ -0,0 +1,49 @@
+/**
+ * Copyright 2020 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { promises as fs } from 'fs';
+import { basename } from 'path';
+
+const defaultOpts = {
+  prefix: 'chunk-url',
+};
+
+export default function chunkPlugin(opts) {
+  opts = { ...defaultOpts, ...opts };
+
+  const prefix = opts.prefix + ':';
+  return {
+    name: 'chunk-plugin',
+    async resolveId(id, importer) {
+      if (!id.startsWith(prefix)) return;
+      const realId = id.slice(prefix.length);
+      const resolveResult = await this.resolve(realId, importer);
+
+      if (!resolveResult) {
+        throw Error(`Cannot find ${realId}`);
+      }
+      return prefix + resolveResult.id;
+    },
+    async load(id) {
+      if (!id.startsWith(prefix)) return;
+      const realId = id.slice(prefix.length);
+      const source = await fs.readFile(realId);
+      this.addWatchFile(realId);
+
+      return `export default import.meta.ROLLUP_FILE_URL_${this.emitFile({
+        type: 'chunk',
+        source,
+        id: realId,
+      })}`;
+    },
+  };
+}

libsquoosh/package-lock.json

@@ -9,6 +9,7 @@
       "version": "0.3.1",
       "license": "Apache-2.0",
       "dependencies": {
+        "wasm-feature-detect": "^1.2.11",
         "web-streams-polyfill": "^3.0.3"
       },
       "devDependencies": {
@@ -22,6 +23,9 @@
         "rollup-plugin-terser": "^7.0.2",
         "typescript": "^4.1.3",
         "which": "^2.0.2"
+      },
+      "engines": {
+        "node": " ^12.5.0 || ^14.0.0 || ^16.0.0 "
       }
     },
     "node_modules/@babel/code-frame": {
@@ -2055,6 +2059,11 @@
         "node": ">=4"
       }
     },
+    "node_modules/wasm-feature-detect": {
+      "version": "1.2.11",
+      "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.2.11.tgz",
+      "integrity": "sha512-HUqwaodrQGaZgz1lZaNioIkog9tkeEJjrM3eq4aUL04whXOVDRc/o2EGb/8kV0QX411iAYWEqq7fMBmJ6dKS6w=="
+    },
     "node_modules/web-streams-polyfill": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.0.3.tgz",
@@ -3948,6 +3957,11 @@
       "integrity": "sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg==",
       "dev": true
     },
+    "wasm-feature-detect": {
+      "version": "1.2.11",
+      "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.2.11.tgz",
+      "integrity": "sha512-HUqwaodrQGaZgz1lZaNioIkog9tkeEJjrM3eq4aUL04whXOVDRc/o2EGb/8kV0QX411iAYWEqq7fMBmJ6dKS6w=="
+    },
     "web-streams-polyfill": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.0.3.tgz",

libsquoosh/package.json

@@ -22,6 +22,7 @@
     "node": " ^12.5.0 || ^14.0.0 || ^16.0.0 "
   },
   "dependencies": {
+    "wasm-feature-detect": "^1.2.11",
     "web-streams-polyfill": "^3.0.3"
   },
   "devDependencies": {

libsquoosh/rollup.config.js

@@ -2,6 +2,7 @@ import resolve from '@rollup/plugin-node-resolve';
 import cjs from '@rollup/plugin-commonjs';
 import simpleTS from './lib/simple-ts';
 import asset from './lib/asset-plugin.js';
+import chunk from './lib/chunk-plugin.js';
 import json from './lib/json-plugin.js';
 import autojson from './lib/autojson-plugin.js';
 import { getBabelOutputPlugin } from '@rollup/plugin-babel';
@@ -18,6 +19,7 @@ export default {
   plugins: [
     resolve(),
     cjs(),
+    chunk(),
     asset(),
     autojson(),
     json(),

libsquoosh/src/codecs.ts

@@ -1,5 +1,13 @@
 import { promises as fsp } from 'fs';
 import { instantiateEmscriptenWasm, pathify } from './emscripten-utils.js';
+import { threads } from 'wasm-feature-detect';
+import { cpus } from 'os';
+
+// We use `navigator.hardwareConcurrency` for Emscripten’s pthread pool size.
+// This is the only workaround I can get working without crying.
+(globalThis as any).navigator = {
+  hardwareConcurrency: cpus().length,
+};
 
 interface RotateModuleInstance {
   exports: {
@@ -47,6 +55,9 @@ import webpDecWasm from 'asset-url:../../codecs/webp/dec/webp_node_dec.wasm';
 // AVIF
 import avifEnc from '../../codecs/avif/enc/avif_node_enc.js';
 import avifEncWasm from 'asset-url:../../codecs/avif/enc/avif_node_enc.wasm';
+import avifEncMt from '../../codecs/avif/enc/avif_node_enc_mt.js';
+import avifEncMtWorker from 'chunk-url:../../codecs/avif/enc/avif_node_enc_mt.worker.js';
+import avifEncMtWasm from 'asset-url:../../codecs/avif/enc/avif_node_enc_mt.wasm';
 import avifDec from '../../codecs/avif/dec/avif_node_dec.js';
 import avifDecWasm from 'asset-url:../../codecs/avif/dec/avif_node_dec.wasm';
 
@@ -325,7 +336,16 @@ export const codecs = {
     extension: 'avif',
     detectors: [/^\x00\x00\x00 ftypavif\x00\x00\x00\x00/],
     dec: () => instantiateEmscriptenWasm(avifDec, avifDecWasm),
-    enc: () => instantiateEmscriptenWasm(avifEnc, avifEncWasm),
+    enc: async () => {
+      if (await threads()) {
+        return instantiateEmscriptenWasm(
+          avifEncMt,
+          avifEncMtWasm,
+          avifEncMtWorker,
+        );
+      }
+      return instantiateEmscriptenWasm(avifEnc, avifEncWasm);
+    },
     defaultEncoderOptions: {
       cqLevel: 33,
       cqAlphaLevel: -1,

libsquoosh/src/emscripten-utils.ts

@@ -1,4 +1,4 @@
-import { fileURLToPath } from 'url';
+import { fileURLToPath, URL } from 'url';
 
 export function pathify(path: string): string {
   if (path.startsWith('file://')) {
@@ -10,10 +10,17 @@ export function pathify(path: string): string {
 export function instantiateEmscriptenWasm<T extends EmscriptenWasm.Module>(
   factory: EmscriptenWasm.ModuleFactory<T>,
   path: string,
+  workerJS: string = '',
 ): Promise<T> {
   return factory({
-    locateFile() {
-      return pathify(path);
+    locateFile(requestPath) {
+      // The glue code generated by emscripten uses the original
+      // file names of the worker file and the wasm binary.
+      // These will have changed in the bundling process and
+      // we need to inject them here.
+      if (requestPath.endsWith('.wasm')) return pathify(path);
+      if (requestPath.endsWith('.worker.js')) return new URL(workerJS).pathname;
+      return requestPath;
     },
   });
 }

libsquoosh/src/missing-types.d.ts

@@ -23,6 +23,11 @@ declare module 'asset-url:../../codecs/resize/pkg/squoosh_resize_bg.wasm' {
   export default value;
 }
 
+declare module 'chunk-url:../../codecs/avif/enc/avif_node_enc_mt.worker.js' {
+  const value: string;
+  export default value;
+}
+
 // These don't exist in NodeJS types so we're not able to use them but they are referenced in some emscripten and codec types
 // Thus, we need to explicitly assign them to be `never`
 // We're also not able to use the APIs that use these types