From d4056026fbad37a9d1f7a8f9fda0899e080f8bf3 Mon Sep 17 00:00:00 2001 From: Surma Date: Fri, 25 Jun 2021 00:11:25 +0100 Subject: [PATCH] Add AVIF thread support --- libsquoosh/lib/chunk-plugin.js | 49 ++++++++++++++++++++++++++++++ libsquoosh/package-lock.json | 14 +++++++++ libsquoosh/package.json | 1 + libsquoosh/rollup.config.js | 2 ++ libsquoosh/src/codecs.ts | 22 +++++++++++++- libsquoosh/src/emscripten-utils.ts | 10 ++++-- libsquoosh/src/missing-types.d.ts | 5 +++ 7 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 libsquoosh/lib/chunk-plugin.js diff --git a/libsquoosh/lib/chunk-plugin.js b/libsquoosh/lib/chunk-plugin.js new file mode 100644 index 00000000..84a5f9c2 --- /dev/null +++ b/libsquoosh/lib/chunk-plugin.js @@ -0,0 +1,49 @@ +/** + * Copyright 2020 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { promises as fs } from 'fs'; +import { basename } from 'path'; + +const defaultOpts = { + prefix: 'chunk-url', +}; + +export default function chunkPlugin(opts) { + opts = { ...defaultOpts, ...opts }; + + const prefix = opts.prefix + ':'; + return { + name: 'chunk-plugin', + async resolveId(id, importer) { + if (!id.startsWith(prefix)) return; + const realId = id.slice(prefix.length); + const resolveResult = await this.resolve(realId, importer); + + if (!resolveResult) { + throw Error(`Cannot find ${realId}`); + } + return prefix + resolveResult.id; + }, + async load(id) { + if (!id.startsWith(prefix)) return; + const realId = id.slice(prefix.length); + const source = await fs.readFile(realId); + this.addWatchFile(realId); + + return `export default import.meta.ROLLUP_FILE_URL_${this.emitFile({ + type: 'chunk', + source, + id: realId, + })}`; + }, + }; +} diff --git a/libsquoosh/package-lock.json b/libsquoosh/package-lock.json index 58ab8776..2674ae10 100644 --- a/libsquoosh/package-lock.json +++ b/libsquoosh/package-lock.json @@ -9,6 +9,7 @@ "version": "0.3.1", "license": "Apache-2.0", "dependencies": { + "wasm-feature-detect": "^1.2.11", "web-streams-polyfill": "^3.0.3" }, "devDependencies": { @@ -22,6 +23,9 @@ "rollup-plugin-terser": "^7.0.2", "typescript": "^4.1.3", "which": "^2.0.2" + }, + "engines": { + "node": " ^12.5.0 || ^14.0.0 || ^16.0.0 " } }, "node_modules/@babel/code-frame": { @@ -2055,6 +2059,11 @@ "node": ">=4" } }, + "node_modules/wasm-feature-detect": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.2.11.tgz", + "integrity": "sha512-HUqwaodrQGaZgz1lZaNioIkog9tkeEJjrM3eq4aUL04whXOVDRc/o2EGb/8kV0QX411iAYWEqq7fMBmJ6dKS6w==" + }, "node_modules/web-streams-polyfill": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.0.3.tgz", @@ -3948,6 +3957,11 @@ "integrity": "sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg==", "dev": true }, + "wasm-feature-detect": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.2.11.tgz", + "integrity": "sha512-HUqwaodrQGaZgz1lZaNioIkog9tkeEJjrM3eq4aUL04whXOVDRc/o2EGb/8kV0QX411iAYWEqq7fMBmJ6dKS6w==" + }, "web-streams-polyfill": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.0.3.tgz", diff --git a/libsquoosh/package.json b/libsquoosh/package.json index e67f1751..27501208 100644 --- a/libsquoosh/package.json +++ b/libsquoosh/package.json @@ -22,6 +22,7 @@ "node": " ^12.5.0 || ^14.0.0 || ^16.0.0 " }, "dependencies": { + "wasm-feature-detect": "^1.2.11", "web-streams-polyfill": "^3.0.3" }, "devDependencies": { diff --git a/libsquoosh/rollup.config.js b/libsquoosh/rollup.config.js index d6d6537d..3ca3a7e2 100644 --- a/libsquoosh/rollup.config.js +++ b/libsquoosh/rollup.config.js @@ -2,6 +2,7 @@ import resolve from '@rollup/plugin-node-resolve'; import cjs from '@rollup/plugin-commonjs'; import simpleTS from './lib/simple-ts'; import asset from './lib/asset-plugin.js'; +import chunk from './lib/chunk-plugin.js'; import json from './lib/json-plugin.js'; import autojson from './lib/autojson-plugin.js'; import { getBabelOutputPlugin } from '@rollup/plugin-babel'; @@ -18,6 +19,7 @@ export default { plugins: [ resolve(), cjs(), + chunk(), asset(), autojson(), json(), diff --git a/libsquoosh/src/codecs.ts b/libsquoosh/src/codecs.ts index e97c753c..868a889b 100644 --- a/libsquoosh/src/codecs.ts +++ b/libsquoosh/src/codecs.ts @@ -1,5 +1,13 @@ import { promises as fsp } from 'fs'; import { instantiateEmscriptenWasm, pathify } from './emscripten-utils.js'; +import { threads } from 'wasm-feature-detect'; +import { cpus } from 'os'; + +// We use `navigator.hardwareConcurrency` for Emscripten’s pthread pool size. +// This is the only workaround I can get working without crying. +(globalThis as any).navigator = { + hardwareConcurrency: cpus().length, +}; interface RotateModuleInstance { exports: { @@ -47,6 +55,9 @@ import webpDecWasm from 'asset-url:../../codecs/webp/dec/webp_node_dec.wasm'; // AVIF import avifEnc from '../../codecs/avif/enc/avif_node_enc.js'; import avifEncWasm from 'asset-url:../../codecs/avif/enc/avif_node_enc.wasm'; +import avifEncMt from '../../codecs/avif/enc/avif_node_enc_mt.js'; +import avifEncMtWorker from 'chunk-url:../../codecs/avif/enc/avif_node_enc_mt.worker.js'; +import avifEncMtWasm from 'asset-url:../../codecs/avif/enc/avif_node_enc_mt.wasm'; import avifDec from '../../codecs/avif/dec/avif_node_dec.js'; import avifDecWasm from 'asset-url:../../codecs/avif/dec/avif_node_dec.wasm'; @@ -325,7 +336,16 @@ export const codecs = { extension: 'avif', detectors: [/^\x00\x00\x00 ftypavif\x00\x00\x00\x00/], dec: () => instantiateEmscriptenWasm(avifDec, avifDecWasm), - enc: () => instantiateEmscriptenWasm(avifEnc, avifEncWasm), + enc: async () => { + if (await threads()) { + return instantiateEmscriptenWasm( + avifEncMt, + avifEncMtWasm, + avifEncMtWorker, + ); + } + return instantiateEmscriptenWasm(avifEnc, avifEncWasm); + }, defaultEncoderOptions: { cqLevel: 33, cqAlphaLevel: -1, diff --git a/libsquoosh/src/emscripten-utils.ts b/libsquoosh/src/emscripten-utils.ts index 255aa4cd..428be004 100644 --- a/libsquoosh/src/emscripten-utils.ts +++ b/libsquoosh/src/emscripten-utils.ts @@ -1,4 +1,4 @@ -import { fileURLToPath } from 'url'; +import { fileURLToPath, URL } from 'url'; export function pathify(path: string): string { if (path.startsWith('file://')) { @@ -10,10 +10,14 @@ export function pathify(path: string): string { export function instantiateEmscriptenWasm( factory: EmscriptenWasm.ModuleFactory, path: string, + workerWasm: string = '', ): Promise { return factory({ - locateFile() { - return pathify(path); + locateFile(requestPath) { + if (requestPath.endsWith('.wasm')) return pathify(path); + if (requestPath.endsWith('.worker.js')) + return new URL(workerWasm).pathname; + return requestPath; }, }); } diff --git a/libsquoosh/src/missing-types.d.ts b/libsquoosh/src/missing-types.d.ts index 4319a188..e7a68fe5 100644 --- a/libsquoosh/src/missing-types.d.ts +++ b/libsquoosh/src/missing-types.d.ts @@ -23,6 +23,11 @@ declare module 'asset-url:../../codecs/resize/pkg/squoosh_resize_bg.wasm' { export default value; } +declare module 'chunk-url:../../codecs/avif/enc/avif_node_enc_mt.worker.js' { + const value: string; + export default value; +} + // These don't exist in NodeJS types so we're not able to use them but they are referenced in some emscripten and codec types // Thus, we need to explicitly assign them to be `never` // We're also not able to use the APIs that use these types