mirror of
https://github.com/GoogleChromeLabs/squoosh.git
synced 2025-11-15 10:09:45 +00:00
Separate CLI and API
This commit is contained in:
atjn
@@ -1,70 +0,0 @@
|
||||
import { instantiateEmscriptenWasm } from "./emscripten-utils.js";
|
||||
|
||||
import visdif from "../../codecs/visdif/visdif.js";
|
||||
import visdifWasm from "asset-url:../../codecs/visdif/visdif.wasm";
|
||||
|
||||
// `measure` is a (async) function that takes exactly one numeric parameter and
|
||||
// returns a value. The function is assumed to be monotonic (an increase in `parameter`
|
||||
// will result in an increase in the return value. The function uses binary search
|
||||
// to find `parameter` such that `measure` returns `measureGoal`, within an error
|
||||
// of `epsilon`. It will use at most `maxRounds` attempts.
|
||||
export async function binarySearch(
|
||||
measureGoal,
|
||||
measure,
|
||||
{ min = 0, max = 100, epsilon = 0.1, maxRounds = 8 } = {}
|
||||
) {
|
||||
let parameter = (max - min) / 2 + min;
|
||||
let delta = (max - min) / 4;
|
||||
let value;
|
||||
let round = 1;
|
||||
while (true) {
|
||||
value = await measure(parameter);
|
||||
if (Math.abs(value - measureGoal) < epsilon || round >= maxRounds) {
|
||||
return { parameter, round, value };
|
||||
}
|
||||
if (value > measureGoal) {
|
||||
parameter -= delta;
|
||||
} else if (value < measureGoal) {
|
||||
parameter += delta;
|
||||
}
|
||||
delta /= 2;
|
||||
round++;
|
||||
}
|
||||
}
|
||||
|
||||
export async function autoOptimize(
|
||||
bitmapIn,
|
||||
encode,
|
||||
decode,
|
||||
{ butteraugliDistanceGoal = 1.4, ...otherOpts } = {}
|
||||
) {
|
||||
const { VisDiff } = await instantiateEmscriptenWasm(visdif, visdifWasm);
|
||||
|
||||
const comparator = new VisDiff(
|
||||
bitmapIn.data,
|
||||
bitmapIn.width,
|
||||
bitmapIn.height
|
||||
);
|
||||
|
||||
let bitmapOut;
|
||||
let binaryOut;
|
||||
// Increasing quality means _decrease_ in Butteraugli distance.
|
||||
// `binarySearch` assumes that increasing `parameter` will
|
||||
// increase the metric value. So multipliy Butteraugli values by -1.
|
||||
const { parameter } = await binarySearch(
|
||||
-1 * butteraugliDistanceGoal,
|
||||
async quality => {
|
||||
binaryOut = await encode(bitmapIn, quality);
|
||||
bitmapOut = await decode(binaryOut);
|
||||
return -1 * comparator.distance(bitmapOut.data);
|
||||
},
|
||||
otherOpts
|
||||
);
|
||||
comparator.delete();
|
||||
|
||||
return {
|
||||
bitmap: bitmapOut,
|
||||
binary: binaryOut,
|
||||
quality: parameter
|
||||
};
|
||||
}
|
||||
@@ -1,361 +0,0 @@
|
||||
import { promises as fsp } from 'fs';
|
||||
import { instantiateEmscriptenWasm, pathify } from './emscripten-utils.js';
|
||||
|
||||
// MozJPEG
|
||||
import mozEnc from '../../codecs/mozjpeg/enc/mozjpeg_node_enc.js';
|
||||
import mozEncWasm from 'asset-url:../../codecs/mozjpeg/enc/mozjpeg_node_enc.wasm';
|
||||
import mozDec from '../../codecs/mozjpeg/dec/mozjpeg_node_dec.js';
|
||||
import mozDecWasm from 'asset-url:../../codecs/mozjpeg/dec/mozjpeg_node_dec.wasm';
|
||||
|
||||
// WebP
|
||||
import webpEnc from '../../codecs/webp/enc/webp_node_enc.js';
|
||||
import webpEncWasm from 'asset-url:../../codecs/webp/enc/webp_node_enc.wasm';
|
||||
import webpDec from '../../codecs/webp/dec/webp_node_dec.js';
|
||||
import webpDecWasm from 'asset-url:../../codecs/webp/dec/webp_node_dec.wasm';
|
||||
|
||||
// AVIF
|
||||
import avifEnc from '../../codecs/avif/enc/avif_node_enc.js';
|
||||
import avifEncWasm from 'asset-url:../../codecs/avif/enc/avif_node_enc.wasm';
|
||||
import avifDec from '../../codecs/avif/dec/avif_node_dec.js';
|
||||
import avifDecWasm from 'asset-url:../../codecs/avif/dec/avif_node_dec.wasm';
|
||||
|
||||
// JXL
|
||||
import jxlEnc from '../../codecs/jxl/enc/jxl_node_enc.js';
|
||||
import jxlEncWasm from 'asset-url:../../codecs/jxl/enc/jxl_node_enc.wasm';
|
||||
import jxlDec from '../../codecs/jxl/dec/jxl_node_dec.js';
|
||||
import jxlDecWasm from 'asset-url:../../codecs/jxl/dec/jxl_node_dec.wasm';
|
||||
|
||||
// WP2
|
||||
import wp2Enc from '../../codecs/wp2/enc/wp2_node_enc.js';
|
||||
import wp2EncWasm from 'asset-url:../../codecs/wp2/enc/wp2_node_enc.wasm';
|
||||
import wp2Dec from '../../codecs/wp2/dec/wp2_node_dec.js';
|
||||
import wp2DecWasm from 'asset-url:../../codecs/wp2/dec/wp2_node_dec.wasm';
|
||||
|
||||
// PNG
|
||||
import * as pngEncDec from '../../codecs/png/pkg/squoosh_png.js';
|
||||
import pngEncDecWasm from 'asset-url:../../codecs/png/pkg/squoosh_png_bg.wasm';
|
||||
const pngEncDecPromise = pngEncDec.default(
|
||||
fsp.readFile(pathify(pngEncDecWasm)),
|
||||
);
|
||||
|
||||
// OxiPNG
|
||||
import * as oxipng from '../../codecs/oxipng/pkg/squoosh_oxipng.js';
|
||||
import oxipngWasm from 'asset-url:../../codecs/oxipng/pkg/squoosh_oxipng_bg.wasm';
|
||||
const oxipngPromise = oxipng.default(fsp.readFile(pathify(oxipngWasm)));
|
||||
|
||||
// Resize
|
||||
import * as resize from '../../codecs/resize/pkg/squoosh_resize.js';
|
||||
import resizeWasm from 'asset-url:../../codecs/resize/pkg/squoosh_resize_bg.wasm';
|
||||
const resizePromise = resize.default(fsp.readFile(pathify(resizeWasm)));
|
||||
|
||||
// rotate
|
||||
import rotateWasm from 'asset-url:../../codecs/rotate/rotate.wasm';
|
||||
|
||||
// ImageQuant
|
||||
import imageQuant from '../../codecs/imagequant/imagequant_node.js';
|
||||
import imageQuantWasm from 'asset-url:../../codecs/imagequant/imagequant_node.wasm';
|
||||
const imageQuantPromise = instantiateEmscriptenWasm(imageQuant, imageQuantWasm);
|
||||
|
||||
// Our decoders currently rely on a `ImageData` global.
|
||||
import ImageData from './image_data.js';
|
||||
globalThis.ImageData = ImageData;
|
||||
|
||||
function resizeNameToIndex(name) {
|
||||
switch (name) {
|
||||
case 'triangle':
|
||||
return 0;
|
||||
case 'catrom':
|
||||
return 1;
|
||||
case 'mitchell':
|
||||
return 2;
|
||||
case 'lanczos3':
|
||||
return 3;
|
||||
default:
|
||||
throw Error(`Unknown resize algorithm "${name}"`);
|
||||
}
|
||||
}
|
||||
|
||||
function resizeWithAspect({
|
||||
input_width,
|
||||
input_height,
|
||||
target_width,
|
||||
target_height,
|
||||
}) {
|
||||
if (!target_width && !target_height) {
|
||||
throw Error('Need to specify at least width or height when resizing');
|
||||
}
|
||||
if (target_width && target_height) {
|
||||
return { width: target_width, height: target_height };
|
||||
}
|
||||
if (!target_width) {
|
||||
return {
|
||||
width: Math.round((input_width / input_height) * target_height),
|
||||
height: target_height,
|
||||
};
|
||||
}
|
||||
if (!target_height) {
|
||||
return {
|
||||
width: target_width,
|
||||
height: Math.round((input_height / input_width) * target_width),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const preprocessors = {
|
||||
resize: {
|
||||
name: 'Resize',
|
||||
description: 'Resize the image before compressing',
|
||||
instantiate: async () => {
|
||||
await resizePromise;
|
||||
return (
|
||||
buffer,
|
||||
input_width,
|
||||
input_height,
|
||||
{ width, height, method, premultiply, linearRGB },
|
||||
) => {
|
||||
({ width, height } = resizeWithAspect({
|
||||
input_width,
|
||||
input_height,
|
||||
target_width: width,
|
||||
target_height: height,
|
||||
}));
|
||||
return new ImageData(
|
||||
resize.resize(
|
||||
buffer,
|
||||
input_width,
|
||||
input_height,
|
||||
width,
|
||||
height,
|
||||
resizeNameToIndex(method),
|
||||
premultiply,
|
||||
linearRGB,
|
||||
),
|
||||
width,
|
||||
height,
|
||||
);
|
||||
};
|
||||
},
|
||||
defaultOptions: {
|
||||
method: 'lanczos3',
|
||||
fitMethod: 'stretch',
|
||||
premultiply: true,
|
||||
linearRGB: true,
|
||||
},
|
||||
},
|
||||
// // TODO: Need to handle SVGs and HQX
|
||||
quant: {
|
||||
name: 'ImageQuant',
|
||||
description: 'Reduce the number of colors used (aka. paletting)',
|
||||
instantiate: async () => {
|
||||
const imageQuant = await imageQuantPromise;
|
||||
return (buffer, width, height, { numColors, dither }) =>
|
||||
new ImageData(
|
||||
imageQuant.quantize(buffer, width, height, numColors, dither),
|
||||
width,
|
||||
height,
|
||||
);
|
||||
},
|
||||
defaultOptions: {
|
||||
numColors: 255,
|
||||
dither: 1.0,
|
||||
},
|
||||
},
|
||||
rotate: {
|
||||
name: 'Rotate',
|
||||
description: 'Rotate image',
|
||||
instantiate: async () => {
|
||||
return async (buffer, width, height, { numRotations }) => {
|
||||
const degrees = (numRotations * 90) % 360;
|
||||
const sameDimensions = degrees == 0 || degrees == 180;
|
||||
const size = width * height * 4;
|
||||
const { instance } = await WebAssembly.instantiate(
|
||||
await fsp.readFile(pathify(rotateWasm)),
|
||||
);
|
||||
const { memory } = instance.exports;
|
||||
const additionalPagesNeeded = Math.ceil(
|
||||
(size * 2 - memory.buffer.byteLength + 8) / (64 * 1024),
|
||||
);
|
||||
if (additionalPagesNeeded > 0) {
|
||||
memory.grow(additionalPagesNeeded);
|
||||
}
|
||||
const view = new Uint8ClampedArray(memory.buffer);
|
||||
view.set(buffer, 8);
|
||||
instance.exports.rotate(width, height, degrees);
|
||||
return new ImageData(
|
||||
view.slice(size + 8, size * 2 + 8),
|
||||
sameDimensions ? width : height,
|
||||
sameDimensions ? height : width,
|
||||
);
|
||||
};
|
||||
},
|
||||
defaultOptions: {
|
||||
numRotations: 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
export const codecs = {
|
||||
mozjpeg: {
|
||||
name: 'MozJPEG',
|
||||
extension: 'jpg',
|
||||
detectors: [/^\xFF\xD8\xFF/],
|
||||
dec: () => instantiateEmscriptenWasm(mozDec, mozDecWasm),
|
||||
enc: () => instantiateEmscriptenWasm(mozEnc, mozEncWasm),
|
||||
defaultEncoderOptions: {
|
||||
quality: 75,
|
||||
baseline: false,
|
||||
arithmetic: false,
|
||||
progressive: true,
|
||||
optimize_coding: true,
|
||||
smoothing: 0,
|
||||
color_space: 3 /*YCbCr*/,
|
||||
quant_table: 3,
|
||||
trellis_multipass: false,
|
||||
trellis_opt_zero: false,
|
||||
trellis_opt_table: false,
|
||||
trellis_loops: 1,
|
||||
auto_subsample: true,
|
||||
chroma_subsample: 2,
|
||||
separate_chroma_quality: false,
|
||||
chroma_quality: 75,
|
||||
},
|
||||
autoOptimize: {
|
||||
option: 'quality',
|
||||
min: 0,
|
||||
max: 100,
|
||||
},
|
||||
},
|
||||
webp: {
|
||||
name: 'WebP',
|
||||
extension: 'webp',
|
||||
detectors: [/^RIFF....WEBPVP8[LX ]/],
|
||||
dec: () => instantiateEmscriptenWasm(webpDec, webpDecWasm),
|
||||
enc: () => instantiateEmscriptenWasm(webpEnc, webpEncWasm),
|
||||
defaultEncoderOptions: {
|
||||
quality: 75,
|
||||
target_size: 0,
|
||||
target_PSNR: 0,
|
||||
method: 4,
|
||||
sns_strength: 50,
|
||||
filter_strength: 60,
|
||||
filter_sharpness: 0,
|
||||
filter_type: 1,
|
||||
partitions: 0,
|
||||
segments: 4,
|
||||
pass: 1,
|
||||
show_compressed: 0,
|
||||
preprocessing: 0,
|
||||
autofilter: 0,
|
||||
partition_limit: 0,
|
||||
alpha_compression: 1,
|
||||
alpha_filtering: 1,
|
||||
alpha_quality: 100,
|
||||
lossless: 0,
|
||||
exact: 0,
|
||||
image_hint: 0,
|
||||
emulate_jpeg_size: 0,
|
||||
thread_level: 0,
|
||||
low_memory: 0,
|
||||
near_lossless: 100,
|
||||
use_delta_palette: 0,
|
||||
use_sharp_yuv: 0,
|
||||
},
|
||||
autoOptimize: {
|
||||
option: 'quality',
|
||||
min: 0,
|
||||
max: 100,
|
||||
},
|
||||
},
|
||||
avif: {
|
||||
name: 'AVIF',
|
||||
extension: 'avif',
|
||||
detectors: [/^\x00\x00\x00 ftypavif\x00\x00\x00\x00/],
|
||||
dec: () => instantiateEmscriptenWasm(avifDec, avifDecWasm),
|
||||
enc: () => instantiateEmscriptenWasm(avifEnc, avifEncWasm),
|
||||
defaultEncoderOptions: {
|
||||
minQuantizer: 33,
|
||||
maxQuantizer: 63,
|
||||
minQuantizerAlpha: 33,
|
||||
maxQuantizerAlpha: 63,
|
||||
tileColsLog2: 0,
|
||||
tileRowsLog2: 0,
|
||||
speed: 8,
|
||||
subsample: 1,
|
||||
},
|
||||
autoOptimize: {
|
||||
option: 'maxQuantizer',
|
||||
min: 0,
|
||||
max: 62,
|
||||
},
|
||||
},
|
||||
jxl: {
|
||||
name: 'JPEG-XL',
|
||||
extension: 'jxl',
|
||||
detectors: [/^\xff\x0a/],
|
||||
dec: () => instantiateEmscriptenWasm(jxlDec, jxlDecWasm),
|
||||
enc: () => instantiateEmscriptenWasm(jxlEnc, jxlEncWasm),
|
||||
defaultEncoderOptions: {
|
||||
speed: 4,
|
||||
quality: 75,
|
||||
progressive: false,
|
||||
epf: -1,
|
||||
nearLossless: 0,
|
||||
lossyPalette: false,
|
||||
},
|
||||
autoOptimize: {
|
||||
option: 'quality',
|
||||
min: 0,
|
||||
max: 100,
|
||||
},
|
||||
},
|
||||
wp2: {
|
||||
name: 'WebP2',
|
||||
extension: 'wp2',
|
||||
detectors: [/^\xF4\xFF\x6F/],
|
||||
dec: () => instantiateEmscriptenWasm(wp2Dec, wp2DecWasm),
|
||||
enc: () => instantiateEmscriptenWasm(wp2Enc, wp2EncWasm),
|
||||
defaultEncoderOptions: {
|
||||
quality: 75,
|
||||
alpha_quality: 75,
|
||||
effort: 5,
|
||||
pass: 1,
|
||||
sns: 50,
|
||||
uv_mode: 0 /*UVMode.UVModeAuto*/,
|
||||
csp_type: 0 /*Csp.kYCoCg*/,
|
||||
error_diffusion: 0,
|
||||
use_random_matrix: false,
|
||||
},
|
||||
autoOptimize: {
|
||||
option: 'quality',
|
||||
min: 0,
|
||||
max: 100,
|
||||
},
|
||||
},
|
||||
oxipng: {
|
||||
name: 'OxiPNG',
|
||||
extension: 'png',
|
||||
detectors: [/^\x89PNG\x0D\x0A\x1A\x0A/],
|
||||
dec: async () => {
|
||||
await pngEncDecPromise;
|
||||
return { decode: pngEncDec.decode };
|
||||
},
|
||||
enc: async () => {
|
||||
await pngEncDecPromise;
|
||||
await oxipngPromise;
|
||||
return {
|
||||
encode: (buffer, width, height, opts) => {
|
||||
const simplePng = pngEncDec.encode(new Uint8Array(buffer), width, height);
|
||||
return oxipng.optimise(simplePng, opts.level);
|
||||
},
|
||||
};
|
||||
},
|
||||
defaultEncoderOptions: {
|
||||
level: 2,
|
||||
},
|
||||
autoOptimize: {
|
||||
option: 'level',
|
||||
min: 6,
|
||||
max: 1,
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -1,16 +0,0 @@
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
export function pathify(path) {
|
||||
if (path.startsWith('file://')) {
|
||||
path = fileURLToPath(path);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
export function instantiateEmscriptenWasm(factory, path) {
|
||||
return factory({
|
||||
locateFile() {
|
||||
return pathify(path);
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
export default class ImageData {
|
||||
constructor(data, width, height) {
|
||||
this.data = data;
|
||||
this.width = width;
|
||||
this.height = height;
|
||||
}
|
||||
}
|
||||
316
cli/src/index.js
316
cli/src/index.js
@@ -1,17 +1,14 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { program } from 'commander';
|
||||
import JSON5 from 'json5';
|
||||
import { isMainThread } from 'worker_threads';
|
||||
import { cpus } from 'os';
|
||||
import { extname, join, basename } from 'path';
|
||||
import path from 'path';
|
||||
import { promises as fsp } from 'fs';
|
||||
import { resolve as resolvePath } from 'path';
|
||||
import { version } from 'json:../package.json';
|
||||
import ora from 'ora';
|
||||
import kleur from 'kleur';
|
||||
|
||||
import { codecs as supportedFormats, preprocessors } from './codecs.js';
|
||||
import WorkerPool from './worker_pool.js';
|
||||
import { autoOptimize } from './auto-optimizer.js';
|
||||
//Replace package name with '../../api/build/index.js' to test unpublished changes in the API
|
||||
import { ImagePool, manipulations, encoders } from '@squoosh/api';
|
||||
|
||||
function clamp(v, min, max) {
|
||||
if (v < min) return min;
|
||||
@@ -26,114 +23,6 @@ function prettyPrintSize(size) {
|
||||
return (size / 2 ** (10 * index)).toFixed(2) + suffix[index];
|
||||
}
|
||||
|
||||
async function decodeFile(file) {
|
||||
const buffer = await fsp.readFile(file);
|
||||
const firstChunk = buffer.slice(0, 16);
|
||||
const firstChunkString = Array.from(firstChunk)
|
||||
.map((v) => String.fromCodePoint(v))
|
||||
.join('');
|
||||
const key = Object.entries(supportedFormats).find(([name, { detectors }]) =>
|
||||
detectors.some((detector) => detector.exec(firstChunkString)),
|
||||
)?.[0];
|
||||
if (!key) {
|
||||
throw Error(`${file} has an unsupported format`);
|
||||
}
|
||||
const rgba = (await supportedFormats[key].dec()).decode(
|
||||
new Uint8Array(buffer),
|
||||
);
|
||||
return {
|
||||
file,
|
||||
bitmap: rgba,
|
||||
size: buffer.length,
|
||||
};
|
||||
}
|
||||
|
||||
async function preprocessImage({ preprocessorName, options, file }) {
|
||||
const preprocessor = await preprocessors[preprocessorName].instantiate();
|
||||
file.bitmap = await preprocessor(
|
||||
file.bitmap.data,
|
||||
file.bitmap.width,
|
||||
file.bitmap.height,
|
||||
options,
|
||||
);
|
||||
return file;
|
||||
}
|
||||
|
||||
async function encodeFile({
|
||||
file,
|
||||
size,
|
||||
bitmap: bitmapIn,
|
||||
outputFile,
|
||||
encName,
|
||||
encConfig,
|
||||
optimizerButteraugliTarget,
|
||||
maxOptimizerRounds,
|
||||
}) {
|
||||
let out, infoText;
|
||||
const encoder = await supportedFormats[encName].enc();
|
||||
if (encConfig === 'auto') {
|
||||
const optionToOptimize = supportedFormats[encName].autoOptimize.option;
|
||||
const decoder = await supportedFormats[encName].dec();
|
||||
const encode = (bitmapIn, quality) =>
|
||||
encoder.encode(
|
||||
bitmapIn.data,
|
||||
bitmapIn.width,
|
||||
bitmapIn.height,
|
||||
Object.assign({}, supportedFormats[encName].defaultEncoderOptions, {
|
||||
[optionToOptimize]: quality,
|
||||
}),
|
||||
);
|
||||
const decode = (binary) => decoder.decode(binary);
|
||||
const { bitmap, binary, quality } = await autoOptimize(
|
||||
bitmapIn,
|
||||
encode,
|
||||
decode,
|
||||
{
|
||||
min: supportedFormats[encName].autoOptimize.min,
|
||||
max: supportedFormats[encName].autoOptimize.max,
|
||||
butteraugliDistanceGoal: optimizerButteraugliTarget,
|
||||
maxRounds: maxOptimizerRounds,
|
||||
},
|
||||
);
|
||||
out = binary;
|
||||
const opts = {
|
||||
// 5 significant digits is enough
|
||||
[optionToOptimize]: Math.round(quality * 10000) / 10000,
|
||||
};
|
||||
infoText = ` using --${encName} '${JSON5.stringify(opts)}'`;
|
||||
} else {
|
||||
out = encoder.encode(
|
||||
bitmapIn.data.buffer,
|
||||
bitmapIn.width,
|
||||
bitmapIn.height,
|
||||
encConfig,
|
||||
);
|
||||
}
|
||||
await fsp.writeFile(outputFile, out);
|
||||
return {
|
||||
infoText,
|
||||
inputSize: size,
|
||||
inputFile: file,
|
||||
outputFile,
|
||||
outputSize: out.length,
|
||||
};
|
||||
}
|
||||
|
||||
// both decoding and encoding go through the worker pool
|
||||
function handleJob(params) {
|
||||
const { operation } = params;
|
||||
switch (operation) {
|
||||
case 'encode':
|
||||
return encodeFile(params);
|
||||
case 'decode':
|
||||
return decodeFile(params.file);
|
||||
case 'preprocess':
|
||||
return preprocessImage(params);
|
||||
default:
|
||||
throw Error(`Invalid job "${operation}"`);
|
||||
}
|
||||
}
|
||||
|
||||
function progressTracker(results) {
|
||||
const spinner = ora();
|
||||
const tracker = {};
|
||||
@@ -163,11 +52,10 @@ function progressTracker(results) {
|
||||
};
|
||||
function getResultsText() {
|
||||
let out = '';
|
||||
for (const [filename, result] of results.entries()) {
|
||||
out += `\n ${kleur.cyan(filename)}: ${prettyPrintSize(result.size)}`;
|
||||
for (const { outputFile, outputSize, infoText } of result.outputs) {
|
||||
const name = (program.suffix + extname(outputFile)).padEnd(5);
|
||||
out += `\n ${kleur.dim('└')} ${kleur.cyan(name)} → ${prettyPrintSize(
|
||||
for (const result of results.values()) {
|
||||
out += `\n ${kleur.cyan(result.file)}: ${prettyPrintSize(result.size)}`;
|
||||
for (const { outputFile, size: outputSize, infoText } of result.outputs) {
|
||||
out += `\n ${kleur.dim('└')} ${kleur.cyan(outputFile.padEnd(5))} → ${prettyPrintSize(
|
||||
outputSize,
|
||||
)}`;
|
||||
const percent = ((outputSize / result.size) * 100).toPrecision(3);
|
||||
@@ -186,17 +74,17 @@ function progressTracker(results) {
|
||||
async function getInputFiles(paths) {
|
||||
const validFiles = [];
|
||||
|
||||
for (const path of paths) {
|
||||
const files = (await fsp.lstat(path)).isDirectory()
|
||||
? (await fsp.readdir(path)).map(file => join(path, file))
|
||||
: [path];
|
||||
for (const inputPath of paths) {
|
||||
const files = (await fsp.lstat(inputPath)).isDirectory()
|
||||
? (await fsp.readdir(inputPath)).map(file => path.join(inputPath, file))
|
||||
: [inputPath];
|
||||
for (const file of files) {
|
||||
try {
|
||||
await fsp.stat(file);
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') {
|
||||
console.warn(
|
||||
`Warning: Input file does not exist: ${resolvePath(file)}`,
|
||||
`Warning: Input file does not exist: ${path.resolve(file)}`,
|
||||
);
|
||||
continue;
|
||||
} else {
|
||||
@@ -214,7 +102,7 @@ async function getInputFiles(paths) {
|
||||
async function processFiles(files) {
|
||||
files = await getInputFiles(files);
|
||||
|
||||
const parallelism = cpus().length;
|
||||
const imagePool = new ImagePool();
|
||||
|
||||
const results = new Map();
|
||||
const progress = progressTracker(results);
|
||||
@@ -223,140 +111,120 @@ async function processFiles(files) {
|
||||
progress.totalOffset = files.length;
|
||||
progress.setProgress(0, files.length);
|
||||
|
||||
const workerPool = new WorkerPool(parallelism, __filename);
|
||||
// Create output directory
|
||||
await fsp.mkdir(program.outputDir, { recursive: true });
|
||||
|
||||
let decoded = 0;
|
||||
let decodedFiles = await Promise.all(
|
||||
files.map(async (file) => {
|
||||
const result = await workerPool.dispatchJob({
|
||||
operation: 'decode',
|
||||
const image = imagePool.ingestImage(file);
|
||||
await image.decoded;
|
||||
results.set(image, {
|
||||
file,
|
||||
});
|
||||
results.set(file, {
|
||||
file: result.file,
|
||||
size: result.size,
|
||||
size: (await image.decoded).size,
|
||||
outputs: [],
|
||||
});
|
||||
progress.setProgress(++decoded, files.length);
|
||||
return result;
|
||||
return image;
|
||||
}),
|
||||
);
|
||||
|
||||
for (const [preprocessorName, value] of Object.entries(preprocessors)) {
|
||||
if (!program[preprocessorName]) {
|
||||
const manipulationOptions = {};
|
||||
|
||||
for (const manipulatorType of Object.keys(manipulations)) {
|
||||
if (!program[manipulatorType]) {
|
||||
continue;
|
||||
}
|
||||
const preprocessorParam = program[preprocessorName];
|
||||
const preprocessorOptions = Object.assign(
|
||||
{},
|
||||
value.defaultOptions,
|
||||
JSON5.parse(preprocessorParam),
|
||||
);
|
||||
|
||||
decodedFiles = await Promise.all(
|
||||
decodedFiles.map(async (file) => {
|
||||
return workerPool.dispatchJob({
|
||||
file,
|
||||
operation: 'preprocess',
|
||||
preprocessorName,
|
||||
options: preprocessorOptions,
|
||||
});
|
||||
}),
|
||||
);
|
||||
manipulationOptions[manipulatorType] = JSON5.parse(program[manipulatorType]);
|
||||
}
|
||||
|
||||
for(const image of decodedFiles){
|
||||
image.manipulate(manipulationOptions);
|
||||
}
|
||||
|
||||
await Promise.all(decodedFiles.map( (image) => image.decoded ));
|
||||
|
||||
progress.progressOffset = decoded;
|
||||
progress.setStatus('Encoding ' + kleur.dim(`(${parallelism} threads)`));
|
||||
progress.setStatus('Encoding ' + kleur.dim(`(${imagePool.workerPool.numWorkers} threads)`));
|
||||
progress.setProgress(0, files.length);
|
||||
|
||||
const jobs = [];
|
||||
let jobsStarted = 0;
|
||||
let jobsFinished = 0;
|
||||
for (const { file, bitmap, size } of decodedFiles) {
|
||||
const ext = extname(file);
|
||||
const base = basename(file, ext) + program.suffix;
|
||||
for (const image of decodedFiles) {
|
||||
const originalFile = results.get(image).file;
|
||||
|
||||
for (const [encName, value] of Object.entries(supportedFormats)) {
|
||||
const encodeOptions = {
|
||||
optimizerButteraugliTarget: Number(program.optimizerButteraugliTarget),
|
||||
maxOptimizerRounds: Number(program.maxOptimizerRounds),
|
||||
}
|
||||
for (const encName of Object.keys(encoders)) {
|
||||
if (!program[encName]) {
|
||||
continue;
|
||||
}
|
||||
const encParam =
|
||||
typeof program[encName] === 'string' ? program[encName] : '{}';
|
||||
const encConfig =
|
||||
encParam.toLowerCase() === 'auto'
|
||||
? 'auto'
|
||||
: Object.assign(
|
||||
{},
|
||||
value.defaultEncoderOptions,
|
||||
JSON5.parse(encParam),
|
||||
);
|
||||
const outputFile = join(program.outputDir, `${base}.${value.extension}`);
|
||||
jobsStarted++;
|
||||
const p = workerPool
|
||||
.dispatchJob({
|
||||
operation: 'encode',
|
||||
file,
|
||||
size,
|
||||
bitmap,
|
||||
outputFile,
|
||||
encName,
|
||||
encConfig,
|
||||
optimizerButteraugliTarget: Number(
|
||||
program.optimizerButteraugliTarget,
|
||||
),
|
||||
maxOptimizerRounds: Number(program.maxOptimizerRounds),
|
||||
})
|
||||
.then((output) => {
|
||||
jobsFinished++;
|
||||
results.get(file).outputs.push(output);
|
||||
progress.setProgress(jobsFinished, jobsStarted);
|
||||
});
|
||||
jobs.push(p);
|
||||
const encParam = program[encName];
|
||||
const encConfig = encParam.toLowerCase() === 'auto' ? 'auto' : JSON5.parse(encParam);
|
||||
encodeOptions[encName] = encConfig;
|
||||
}
|
||||
jobsStarted++;
|
||||
const job = image.encode(encodeOptions)
|
||||
.then(async () => {
|
||||
jobsFinished++;
|
||||
const outputPath = path.join(program.outputDir, program.suffix + path.basename(originalFile, path.extname(originalFile)));
|
||||
for(const [extension, output] of Object.entries(image.encodedAs)){
|
||||
const outputFile = `${outputPath}.${extension}`;
|
||||
await fsp.writeFile(outputFile, (await output).binary);
|
||||
results.get(image).outputs.push(
|
||||
Object.assign(
|
||||
await output,
|
||||
{outputFile},
|
||||
)
|
||||
);
|
||||
}
|
||||
progress.setProgress(jobsFinished, jobsStarted);
|
||||
});
|
||||
jobs.push(job);
|
||||
}
|
||||
|
||||
// update the progress to account for multi-format
|
||||
progress.setProgress(jobsFinished, jobsStarted);
|
||||
// Wait for all jobs to finish
|
||||
await workerPool.join();
|
||||
await Promise.all(jobs);
|
||||
await imagePool.close();
|
||||
progress.finish('Squoosh results:');
|
||||
}
|
||||
|
||||
if (isMainThread) {
|
||||
program
|
||||
.name('squoosh-cli')
|
||||
.version(version)
|
||||
.arguments('<files...>')
|
||||
.option('-d, --output-dir <dir>', 'Output directory', '.')
|
||||
.option('-s, --suffix <suffix>', 'Append suffix to output files', '')
|
||||
.option(
|
||||
'--max-optimizer-rounds <rounds>',
|
||||
'Maximum number of compressions to use for auto optimizations',
|
||||
'6',
|
||||
)
|
||||
.option(
|
||||
'--optimizer-butteraugli-target <butteraugli distance>',
|
||||
'Target Butteraugli distance for auto optimizer',
|
||||
'1.4',
|
||||
)
|
||||
.action(processFiles);
|
||||
|
||||
// Create a CLI option for each supported preprocessor
|
||||
for (const [key, value] of Object.entries(preprocessors)) {
|
||||
program.option(`--${key} [config]`, value.description);
|
||||
}
|
||||
// Create a CLI option for each supported encoder
|
||||
for (const [key, value] of Object.entries(supportedFormats)) {
|
||||
program.option(
|
||||
`--${key} [config]`,
|
||||
`Use ${value.name} to generate a .${value.extension} file with the given configuration`,
|
||||
);
|
||||
}
|
||||
|
||||
program.parse(process.argv);
|
||||
} else {
|
||||
WorkerPool.useThisThreadAsWorker(handleJob);
|
||||
program
|
||||
.name('squoosh-cli')
|
||||
.arguments('<files...>')
|
||||
.option('-d, --output-dir <dir>', 'Output directory', '.')
|
||||
.option('-s, --suffix <suffix>', 'Append suffix to output files', '')
|
||||
.option(
|
||||
'--max-optimizer-rounds <rounds>',
|
||||
'Maximum number of compressions to use for auto optimizations',
|
||||
'6',
|
||||
)
|
||||
.option(
|
||||
'--optimizer-butteraugli-target <butteraugli distance>',
|
||||
'Target Butteraugli distance for auto optimizer',
|
||||
'1.4',
|
||||
)
|
||||
.action(processFiles);
|
||||
|
||||
// Create a CLI option for each supported preprocessor
|
||||
for (const [key, value] of Object.entries(manipulations)) {
|
||||
program.option(`--${key} [config]`, value.description);
|
||||
}
|
||||
// Create a CLI option for each supported encoder
|
||||
for (const [key, value] of Object.entries(encoders)) {
|
||||
program.option(
|
||||
`--${key} [config]`,
|
||||
`Use ${value.name} to generate a .${value.extension} file with the given configuration`,
|
||||
);
|
||||
}
|
||||
|
||||
program.parse(process.argv);
|
||||
|
||||
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
import { Worker, parentPort } from 'worker_threads';
|
||||
import { TransformStream } from 'web-streams-polyfill';
|
||||
|
||||
function uuid() {
|
||||
return Array.from({ length: 16 }, () =>
|
||||
Math.floor(Math.random() * 256).toString(16),
|
||||
).join('');
|
||||
}
|
||||
|
||||
function jobPromise(worker, msg) {
|
||||
return new Promise((resolve) => {
|
||||
const id = uuid();
|
||||
worker.postMessage({ msg, id });
|
||||
worker.on('message', function f({ result, id: rid }) {
|
||||
if (rid !== id) {
|
||||
return;
|
||||
}
|
||||
worker.off('message', f);
|
||||
resolve(result);
|
||||
});
|
||||
worker.on('error', (error) => console.error('Worker error: ', error));
|
||||
});
|
||||
}
|
||||
|
||||
export default class WorkerPool {
|
||||
constructor(numWorkers, workerFile) {
|
||||
this.numWorkers = numWorkers;
|
||||
this.jobQueue = new TransformStream();
|
||||
this.workerQueue = new TransformStream();
|
||||
|
||||
const writer = this.workerQueue.writable.getWriter();
|
||||
for (let i = 0; i < numWorkers; i++) {
|
||||
writer.write(new Worker(workerFile));
|
||||
}
|
||||
writer.releaseLock();
|
||||
|
||||
this.done = this._readLoop();
|
||||
}
|
||||
|
||||
async _readLoop() {
|
||||
const reader = this.jobQueue.readable.getReader();
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) {
|
||||
await this._terminateAll();
|
||||
return;
|
||||
}
|
||||
const { msg, resolve } = value;
|
||||
const worker = await this._nextWorker();
|
||||
jobPromise(worker, msg).then((result) => {
|
||||
resolve(result);
|
||||
const writer = this.workerQueue.writable.getWriter();
|
||||
writer.write(worker);
|
||||
writer.releaseLock();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async _nextWorker() {
|
||||
const reader = this.workerQueue.readable.getReader();
|
||||
const { value, done } = await reader.read();
|
||||
reader.releaseLock();
|
||||
return value;
|
||||
}
|
||||
|
||||
async _terminateAll() {
|
||||
for (let n = 0; n < this.numWorkers; n++) {
|
||||
const worker = await this._nextWorker();
|
||||
worker.terminate();
|
||||
}
|
||||
this.workerQueue.writable.close();
|
||||
}
|
||||
|
||||
async join() {
|
||||
this.jobQueue.writable.getWriter().close();
|
||||
await this.done;
|
||||
}
|
||||
|
||||
dispatchJob(msg) {
|
||||
return new Promise((resolve) => {
|
||||
const writer = this.jobQueue.writable.getWriter();
|
||||
writer.write({ msg, resolve });
|
||||
writer.releaseLock();
|
||||
});
|
||||
}
|
||||
|
||||
static useThisThreadAsWorker(cb) {
|
||||
parentPort.on('message', async (data) => {
|
||||
const { msg, id } = data;
|
||||
const result = await cb(msg);
|
||||
parentPort.postMessage({ result, id });
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user