mirror of
https://github.com/GoogleChromeLabs/squoosh.git
synced 2025-11-15 18:19:47 +00:00
Separate CLI and API
This commit is contained in:
atjn
316
cli/src/index.js
316
cli/src/index.js
@@ -1,17 +1,14 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { program } from 'commander';
|
||||
import JSON5 from 'json5';
|
||||
import { isMainThread } from 'worker_threads';
|
||||
import { cpus } from 'os';
|
||||
import { extname, join, basename } from 'path';
|
||||
import path from 'path';
|
||||
import { promises as fsp } from 'fs';
|
||||
import { resolve as resolvePath } from 'path';
|
||||
import { version } from 'json:../package.json';
|
||||
import ora from 'ora';
|
||||
import kleur from 'kleur';
|
||||
|
||||
import { codecs as supportedFormats, preprocessors } from './codecs.js';
|
||||
import WorkerPool from './worker_pool.js';
|
||||
import { autoOptimize } from './auto-optimizer.js';
|
||||
//Replace package name with '../../api/build/index.js' to test unpublished changes in the API
|
||||
import { ImagePool, manipulations, encoders } from '@squoosh/api';
|
||||
|
||||
function clamp(v, min, max) {
|
||||
if (v < min) return min;
|
||||
@@ -26,114 +23,6 @@ function prettyPrintSize(size) {
|
||||
return (size / 2 ** (10 * index)).toFixed(2) + suffix[index];
|
||||
}
|
||||
|
||||
async function decodeFile(file) {
|
||||
const buffer = await fsp.readFile(file);
|
||||
const firstChunk = buffer.slice(0, 16);
|
||||
const firstChunkString = Array.from(firstChunk)
|
||||
.map((v) => String.fromCodePoint(v))
|
||||
.join('');
|
||||
const key = Object.entries(supportedFormats).find(([name, { detectors }]) =>
|
||||
detectors.some((detector) => detector.exec(firstChunkString)),
|
||||
)?.[0];
|
||||
if (!key) {
|
||||
throw Error(`${file} has an unsupported format`);
|
||||
}
|
||||
const rgba = (await supportedFormats[key].dec()).decode(
|
||||
new Uint8Array(buffer),
|
||||
);
|
||||
return {
|
||||
file,
|
||||
bitmap: rgba,
|
||||
size: buffer.length,
|
||||
};
|
||||
}
|
||||
|
||||
async function preprocessImage({ preprocessorName, options, file }) {
|
||||
const preprocessor = await preprocessors[preprocessorName].instantiate();
|
||||
file.bitmap = await preprocessor(
|
||||
file.bitmap.data,
|
||||
file.bitmap.width,
|
||||
file.bitmap.height,
|
||||
options,
|
||||
);
|
||||
return file;
|
||||
}
|
||||
|
||||
async function encodeFile({
|
||||
file,
|
||||
size,
|
||||
bitmap: bitmapIn,
|
||||
outputFile,
|
||||
encName,
|
||||
encConfig,
|
||||
optimizerButteraugliTarget,
|
||||
maxOptimizerRounds,
|
||||
}) {
|
||||
let out, infoText;
|
||||
const encoder = await supportedFormats[encName].enc();
|
||||
if (encConfig === 'auto') {
|
||||
const optionToOptimize = supportedFormats[encName].autoOptimize.option;
|
||||
const decoder = await supportedFormats[encName].dec();
|
||||
const encode = (bitmapIn, quality) =>
|
||||
encoder.encode(
|
||||
bitmapIn.data,
|
||||
bitmapIn.width,
|
||||
bitmapIn.height,
|
||||
Object.assign({}, supportedFormats[encName].defaultEncoderOptions, {
|
||||
[optionToOptimize]: quality,
|
||||
}),
|
||||
);
|
||||
const decode = (binary) => decoder.decode(binary);
|
||||
const { bitmap, binary, quality } = await autoOptimize(
|
||||
bitmapIn,
|
||||
encode,
|
||||
decode,
|
||||
{
|
||||
min: supportedFormats[encName].autoOptimize.min,
|
||||
max: supportedFormats[encName].autoOptimize.max,
|
||||
butteraugliDistanceGoal: optimizerButteraugliTarget,
|
||||
maxRounds: maxOptimizerRounds,
|
||||
},
|
||||
);
|
||||
out = binary;
|
||||
const opts = {
|
||||
// 5 significant digits is enough
|
||||
[optionToOptimize]: Math.round(quality * 10000) / 10000,
|
||||
};
|
||||
infoText = ` using --${encName} '${JSON5.stringify(opts)}'`;
|
||||
} else {
|
||||
out = encoder.encode(
|
||||
bitmapIn.data.buffer,
|
||||
bitmapIn.width,
|
||||
bitmapIn.height,
|
||||
encConfig,
|
||||
);
|
||||
}
|
||||
await fsp.writeFile(outputFile, out);
|
||||
return {
|
||||
infoText,
|
||||
inputSize: size,
|
||||
inputFile: file,
|
||||
outputFile,
|
||||
outputSize: out.length,
|
||||
};
|
||||
}
|
||||
|
||||
// both decoding and encoding go through the worker pool
|
||||
function handleJob(params) {
|
||||
const { operation } = params;
|
||||
switch (operation) {
|
||||
case 'encode':
|
||||
return encodeFile(params);
|
||||
case 'decode':
|
||||
return decodeFile(params.file);
|
||||
case 'preprocess':
|
||||
return preprocessImage(params);
|
||||
default:
|
||||
throw Error(`Invalid job "${operation}"`);
|
||||
}
|
||||
}
|
||||
|
||||
function progressTracker(results) {
|
||||
const spinner = ora();
|
||||
const tracker = {};
|
||||
@@ -163,11 +52,10 @@ function progressTracker(results) {
|
||||
};
|
||||
function getResultsText() {
|
||||
let out = '';
|
||||
for (const [filename, result] of results.entries()) {
|
||||
out += `\n ${kleur.cyan(filename)}: ${prettyPrintSize(result.size)}`;
|
||||
for (const { outputFile, outputSize, infoText } of result.outputs) {
|
||||
const name = (program.suffix + extname(outputFile)).padEnd(5);
|
||||
out += `\n ${kleur.dim('└')} ${kleur.cyan(name)} → ${prettyPrintSize(
|
||||
for (const result of results.values()) {
|
||||
out += `\n ${kleur.cyan(result.file)}: ${prettyPrintSize(result.size)}`;
|
||||
for (const { outputFile, size: outputSize, infoText } of result.outputs) {
|
||||
out += `\n ${kleur.dim('└')} ${kleur.cyan(outputFile.padEnd(5))} → ${prettyPrintSize(
|
||||
outputSize,
|
||||
)}`;
|
||||
const percent = ((outputSize / result.size) * 100).toPrecision(3);
|
||||
@@ -186,17 +74,17 @@ function progressTracker(results) {
|
||||
async function getInputFiles(paths) {
|
||||
const validFiles = [];
|
||||
|
||||
for (const path of paths) {
|
||||
const files = (await fsp.lstat(path)).isDirectory()
|
||||
? (await fsp.readdir(path)).map(file => join(path, file))
|
||||
: [path];
|
||||
for (const inputPath of paths) {
|
||||
const files = (await fsp.lstat(inputPath)).isDirectory()
|
||||
? (await fsp.readdir(inputPath)).map(file => path.join(inputPath, file))
|
||||
: [inputPath];
|
||||
for (const file of files) {
|
||||
try {
|
||||
await fsp.stat(file);
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') {
|
||||
console.warn(
|
||||
`Warning: Input file does not exist: ${resolvePath(file)}`,
|
||||
`Warning: Input file does not exist: ${path.resolve(file)}`,
|
||||
);
|
||||
continue;
|
||||
} else {
|
||||
@@ -214,7 +102,7 @@ async function getInputFiles(paths) {
|
||||
async function processFiles(files) {
|
||||
files = await getInputFiles(files);
|
||||
|
||||
const parallelism = cpus().length;
|
||||
const imagePool = new ImagePool();
|
||||
|
||||
const results = new Map();
|
||||
const progress = progressTracker(results);
|
||||
@@ -223,140 +111,120 @@ async function processFiles(files) {
|
||||
progress.totalOffset = files.length;
|
||||
progress.setProgress(0, files.length);
|
||||
|
||||
const workerPool = new WorkerPool(parallelism, __filename);
|
||||
// Create output directory
|
||||
await fsp.mkdir(program.outputDir, { recursive: true });
|
||||
|
||||
let decoded = 0;
|
||||
let decodedFiles = await Promise.all(
|
||||
files.map(async (file) => {
|
||||
const result = await workerPool.dispatchJob({
|
||||
operation: 'decode',
|
||||
const image = imagePool.ingestImage(file);
|
||||
await image.decoded;
|
||||
results.set(image, {
|
||||
file,
|
||||
});
|
||||
results.set(file, {
|
||||
file: result.file,
|
||||
size: result.size,
|
||||
size: (await image.decoded).size,
|
||||
outputs: [],
|
||||
});
|
||||
progress.setProgress(++decoded, files.length);
|
||||
return result;
|
||||
return image;
|
||||
}),
|
||||
);
|
||||
|
||||
for (const [preprocessorName, value] of Object.entries(preprocessors)) {
|
||||
if (!program[preprocessorName]) {
|
||||
const manipulationOptions = {};
|
||||
|
||||
for (const manipulatorType of Object.keys(manipulations)) {
|
||||
if (!program[manipulatorType]) {
|
||||
continue;
|
||||
}
|
||||
const preprocessorParam = program[preprocessorName];
|
||||
const preprocessorOptions = Object.assign(
|
||||
{},
|
||||
value.defaultOptions,
|
||||
JSON5.parse(preprocessorParam),
|
||||
);
|
||||
|
||||
decodedFiles = await Promise.all(
|
||||
decodedFiles.map(async (file) => {
|
||||
return workerPool.dispatchJob({
|
||||
file,
|
||||
operation: 'preprocess',
|
||||
preprocessorName,
|
||||
options: preprocessorOptions,
|
||||
});
|
||||
}),
|
||||
);
|
||||
manipulationOptions[manipulatorType] = JSON5.parse(program[manipulatorType]);
|
||||
}
|
||||
|
||||
for(const image of decodedFiles){
|
||||
image.manipulate(manipulationOptions);
|
||||
}
|
||||
|
||||
await Promise.all(decodedFiles.map( (image) => image.decoded ));
|
||||
|
||||
progress.progressOffset = decoded;
|
||||
progress.setStatus('Encoding ' + kleur.dim(`(${parallelism} threads)`));
|
||||
progress.setStatus('Encoding ' + kleur.dim(`(${imagePool.workerPool.numWorkers} threads)`));
|
||||
progress.setProgress(0, files.length);
|
||||
|
||||
const jobs = [];
|
||||
let jobsStarted = 0;
|
||||
let jobsFinished = 0;
|
||||
for (const { file, bitmap, size } of decodedFiles) {
|
||||
const ext = extname(file);
|
||||
const base = basename(file, ext) + program.suffix;
|
||||
for (const image of decodedFiles) {
|
||||
const originalFile = results.get(image).file;
|
||||
|
||||
for (const [encName, value] of Object.entries(supportedFormats)) {
|
||||
const encodeOptions = {
|
||||
optimizerButteraugliTarget: Number(program.optimizerButteraugliTarget),
|
||||
maxOptimizerRounds: Number(program.maxOptimizerRounds),
|
||||
}
|
||||
for (const encName of Object.keys(encoders)) {
|
||||
if (!program[encName]) {
|
||||
continue;
|
||||
}
|
||||
const encParam =
|
||||
typeof program[encName] === 'string' ? program[encName] : '{}';
|
||||
const encConfig =
|
||||
encParam.toLowerCase() === 'auto'
|
||||
? 'auto'
|
||||
: Object.assign(
|
||||
{},
|
||||
value.defaultEncoderOptions,
|
||||
JSON5.parse(encParam),
|
||||
);
|
||||
const outputFile = join(program.outputDir, `${base}.${value.extension}`);
|
||||
jobsStarted++;
|
||||
const p = workerPool
|
||||
.dispatchJob({
|
||||
operation: 'encode',
|
||||
file,
|
||||
size,
|
||||
bitmap,
|
||||
outputFile,
|
||||
encName,
|
||||
encConfig,
|
||||
optimizerButteraugliTarget: Number(
|
||||
program.optimizerButteraugliTarget,
|
||||
),
|
||||
maxOptimizerRounds: Number(program.maxOptimizerRounds),
|
||||
})
|
||||
.then((output) => {
|
||||
jobsFinished++;
|
||||
results.get(file).outputs.push(output);
|
||||
progress.setProgress(jobsFinished, jobsStarted);
|
||||
});
|
||||
jobs.push(p);
|
||||
const encParam = program[encName];
|
||||
const encConfig = encParam.toLowerCase() === 'auto' ? 'auto' : JSON5.parse(encParam);
|
||||
encodeOptions[encName] = encConfig;
|
||||
}
|
||||
jobsStarted++;
|
||||
const job = image.encode(encodeOptions)
|
||||
.then(async () => {
|
||||
jobsFinished++;
|
||||
const outputPath = path.join(program.outputDir, program.suffix + path.basename(originalFile, path.extname(originalFile)));
|
||||
for(const [extension, output] of Object.entries(image.encodedAs)){
|
||||
const outputFile = `${outputPath}.${extension}`;
|
||||
await fsp.writeFile(outputFile, (await output).binary);
|
||||
results.get(image).outputs.push(
|
||||
Object.assign(
|
||||
await output,
|
||||
{outputFile},
|
||||
)
|
||||
);
|
||||
}
|
||||
progress.setProgress(jobsFinished, jobsStarted);
|
||||
});
|
||||
jobs.push(job);
|
||||
}
|
||||
|
||||
// update the progress to account for multi-format
|
||||
progress.setProgress(jobsFinished, jobsStarted);
|
||||
// Wait for all jobs to finish
|
||||
await workerPool.join();
|
||||
await Promise.all(jobs);
|
||||
await imagePool.close();
|
||||
progress.finish('Squoosh results:');
|
||||
}
|
||||
|
||||
if (isMainThread) {
|
||||
program
|
||||
.name('squoosh-cli')
|
||||
.version(version)
|
||||
.arguments('<files...>')
|
||||
.option('-d, --output-dir <dir>', 'Output directory', '.')
|
||||
.option('-s, --suffix <suffix>', 'Append suffix to output files', '')
|
||||
.option(
|
||||
'--max-optimizer-rounds <rounds>',
|
||||
'Maximum number of compressions to use for auto optimizations',
|
||||
'6',
|
||||
)
|
||||
.option(
|
||||
'--optimizer-butteraugli-target <butteraugli distance>',
|
||||
'Target Butteraugli distance for auto optimizer',
|
||||
'1.4',
|
||||
)
|
||||
.action(processFiles);
|
||||
|
||||
// Create a CLI option for each supported preprocessor
|
||||
for (const [key, value] of Object.entries(preprocessors)) {
|
||||
program.option(`--${key} [config]`, value.description);
|
||||
}
|
||||
// Create a CLI option for each supported encoder
|
||||
for (const [key, value] of Object.entries(supportedFormats)) {
|
||||
program.option(
|
||||
`--${key} [config]`,
|
||||
`Use ${value.name} to generate a .${value.extension} file with the given configuration`,
|
||||
);
|
||||
}
|
||||
|
||||
program.parse(process.argv);
|
||||
} else {
|
||||
WorkerPool.useThisThreadAsWorker(handleJob);
|
||||
program
|
||||
.name('squoosh-cli')
|
||||
.arguments('<files...>')
|
||||
.option('-d, --output-dir <dir>', 'Output directory', '.')
|
||||
.option('-s, --suffix <suffix>', 'Append suffix to output files', '')
|
||||
.option(
|
||||
'--max-optimizer-rounds <rounds>',
|
||||
'Maximum number of compressions to use for auto optimizations',
|
||||
'6',
|
||||
)
|
||||
.option(
|
||||
'--optimizer-butteraugli-target <butteraugli distance>',
|
||||
'Target Butteraugli distance for auto optimizer',
|
||||
'1.4',
|
||||
)
|
||||
.action(processFiles);
|
||||
|
||||
// Create a CLI option for each supported preprocessor
|
||||
for (const [key, value] of Object.entries(manipulations)) {
|
||||
program.option(`--${key} [config]`, value.description);
|
||||
}
|
||||
// Create a CLI option for each supported encoder
|
||||
for (const [key, value] of Object.entries(encoders)) {
|
||||
program.option(
|
||||
`--${key} [config]`,
|
||||
`Use ${value.name} to generate a .${value.extension} file with the given configuration`,
|
||||
);
|
||||
}
|
||||
|
||||
program.parse(process.argv);
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user