external-repos/squoosh
1
0
Fork 1
mirror of https://github.com/GoogleChromeLabs/squoosh.git synced 2025-11-13 17:27:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1123 from styfle/load-file-async

Browse Source 
Reduce `@squoosh/lib` Node.js API usage (make it run on the web)
This commit is contained in:
Surma
2021-09-08 22:30:53 +01:00
committed by GitHub
parent 1ab9e8b3ab 87f25d909b
commit 64cad1cc23
3 changed files with 36 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
cli/src/index.js
View File

@@ -4,6 +4,7 @@ import { program } from 'commander/esm.mjs';
import JSON5 from 'json5';
import path from 'path';
import { promises as fsp } from 'fs';
import { cpus } from 'os';
import ora from 'ora';
import kleur from 'kleur';
@@ -75,7 +76,9 @@ async function getInputFiles(paths) {
for (const inputPath of paths) {
const files = (await fsp.lstat(inputPath)).isDirectory()
? (await fsp.readdir(inputPath, {withFileTypes: true})).filter(dirent => dirent.isFile()).map(dirent => path.join(inputPath, dirent.name))
? (await fsp.readdir(inputPath, { withFileTypes: true }))
.filter((dirent) => dirent.isFile())
.map((dirent) => path.join(inputPath, dirent.name))
: [inputPath];
for (const file of files) {
try {
@@ -101,7 +104,7 @@ async function getInputFiles(paths) {
async function processFiles(files) {
files = await getInputFiles(files);
const imagePool = new ImagePool();
const imagePool = new ImagePool(cpus().length);
const results = new Map();
const progress = progressTracker(results);
@@ -116,7 +119,8 @@ async function processFiles(files) {
let decoded = 0;
let decodedFiles = await Promise.all(
files.map(async (file) => {
const image = imagePool.ingestImage(file);
const buffer = await fsp.readFile(file);
const image = imagePool.ingestImage(buffer);
await image.decoded;
results.set(image, {
file,
@@ -178,7 +182,7 @@ async function processFiles(files) {
const outputPath = path.join(
program.opts().outputDir,
path.basename(originalFile, path.extname(originalFile)) +
program.opts().suffix
program.opts().suffix,
);
for (const output of Object.values(image.encodedWith)) {
const outputFile = `${outputPath}.${(await output).extension}`;

23
libsquoosh/README.md
View File

@@ -16,21 +16,23 @@ You can start using the libSquoosh by adding these lines to the top of your JS p
```js
import { ImagePool } from '@squoosh/lib';
const imagePool = new ImagePool();
import { cpus } from 'os';
const imagePool = new ImagePool(cpus().length);
```
This will create an image pool with an underlying processing pipeline that you can use to ingest and encode images. The ImagePool constructor takes one argument that defines how many parallel operations it is allowed to run at any given time. By default, this number is set to the amount of CPU cores available in the system it is running on.
This will create an image pool with an underlying processing pipeline that you can use to ingest and encode images. The ImagePool constructor takes one argument that defines how many parallel operations it is allowed to run at any given time.
## Ingesting images
You can ingest a new image like so:
```js
const imagePath = 'path/to/image.png';
const image = imagePool.ingestImage(imagePath);
import fs from 'fs/promises';
const file = await fs.readFile('./path/to/image.png');
const image = imagePool.ingestImage(file);
```
The `ingestImage` function can take anything the node [`readFile`][readfile] function can take, including a buffer and `FileHandle`.
The `ingestImage` function can accept any [`ArrayBuffer`][arraybuffer] whether that is from `readFile()` or `fetch()`.
The returned `image` object is a representation of the original image, that you can now preprocess, encode, and extract information about.
@@ -39,7 +41,7 @@ The returned `image` object is a representation of the original image, that you
When an image has been ingested, you can start preprocessing it and encoding it to other formats. This example will resize the image and then encode it to a `.jpg` and `.jxl` image:
```js
await image.decoded; //Wait until the image is decoded before running preprocessors.
await image.decoded; //Wait until the image is decoded before running preprocessors.
const preprocessOptions = {
//When both width and height are specified, the image resized to specified size.
@@ -47,7 +49,7 @@ const preprocessOptions = {
enabled: true,
width: 100,
height: 50,
}
},
/*
//When either width or height is specified, the image resized to specified size keeping aspect ratio.
resize: {
@@ -55,7 +57,7 @@ const preprocessOptions = {
width: 100,
}
*/
}
};
await image.preprocess(preprocessOptions);
const encodeOptions = {
@@ -63,9 +65,8 @@ const encodeOptions = {
jxl: {
quality: 90,
},
}
};
await image.encode(encodeOptions);
```
The default values for each option can be found in the [`codecs.ts`][codecs.ts] file under `defaultEncoderOptions`. Every unspecified value will use the default value specified there. _Better documentation is needed here._
@@ -168,4 +169,4 @@ const encodeOptions: {
[squoosh]: https://squoosh.app
[codecs.ts]: https://github.com/GoogleChromeLabs/squoosh/blob/dev/libsquoosh/src/codecs.ts
[butteraugli]: https://github.com/google/butteraugli
[readfile]: https://nodejs.org/api/fs.html#fs_fspromises_readfile_path_options
[arraybuffer]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer

47
libsquoosh/src/index.ts
View File

@@ -1,6 +1,4 @@
import { isMainThread } from 'worker_threads';
import { cpus } from 'os';
import { promises as fsp } from 'fs';
import {
AvifEncodeOptions,
@@ -22,7 +20,6 @@ import type ImageData from './image_data';
export { ImagePool, encoders, preprocessors };
type EncoderKey = keyof typeof encoders;
type PreprocessorKey = keyof typeof preprocessors;
type FileLike = Buffer | ArrayBuffer | string | ArrayBufferView;
type PreprocessOptions = {
resize?: ResizeOptions;
@@ -33,25 +30,10 @@ type PreprocessOptions = {
async function decodeFile({
file,
}: {
file: FileLike;
file: ArrayBuffer | ArrayLike<number>;
}): Promise<{ bitmap: ImageData; size: number }> {
let buffer;
if (ArrayBuffer.isView(file)) {
buffer = Buffer.from(file.buffer);
file = 'Binary blob';
} else if (file instanceof ArrayBuffer) {
buffer = Buffer.from(file);
file = 'Binary blob';
} else if ((file as unknown) instanceof Buffer) {
// TODO: Check why we need type assertions here.
buffer = (file as unknown) as Buffer;
file = 'Binary blob';
} else if (typeof file === 'string') {
buffer = await fsp.readFile(file);
} else {
throw Error('Unexpected input type');
}
const firstChunk = buffer.slice(0, 16);
const array = new Uint8Array(file);
const firstChunk = array.slice(0, 16);
const firstChunkString = Array.from(firstChunk)
.map((v) => String.fromCodePoint(v))
.join('');
@@ -59,14 +41,14 @@ async function decodeFile({
detectors.some((detector) => detector.exec(firstChunkString)),
)?.[0] as EncoderKey | undefined;
if (!key) {
throw Error(`${file} has an unsupported format`);
throw Error(`File has an unsupported format`);
}
const encoder = encoders[key];
const mod = await encoder.dec();
const rgba = mod.decode(new Uint8Array(buffer));
const rgba = mod.decode(array);
return {
bitmap: rgba,
size: buffer.length,
size: array.length,
};
}
@@ -187,12 +169,15 @@ function handleJob(params: JobMessage) {
* Represents an ingested image.
*/
class Image {
public file: FileLike;
public file: ArrayBuffer | ArrayLike<number>;
public workerPool: WorkerPool<JobMessage, any>;
public decoded: Promise<{ bitmap: ImageData }>;
public encodedWith: { [key: string]: any };
constructor(workerPool: WorkerPool<JobMessage, any>, file: FileLike) {
constructor(
workerPool: WorkerPool<JobMessage, any>,
file: ArrayBuffer | ArrayLike<number>,
) {
this.file = file;
this.workerPool = workerPool;
this.decoded = workerPool.dispatchJob({ operation: 'decode', file });
@@ -277,19 +262,19 @@ class ImagePool {
/**
* Create a new pool.
* @param {number} [threads] - Number of concurrent image processes to run in the pool. Defaults to the number of CPU cores in the system.
* @param {number} [threads] - Number of concurrent image processes to run in the pool.
*/
constructor(threads: number) {
this.workerPool = new WorkerPool(threads || cpus().length, __filename);
this.workerPool = new WorkerPool(threads, __filename);
}
/**
* Ingest an image into the image pool.
* @param {FileLike} image - The image or path to the image that should be ingested and decoded.
* @param {ArrayBuffer | ArrayLike<number>} file - The image that should be ingested and decoded.
* @returns {Image} - A custom class reference to the decoded image.
*/
ingestImage(image: FileLike): Image {
return new Image(this.workerPool, image);
ingestImage(file: ArrayBuffer | ArrayLike<number>): Image {
return new Image(this.workerPool, file);
}
/**