external-repos/squoosh
1
0
Fork 1
mirror of https://github.com/GoogleChromeLabs/squoosh.git synced 2025-11-13 17:27:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Generating feature metadata

Browse Source
This commit is contained in:
Jake Archibald
2020-09-29 17:56:19 +01:00
parent db2d6f4ca6
commit 7836d7e97c
16 changed files with 332 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -5,7 +5,8 @@ node_modules
build
*.o
# Auto-generated by lib/image-worker-plugin.js
# Auto-generated by lib/feature-plugin.js
src/features-worker/index.ts
src/features-worker/tsconfig.json
src/client/lazy-app/worker-bridge/meta.ts
src/client/lazy-app/feature-meta/index.ts

213
lib/feature-plugin.js Normal file
View File

@@ -0,0 +1,213 @@
/**
* Copyright 2020 Google Inc. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { promisify } from 'util';
import * as path from 'path';
import { promises as fsp } from 'fs';
import glob from 'glob';
const globP = promisify(glob);
const autoGenComment =
'// This file is autogenerated by lib/feature-plugin.js\n';
export default function () {
let previousWorkerImports;
let previousEncoderMetas;
/**
* Generates the worker file & tsconfig for all features
*
* @param {string[]} workerImports
*/
async function generateWorkerFile(workerImports) {
const workerBasePath = path.join(process.cwd(), 'src', 'features-worker');
const featuresWorkerTsNames = workerImports.map((tsImport) => [
path.relative(workerBasePath, tsImport),
path.basename(tsImport),
]);
const workerFile = [
autoGenComment,
`import { expose } from 'comlink';`,
`import { timed } from './util';`,
featuresWorkerTsNames.map(
([path, name]) => `import ${name} from './${path}';`,
),
`const exports = {`,
featuresWorkerTsNames.map(([_, name]) => [
` ${name}(`,
` ...args: Parameters<typeof ${name}>`,
` ): ReturnType<typeof ${name}> {`,
` return timed('${name}', () => ${name}(...args));`,
` },`,
]),
`};`,
`export type ProcessorWorkerApi = typeof exports;`,
`expose(exports, self);`,
]
.flat(Infinity)
.join('\n');
const workerTsConfig = {
extends: '../../generic-tsconfig.json',
compilerOptions: {
lib: ['webworker', 'esnext'],
},
references: featuresWorkerTsNames.map(([tsImport]) => ({
path: path.dirname(tsImport),
})),
};
await Promise.all([
fsp.writeFile(
path.join(workerBasePath, 'tsconfig.json'),
JSON.stringify(workerTsConfig, null, ' '),
),
fsp.writeFile(path.join(workerBasePath, 'index.ts'), workerFile),
]);
}
/**
* Generates the client JS to call worker methods.
*
* @param {string[]} workerImports
*/
async function generateWorkerBridge(workerImports) {
const workerBridgeBasePath = path.join(
process.cwd(),
'src',
'client',
'lazy-app',
'worker-bridge',
);
const featuresWorkerBridgeTsNames = workerImports.map((tsImport) => [
path.relative(workerBridgeBasePath, tsImport),
path.basename(tsImport),
]);
const bridgeMeta = [
autoGenComment,
featuresWorkerBridgeTsNames.map(
([path, name]) => `import type ${name} from '${path}';`,
),
`export const methodNames = ${JSON.stringify(
featuresWorkerBridgeTsNames.map(([_, name]) => name),
null,
' ',
)} as const;`,
`export interface BridgeMethods {`,
featuresWorkerBridgeTsNames.map(([_, name]) => [
` ${name}(`,
` signal: AbortSignal,`,
` ...args: Parameters<typeof ${name}>`,
` ): Promise<ReturnType<typeof ${name}>>;`,
]),
`}`,
]
.flat(Infinity)
.join('\n');
await fsp.writeFile(path.join(workerBridgeBasePath, 'meta.ts'), bridgeMeta);
}
async function generateWorkerFiles() {
const workerImports = (
await globP('src/features/*/**/worker/*.ts', {
absolute: true,
})
)
.filter((tsFile) => !tsFile.endsWith('.d.ts'))
.map((tsFile) => tsFile.slice(0, -'.ts'.length));
const joinedWorkerImports = workerImports.join();
// Avoid regenerating if nothing's changed.
// This also prevents an infinite look in the watcher.
if (joinedWorkerImports === previousWorkerImports) return;
previousWorkerImports = joinedWorkerImports;
await Promise.all([
generateWorkerFile(workerImports),
generateWorkerBridge(workerImports),
]);
}
async function generateFeatureMeta() {
const encoderMetas = (
await globP('src/features/encoders/*/shared/meta.ts', {
absolute: true,
})
)
.filter((tsFile) => !tsFile.endsWith('.d.ts'))
.map((tsFile) => tsFile.slice(0, -'.ts'.length));
const featureMetaBasePath = path.join(
process.cwd(),
'src',
'client',
'lazy-app',
'feature-meta',
);
const joinedEncoderMetas = encoderMetas.join();
// Avoid regenerating if nothing's changed.
// This also prevents an infinite look in the watcher.
if (joinedEncoderMetas === previousEncoderMetas) return;
previousEncoderMetas = joinedEncoderMetas;
const encoderMetaTsNames = encoderMetas.map((tsImport) => [
path.relative(featureMetaBasePath, tsImport),
path.basename(tsImport.slice(0, -'/shared/meta'.length)),
]);
const featureMeta = [
autoGenComment,
encoderMetaTsNames.map(
([path, name]) => `import * as ${name}EncoderMeta from '${path}';`,
),
`export type EncoderState =`,
encoderMetaTsNames.map(
([_, name]) =>
` | { type: "${name}", options: ${name}EncoderMeta.EncodeOptions }`,
),
`;`,
`export type EncoderOptions =`,
encoderMetaTsNames.map(
([_, name]) => ` | ${name}EncoderMeta.EncodeOptions`,
),
`;`,
`export const encoderMap = {`,
encoderMetaTsNames.map(([_, name]) => ` ${name}: ${name}EncoderMeta,`),
`};`,
`export type EncoderType = keyof typeof encoderMap`,
`export const encoders = [...Object.values(encoderMap)];`,
]
.flat(Infinity)
.join('\n');
await fsp.writeFile(
path.join(featureMetaBasePath, 'index.ts'),
featureMeta,
);
}
return {
name: 'feature-plugin',
async buildStart() {
await Promise.all([generateWorkerFiles(), generateFeatureMeta()]);
},
};
}

129
lib/image-worker-plugin.js
View File

@@ -1,129 +0,0 @@
/**
* Copyright 2020 Google Inc. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { promisify } from 'util';
import * as path from 'path';
import { promises as fsp } from 'fs';
import glob from 'glob';
const globP = promisify(glob);
export default function () {
let previousWorkerContent;
return {
name: 'image-worker-plugin',
async buildStart() {
const featuresWorkerBase = path.join(
process.cwd(),
'src',
'features-worker',
);
const featuresWorkerBridgeBase = path.join(
process.cwd(),
'src',
'client',
'lazy-app',
'worker-bridge',
);
const tsImports = (
await globP('src/features/*/**/worker/*.ts', {
absolute: true,
})
)
.filter((tsFile) => !tsFile.endsWith('.d.ts'))
.map((tsFile) => tsFile.slice(0, -'.ts'.length));
const featuresWorkerTsNames = tsImports.map((tsImport) => [
path.relative(featuresWorkerBase, tsImport),
path.basename(tsImport),
]);
const featuresWorkerBridgeTsNames = tsImports.map((tsImport) => [
path.relative(featuresWorkerBridgeBase, tsImport),
path.basename(tsImport),
]);
const workerFile = [
`// This file is autogenerated by lib/image-worker-plugin.js`,
`import { expose } from 'comlink';`,
`import { timed } from './util';`,
featuresWorkerTsNames.map(
([path, name]) => `import ${name} from './${path}';`,
),
`const exports = {`,
featuresWorkerTsNames.map(([_, name]) => [
` ${name}(`,
` ...args: Parameters<typeof ${name}>`,
` ): ReturnType<typeof ${name}> {`,
` return timed('${name}', () => ${name}(...args));`,
` },`,
]),
`};`,
`export type ProcessorWorkerApi = typeof exports;`,
`expose(exports, self);`,
]
.flat(Infinity)
.join('\n');
// If nothing's changed, avoid touching the file to avoid infinite rebuilding in watch mode
if (previousWorkerContent === workerFile) return;
previousWorkerContent = workerFile;
const workerTsConfig = {
extends: '../../generic-tsconfig.json',
compilerOptions: {
lib: ['webworker', 'esnext'],
},
references: featuresWorkerTsNames.map(([tsImport]) => ({
path: path.dirname(tsImport),
})),
};
const bridgeMeta = [
`// This file is autogenerated by lib/image-worker-plugin.js`,
featuresWorkerBridgeTsNames.map(
([path, name]) => `import type ${name} from '${path}';`,
),
`export const methodNames = ${JSON.stringify(
featuresWorkerBridgeTsNames.map(([_, name]) => name),
null,
' ',
)} as const;`,
`export interface BridgeMethods {`,
featuresWorkerBridgeTsNames.map(([_, name]) => [
` ${name}(`,
` signal: AbortSignal,`,
` ...args: Parameters<typeof ${name}>`,
` ): Promise<ReturnType<typeof ${name}>>;`,
]),
`}`,
]
.flat(Infinity)
.join('\n');
await Promise.all([
fsp.writeFile(
path.join(featuresWorkerBase, 'tsconfig.json'),
JSON.stringify(workerTsConfig, null, ' '),
),
fsp.writeFile(path.join(featuresWorkerBase, 'index.ts'), workerFile),
fsp.writeFile(
path.join(featuresWorkerBridgeBase, 'meta.ts'),
bridgeMeta,
),
]);
},
};
}

4
rollup.config.js
View File

@@ -27,7 +27,7 @@ import urlPlugin from './lib/url-plugin';
import resolveDirsPlugin from './lib/resolve-dirs-plugin';
import runScript from './lib/run-script';
import emitFiles from './lib/emit-files-plugin';
import imageWorkerPlugin from './lib/image-worker-plugin';
import featurePlugin from './lib/feature-plugin';
import initialCssPlugin from './lib/initial-css-plugin';
import serviceWorkerPlugin from './lib/sw-plugin';
import dataURLPlugin from './lib/data-url-plugin';
@@ -122,7 +122,7 @@ export default async function ({ watch }) {
...commonPlugins(),
emitFiles({ include: '**/*', root: path.join(__dirname, 'src', 'copy') }),
nodeExternalPlugin(),
imageWorkerPlugin(),
featurePlugin(),
replace({ __PRERENDER__: true, __PRODUCTION__: isProduction }),
initialCssPlugin(),
runScript(dir + '/static-build/index.js'),

0
src/client/lazy-app/feature-meta/.gitignore vendored Normal file
View File

2
src/client/lazy-app/worker-bridge/index.ts
View File

@@ -5,7 +5,7 @@ import type { ProcessorWorkerApi } from '../../../features-worker';
import { abortable } from '../util';
/** How long the worker should be idle before terminating. */
const workerTimeout = 10000;
const workerTimeout = 10_000;
interface WorkerBridge extends BridgeMethods {}

29
src/features/encoders/avif/shared/meta.ts Normal file
View File

@@ -0,0 +1,29 @@
/**
* Copyright 2020 Google Inc. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { EncodeOptions } from 'codecs/avif/enc/avif_enc';
export { EncodeOptions };
export const label = 'AVIF';
export const mimeType = 'image/avif';
export const extension = 'avif';
export const defaultOptions: EncodeOptions = {
minQuantizer: 33,
maxQuantizer: 63,
minQuantizerAlpha: 33,
maxQuantizerAlpha: 63,
tileColsLog2: 0,
tileRowsLog2: 0,
speed: 8,
subsample: 1,
};

13
src/features/encoders/avif/shared/missing-types.d.ts vendored Normal file
View File

@@ -0,0 +1,13 @@
/**
* Copyright 2020 Google Inc. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/// <reference path="../../../../../missing-types.d.ts" />

7
src/features/encoders/avif/shared/tsconfig.json Normal file
View File

@@ -0,0 +1,7 @@
{
"extends": "../../../../../generic-tsconfig.json",
"compilerOptions": {
"lib": ["webworker", "esnext"]
},
"references": [{ "path": "../../../" }]
}

6
src/features/encoders/avif/worker/avifEncode.ts
View File

@@ -10,10 +10,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import avifEncoder, {
AVIFModule,
EncodeOptions,
} from 'codecs/avif/enc/avif_enc';
import avifEncoder, { AVIFModule } from 'codecs/avif/enc/avif_enc';
import type { EncodeOptions } from '../shared/meta';
import wasmUrl from 'url:codecs/avif/enc/avif_enc.wasm';
import { initEmscriptenModule } from 'features/util';

2
src/features/encoders/avif/worker/tsconfig.json
View File

@@ -3,5 +3,5 @@
"compilerOptions": {
"lib": ["webworker", "esnext"]
},
"references": [{ "path": "../../../" }]
"references": [{ "path": "../../../" }, { "path": "../shared" }]
}

39
src/features/encoders/mozjpeg/shared/meta.ts Normal file
View File

@@ -0,0 +1,39 @@
/**
* Copyright 2020 Google Inc. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
EncodeOptions,
MozJpegColorSpace,
} from 'codecs/mozjpeg_enc/mozjpeg_enc';
export { EncodeOptions, MozJpegColorSpace };
export const label = 'MozJPEG';
export const mimeType = 'image/jpeg';
export const extension = 'jpg';
export const defaultOptions: EncodeOptions = {
quality: 75,
baseline: false,
arithmetic: false,
progressive: true,
optimize_coding: true,
smoothing: 0,
color_space: MozJpegColorSpace.YCbCr,
quant_table: 3,
trellis_multipass: false,
trellis_opt_zero: false,
trellis_opt_table: false,
trellis_loops: 1,
auto_subsample: true,
chroma_subsample: 2,
separate_chroma_quality: false,
chroma_quality: 75,
};

13
src/features/encoders/mozjpeg/shared/missing-types.d.ts vendored Normal file
View File

@@ -0,0 +1,13 @@
/**
* Copyright 2020 Google Inc. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/// <reference path="../../../../../missing-types.d.ts" />

7
src/features/encoders/mozjpeg/shared/tsconfig.json Normal file
View File

@@ -0,0 +1,7 @@
{
"extends": "../../../../../generic-tsconfig.json",
"compilerOptions": {
"lib": ["webworker", "esnext"]
},
"references": [{ "path": "../../../" }]
}

6
src/features/encoders/mozjpeg/worker/mozjpegEncode.ts
View File

@@ -10,10 +10,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import mozjpeg_enc, {
MozJPEGModule,
EncodeOptions,
} from 'codecs/mozjpeg_enc/mozjpeg_enc';
import mozjpeg_enc, { MozJPEGModule } from 'codecs/mozjpeg_enc/mozjpeg_enc';
import { EncodeOptions } from '../shared/meta';
import wasmUrl from 'url:codecs/mozjpeg_enc/mozjpeg_enc.wasm';
import { initEmscriptenModule } from 'features/util';

2
src/features/encoders/mozjpeg/worker/tsconfig.json
View File

@@ -3,5 +3,5 @@
"compilerOptions": {
"lib": ["webworker", "esnext"]
},
"references": [{ "path": "../../../" }]
"references": [{ "path": "../../../" }, { "path": "../shared" }]
}