Generating feature metadata

.gitignore

@@ -5,7 +5,8 @@ node_modules
 build
 *.o
 
-# Auto-generated by lib/image-worker-plugin.js
+# Auto-generated by lib/feature-plugin.js
 src/features-worker/index.ts
 src/features-worker/tsconfig.json
 src/client/lazy-app/worker-bridge/meta.ts
+src/client/lazy-app/feature-meta/index.ts

lib/feature-plugin.js

@@ -0,0 +1,213 @@
+/**
+ * Copyright 2020 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { promisify } from 'util';
+import * as path from 'path';
+import { promises as fsp } from 'fs';
+
+import glob from 'glob';
+
+const globP = promisify(glob);
+const autoGenComment =
+  '// This file is autogenerated by lib/feature-plugin.js\n';
+
+export default function () {
+  let previousWorkerImports;
+  let previousEncoderMetas;
+
+  /**
+   * Generates the worker file & tsconfig for all features
+   *
+   * @param {string[]} workerImports
+   */
+  async function generateWorkerFile(workerImports) {
+    const workerBasePath = path.join(process.cwd(), 'src', 'features-worker');
+
+    const featuresWorkerTsNames = workerImports.map((tsImport) => [
+      path.relative(workerBasePath, tsImport),
+      path.basename(tsImport),
+    ]);
+
+    const workerFile = [
+      autoGenComment,
+      `import { expose } from 'comlink';`,
+      `import { timed } from './util';`,
+      featuresWorkerTsNames.map(
+        ([path, name]) => `import ${name} from './${path}';`,
+      ),
+      `const exports = {`,
+      featuresWorkerTsNames.map(([_, name]) => [
+        `  ${name}(`,
+        `    ...args: Parameters<typeof ${name}>`,
+        `  ): ReturnType<typeof ${name}> {`,
+        `    return timed('${name}', () => ${name}(...args));`,
+        `  },`,
+      ]),
+      `};`,
+      `export type ProcessorWorkerApi = typeof exports;`,
+      `expose(exports, self);`,
+    ]
+      .flat(Infinity)
+      .join('\n');
+
+    const workerTsConfig = {
+      extends: '../../generic-tsconfig.json',
+      compilerOptions: {
+        lib: ['webworker', 'esnext'],
+      },
+      references: featuresWorkerTsNames.map(([tsImport]) => ({
+        path: path.dirname(tsImport),
+      })),
+    };
+
+    await Promise.all([
+      fsp.writeFile(
+        path.join(workerBasePath, 'tsconfig.json'),
+        JSON.stringify(workerTsConfig, null, '  '),
+      ),
+      fsp.writeFile(path.join(workerBasePath, 'index.ts'), workerFile),
+    ]);
+  }
+
+  /**
+   * Generates the client JS to call worker methods.
+   *
+   * @param {string[]} workerImports
+   */
+  async function generateWorkerBridge(workerImports) {
+    const workerBridgeBasePath = path.join(
+      process.cwd(),
+      'src',
+      'client',
+      'lazy-app',
+      'worker-bridge',
+    );
+
+    const featuresWorkerBridgeTsNames = workerImports.map((tsImport) => [
+      path.relative(workerBridgeBasePath, tsImport),
+      path.basename(tsImport),
+    ]);
+
+    const bridgeMeta = [
+      autoGenComment,
+      featuresWorkerBridgeTsNames.map(
+        ([path, name]) => `import type ${name} from '${path}';`,
+      ),
+      `export const methodNames = ${JSON.stringify(
+        featuresWorkerBridgeTsNames.map(([_, name]) => name),
+        null,
+        '  ',
+      )} as const;`,
+      `export interface BridgeMethods {`,
+      featuresWorkerBridgeTsNames.map(([_, name]) => [
+        `  ${name}(`,
+        `    signal: AbortSignal,`,
+        `    ...args: Parameters<typeof ${name}>`,
+        `  ): Promise<ReturnType<typeof ${name}>>;`,
+      ]),
+      `}`,
+    ]
+      .flat(Infinity)
+      .join('\n');
+
+    await fsp.writeFile(path.join(workerBridgeBasePath, 'meta.ts'), bridgeMeta);
+  }
+
+  async function generateWorkerFiles() {
+    const workerImports = (
+      await globP('src/features/*/**/worker/*.ts', {
+        absolute: true,
+      })
+    )
+      .filter((tsFile) => !tsFile.endsWith('.d.ts'))
+      .map((tsFile) => tsFile.slice(0, -'.ts'.length));
+
+    const joinedWorkerImports = workerImports.join();
+
+    // Avoid regenerating if nothing's changed.
+    // This also prevents an infinite look in the watcher.
+    if (joinedWorkerImports === previousWorkerImports) return;
+
+    previousWorkerImports = joinedWorkerImports;
+    await Promise.all([
+      generateWorkerFile(workerImports),
+      generateWorkerBridge(workerImports),
+    ]);
+  }
+
+  async function generateFeatureMeta() {
+    const encoderMetas = (
+      await globP('src/features/encoders/*/shared/meta.ts', {
+        absolute: true,
+      })
+    )
+      .filter((tsFile) => !tsFile.endsWith('.d.ts'))
+      .map((tsFile) => tsFile.slice(0, -'.ts'.length));
+
+    const featureMetaBasePath = path.join(
+      process.cwd(),
+      'src',
+      'client',
+      'lazy-app',
+      'feature-meta',
+    );
+
+    const joinedEncoderMetas = encoderMetas.join();
+
+    // Avoid regenerating if nothing's changed.
+    // This also prevents an infinite look in the watcher.
+    if (joinedEncoderMetas === previousEncoderMetas) return;
+    previousEncoderMetas = joinedEncoderMetas;
+
+    const encoderMetaTsNames = encoderMetas.map((tsImport) => [
+      path.relative(featureMetaBasePath, tsImport),
+      path.basename(tsImport.slice(0, -'/shared/meta'.length)),
+    ]);
+
+    const featureMeta = [
+      autoGenComment,
+      encoderMetaTsNames.map(
+        ([path, name]) => `import * as ${name}EncoderMeta from '${path}';`,
+      ),
+      `export type EncoderState =`,
+      encoderMetaTsNames.map(
+        ([_, name]) =>
+          `  | { type: "${name}", options: ${name}EncoderMeta.EncodeOptions }`,
+      ),
+      `;`,
+      `export type EncoderOptions =`,
+      encoderMetaTsNames.map(
+        ([_, name]) => `  | ${name}EncoderMeta.EncodeOptions`,
+      ),
+      `;`,
+      `export const encoderMap = {`,
+      encoderMetaTsNames.map(([_, name]) => `  ${name}: ${name}EncoderMeta,`),
+      `};`,
+      `export type EncoderType = keyof typeof encoderMap`,
+      `export const encoders = [...Object.values(encoderMap)];`,
+    ]
+      .flat(Infinity)
+      .join('\n');
+
+    await fsp.writeFile(
+      path.join(featureMetaBasePath, 'index.ts'),
+      featureMeta,
+    );
+  }
+
+  return {
+    name: 'feature-plugin',
+    async buildStart() {
+      await Promise.all([generateWorkerFiles(), generateFeatureMeta()]);
+    },
+  };
+}

lib/image-worker-plugin.js

@@ -1,129 +0,0 @@
-/**
- * Copyright 2020 Google Inc. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *     http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import { promisify } from 'util';
-import * as path from 'path';
-import { promises as fsp } from 'fs';
-
-import glob from 'glob';
-
-const globP = promisify(glob);
-
-export default function () {
-  let previousWorkerContent;
-
-  return {
-    name: 'image-worker-plugin',
-    async buildStart() {
-      const featuresWorkerBase = path.join(
-        process.cwd(),
-        'src',
-        'features-worker',
-      );
-      const featuresWorkerBridgeBase = path.join(
-        process.cwd(),
-        'src',
-        'client',
-        'lazy-app',
-        'worker-bridge',
-      );
-
-      const tsImports = (
-        await globP('src/features/*/**/worker/*.ts', {
-          absolute: true,
-        })
-      )
-        .filter((tsFile) => !tsFile.endsWith('.d.ts'))
-        .map((tsFile) => tsFile.slice(0, -'.ts'.length));
-
-      const featuresWorkerTsNames = tsImports.map((tsImport) => [
-        path.relative(featuresWorkerBase, tsImport),
-        path.basename(tsImport),
-      ]);
-
-      const featuresWorkerBridgeTsNames = tsImports.map((tsImport) => [
-        path.relative(featuresWorkerBridgeBase, tsImport),
-        path.basename(tsImport),
-      ]);
-
-      const workerFile = [
-        `// This file is autogenerated by lib/image-worker-plugin.js`,
-        `import { expose } from 'comlink';`,
-        `import { timed } from './util';`,
-        featuresWorkerTsNames.map(
-          ([path, name]) => `import ${name} from './${path}';`,
-        ),
-        `const exports = {`,
-        featuresWorkerTsNames.map(([_, name]) => [
-          `  ${name}(`,
-          `    ...args: Parameters<typeof ${name}>`,
-          `  ): ReturnType<typeof ${name}> {`,
-          `    return timed('${name}', () => ${name}(...args));`,
-          `  },`,
-        ]),
-        `};`,
-        `export type ProcessorWorkerApi = typeof exports;`,
-        `expose(exports, self);`,
-      ]
-        .flat(Infinity)
-        .join('\n');
-
-      // If nothing's changed, avoid touching the file to avoid infinite rebuilding in watch mode
-      if (previousWorkerContent === workerFile) return;
-      previousWorkerContent = workerFile;
-
-      const workerTsConfig = {
-        extends: '../../generic-tsconfig.json',
-        compilerOptions: {
-          lib: ['webworker', 'esnext'],
-        },
-        references: featuresWorkerTsNames.map(([tsImport]) => ({
-          path: path.dirname(tsImport),
-        })),
-      };
-
-      const bridgeMeta = [
-        `// This file is autogenerated by lib/image-worker-plugin.js`,
-        featuresWorkerBridgeTsNames.map(
-          ([path, name]) => `import type ${name} from '${path}';`,
-        ),
-        `export const methodNames = ${JSON.stringify(
-          featuresWorkerBridgeTsNames.map(([_, name]) => name),
-          null,
-          '  ',
-        )} as const;`,
-        `export interface BridgeMethods {`,
-        featuresWorkerBridgeTsNames.map(([_, name]) => [
-          `  ${name}(`,
-          `    signal: AbortSignal,`,
-          `    ...args: Parameters<typeof ${name}>`,
-          `  ): Promise<ReturnType<typeof ${name}>>;`,
-        ]),
-        `}`,
-      ]
-        .flat(Infinity)
-        .join('\n');
-
-      await Promise.all([
-        fsp.writeFile(
-          path.join(featuresWorkerBase, 'tsconfig.json'),
-          JSON.stringify(workerTsConfig, null, '  '),
-        ),
-        fsp.writeFile(path.join(featuresWorkerBase, 'index.ts'), workerFile),
-        fsp.writeFile(
-          path.join(featuresWorkerBridgeBase, 'meta.ts'),
-          bridgeMeta,
-        ),
-      ]);
-    },
-  };
-}

rollup.config.js

@@ -27,7 +27,7 @@ import urlPlugin from './lib/url-plugin';
 import resolveDirsPlugin from './lib/resolve-dirs-plugin';
 import runScript from './lib/run-script';
 import emitFiles from './lib/emit-files-plugin';
-import imageWorkerPlugin from './lib/image-worker-plugin';
+import featurePlugin from './lib/feature-plugin';
 import initialCssPlugin from './lib/initial-css-plugin';
 import serviceWorkerPlugin from './lib/sw-plugin';
 import dataURLPlugin from './lib/data-url-plugin';
@@ -122,7 +122,7 @@ export default async function ({ watch }) {
       ...commonPlugins(),
       emitFiles({ include: '**/*', root: path.join(__dirname, 'src', 'copy') }),
       nodeExternalPlugin(),
-      imageWorkerPlugin(),
+      featurePlugin(),
       replace({ __PRERENDER__: true, __PRODUCTION__: isProduction }),
       initialCssPlugin(),
       runScript(dir + '/static-build/index.js'),

src/client/lazy-app/worker-bridge/index.ts

@@ -5,7 +5,7 @@ import type { ProcessorWorkerApi } from '../../../features-worker';
 import { abortable } from '../util';
 
 /** How long the worker should be idle before terminating. */
-const workerTimeout = 10000;
+const workerTimeout = 10_000;
 
 interface WorkerBridge extends BridgeMethods {}
 

src/features/encoders/avif/shared/meta.ts

@@ -0,0 +1,29 @@
+/**
+ * Copyright 2020 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type { EncodeOptions } from 'codecs/avif/enc/avif_enc';
+
+export { EncodeOptions };
+
+export const label = 'AVIF';
+export const mimeType = 'image/avif';
+export const extension = 'avif';
+export const defaultOptions: EncodeOptions = {
+  minQuantizer: 33,
+  maxQuantizer: 63,
+  minQuantizerAlpha: 33,
+  maxQuantizerAlpha: 63,
+  tileColsLog2: 0,
+  tileRowsLog2: 0,
+  speed: 8,
+  subsample: 1,
+};

src/features/encoders/avif/shared/missing-types.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Copyright 2020 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/// <reference path="../../../../../missing-types.d.ts" />

src/features/encoders/avif/shared/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../../../../generic-tsconfig.json",
+  "compilerOptions": {
+    "lib": ["webworker", "esnext"]
+  },
+  "references": [{ "path": "../../../" }]
+}

src/features/encoders/avif/worker/avifEncode.ts

@@ -10,10 +10,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import avifEncoder, {
+import avifEncoder, { AVIFModule } from 'codecs/avif/enc/avif_enc';
-  AVIFModule,
+import type { EncodeOptions } from '../shared/meta';
-  EncodeOptions,
-} from 'codecs/avif/enc/avif_enc';
 import wasmUrl from 'url:codecs/avif/enc/avif_enc.wasm';
 import { initEmscriptenModule } from 'features/util';
 

src/features/encoders/avif/worker/tsconfig.json

@@ -3,5 +3,5 @@
   "compilerOptions": {
     "lib": ["webworker", "esnext"]
   },
-  "references": [{ "path": "../../../" }]
+  "references": [{ "path": "../../../" }, { "path": "../shared" }]
 }

src/features/encoders/mozjpeg/shared/meta.ts

@@ -0,0 +1,39 @@
+/**
+ * Copyright 2020 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  EncodeOptions,
+  MozJpegColorSpace,
+} from 'codecs/mozjpeg_enc/mozjpeg_enc';
+export { EncodeOptions, MozJpegColorSpace };
+
+export const label = 'MozJPEG';
+export const mimeType = 'image/jpeg';
+export const extension = 'jpg';
+export const defaultOptions: EncodeOptions = {
+  quality: 75,
+  baseline: false,
+  arithmetic: false,
+  progressive: true,
+  optimize_coding: true,
+  smoothing: 0,
+  color_space: MozJpegColorSpace.YCbCr,
+  quant_table: 3,
+  trellis_multipass: false,
+  trellis_opt_zero: false,
+  trellis_opt_table: false,
+  trellis_loops: 1,
+  auto_subsample: true,
+  chroma_subsample: 2,
+  separate_chroma_quality: false,
+  chroma_quality: 75,
+};

src/features/encoders/mozjpeg/shared/missing-types.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Copyright 2020 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/// <reference path="../../../../../missing-types.d.ts" />

src/features/encoders/mozjpeg/shared/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../../../../generic-tsconfig.json",
+  "compilerOptions": {
+    "lib": ["webworker", "esnext"]
+  },
+  "references": [{ "path": "../../../" }]
+}

src/features/encoders/mozjpeg/worker/mozjpegEncode.ts

@@ -10,10 +10,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import mozjpeg_enc, {
+import mozjpeg_enc, { MozJPEGModule } from 'codecs/mozjpeg_enc/mozjpeg_enc';
-  MozJPEGModule,
+import { EncodeOptions } from '../shared/meta';
-  EncodeOptions,
-} from 'codecs/mozjpeg_enc/mozjpeg_enc';
 import wasmUrl from 'url:codecs/mozjpeg_enc/mozjpeg_enc.wasm';
 import { initEmscriptenModule } from 'features/util';
 

src/features/encoders/mozjpeg/worker/tsconfig.json

@@ -3,5 +3,5 @@
   "compilerOptions": {
     "lib": ["webworker", "esnext"]
   },
-  "references": [{ "path": "../../../" }]
+  "references": [{ "path": "../../../" }, { "path": "../shared" }]
 }