Cache correct assets (#887)

* Cache correct assets

* Update lib/entry-data-plugin.js

Co-authored-by: Surma <surma@surma.dev>

* Actually commit the fix this time

Co-authored-by: Surma <surma@surma.dev>

lib/client-bundle-plugin.js

@@ -137,9 +137,10 @@ export default function (inputOptions, outputOptions, resolveFileUrl) {
       const dependencies = getDependencies(clientOutput, clientEntry);
 
       if (property.startsWith(allSrcPlaceholder)) {
-        return JSON.stringify(
+        const depCodes = dependencies.map(
-          [clientEntry.code, ...dependencies.map((d) => d.code)].join(';'),
+          (name) => clientOutput.find((item) => item.fileName === name).code,
         );
+        return JSON.stringify([clientEntry.code, ...depCodes].join(';'));
       }
 
       return (

lib/entry-data-plugin.js

@@ -0,0 +1,87 @@
+/**
+ * Copyright 2020 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { getDependencies } from './client-bundle-plugin';
+import * as path from 'path';
+
+const prefix = 'entry-data:';
+const mainNamePlaceholder = 'ENTRY_DATA_PLUGIN_MAIN_NAME';
+const dependenciesPlaceholder = 'ENTRY_DATA_PLUGIN_DEPS';
+const placeholderRe = /(ENTRY_DATA_PLUGIN_(?:MAIN_NAME|DEPS))(\d+)/g;
+const filenamePrefix = 'static/';
+
+export default function entryDataPlugin() {
+  /** @type {string} */
+  let exportCounter;
+  /** @type {Map<number, string>} */
+  let counterToIdMap;
+
+  return {
+    name: 'entry-data-plugin',
+    buildStart() {
+      exportCounter = 0;
+      counterToIdMap = new Map();
+    },
+    async resolveId(id, importer) {
+      if (!id.startsWith(prefix)) return;
+      const realId = id.slice(prefix.length);
+      const resolveResult = await this.resolve(realId, importer);
+
+      if (!resolveResult) throw Error(`Cannot find ${realId}`);
+      // Add an additional .js to the end so it ends up with .js at the end in the _virtual folder.
+      return prefix + resolveResult.id + '.js';
+    },
+    load(id) {
+      if (!id.startsWith(prefix)) return;
+      const realId = id.slice(prefix.length, -'.js'.length);
+      exportCounter++;
+
+      counterToIdMap.set(exportCounter, path.normalize(realId));
+
+      return [
+        `export const main = ${mainNamePlaceholder + exportCounter};`,
+        `export const deps = ${dependenciesPlaceholder + exportCounter};`,
+      ].join('\n');
+    },
+    generateBundle(_, bundle) {
+      const chunks = Object.values(bundle).filter(
+        (item) => item.type === 'chunk',
+      );
+      for (const chunk of chunks) {
+        chunk.code = chunk.code.replace(
+          placeholderRe,
+          (_, placeholder, numStr) => {
+            const id = counterToIdMap.get(Number(numStr));
+            const chunk = chunks.find(
+              (chunk) =>
+                chunk.facadeModuleId &&
+                path.normalize(chunk.facadeModuleId) === id,
+            );
+            if (!chunk) throw Error(`Cannot find ${id}`);
+
+            if (placeholder === mainNamePlaceholder) {
+              return JSON.stringify(
+                chunk.fileName.slice(filenamePrefix.length),
+              );
+            }
+
+            return JSON.stringify(
+              getDependencies(chunks, chunk).map((item) =>
+                item.slice(filenamePrefix.length),
+              ),
+            );
+          },
+        );
+      }
+    },
+  };
+}

missing-types.d.ts

@@ -12,6 +12,11 @@
  */
 /// <reference path="./emscripten-types.d.ts" />
 
+declare module 'entry-data:*' {
+  export const main: string;
+  export const deps: string[];
+}
+
 declare module 'url:*' {
   const value: string;
   export default value;

rollup.config.js

@@ -31,6 +31,7 @@ import featurePlugin from './lib/feature-plugin';
 import initialCssPlugin from './lib/initial-css-plugin';
 import serviceWorkerPlugin from './lib/sw-plugin';
 import dataURLPlugin from './lib/data-url-plugin';
+import entryDataPlugin from './lib/entry-data-plugin';
 
 function resolveFileUrl({ fileName }) {
   return JSON.stringify(fileName.replace(/^static\//, '/'));
@@ -116,6 +117,7 @@ export default async function ({ watch }) {
             commonjs(),
             resolve(),
             replace({ __PRERENDER__: false, __PRODUCTION__: isProduction }),
+            entryDataPlugin(),
             isProduction ? terser({ module: true }) : {},
           ],
           preserveEntrySignatures: false,

src/features/decoders/avif/worker/avifDecode.ts

@@ -10,7 +10,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import avifDecoder, { AVIFModule } from 'codecs/avif/dec/avif_dec';
+import type { AVIFModule } from 'codecs/avif/dec/avif_dec';
 import wasmUrl from 'url:codecs/avif/dec/avif_dec.wasm';
 import { initEmscriptenModule, blobToArrayBuffer } from 'features/worker-utils';
 
@@ -18,7 +18,8 @@ let emscriptenModule: Promise<AVIFModule>;
 
 export default async function decode(blob: Blob): Promise<ImageData> {
   if (!emscriptenModule) {
-    emscriptenModule = initEmscriptenModule(avifDecoder, wasmUrl);
+    const decoder = await import('codecs/avif/dec/avif_dec');
+    emscriptenModule = initEmscriptenModule(decoder.default, wasmUrl);
   }
 
   const [module, data] = await Promise.all([

src/features/decoders/webp/worker/webpDecode.ts

@@ -10,7 +10,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import webpDecoder, { WebPModule } from 'codecs/webp/dec/webp_dec';
+import type { WebPModule } from 'codecs/webp/dec/webp_dec';
 import wasmUrl from 'url:codecs/webp/dec/webp_dec.wasm';
 import { initEmscriptenModule, blobToArrayBuffer } from 'features/worker-utils';
 
@@ -18,7 +18,8 @@ let emscriptenModule: Promise<WebPModule>;
 
 export default async function decode(blob: Blob): Promise<ImageData> {
   if (!emscriptenModule) {
-    emscriptenModule = initEmscriptenModule(webpDecoder, wasmUrl);
+    const decoder = await import('codecs/webp/dec/webp_dec');
+    emscriptenModule = initEmscriptenModule(decoder.default, wasmUrl);
   }
 
   const [module, data] = await Promise.all([

src/sw/index.ts

@@ -19,11 +19,11 @@ self.addEventListener('install', (event) => {
   event.waitUntil(
     (async function () {
       const promises = [];
-      promises.push(cacheBasics(versionedCache, ASSETS));
+      promises.push(cacheBasics(versionedCache));
 
       // If the user has already interacted with the app, update the codecs too.
       if (await get('user-interacted')) {
-        promises.push(cacheAdditionalProcessors(versionedCache, ASSETS));
+        promises.push(cacheAdditionalProcessors(versionedCache));
       }
 
       await Promise.all(promises);
@@ -53,6 +53,11 @@ self.addEventListener('fetch', (event) => {
   // Don't care about other-origin URLs
   if (url.origin !== location.origin) return;
 
+  if (url.pathname === '/editor') {
+    event.respondWith(Response.redirect('/'));
+    return;
+  }
+
   if (
     url.pathname === '/' &&
     url.searchParams.has('share-target') &&
@@ -77,7 +82,7 @@ self.addEventListener('fetch', (event) => {
 self.addEventListener('message', (event) => {
   switch (event.data) {
     case 'cache-all':
-      event.waitUntil(cacheAdditionalProcessors(versionedCache, ASSETS));
+      event.waitUntil(cacheAdditionalProcessors(versionedCache));
       break;
     case 'skip-waiting':
       self.skipWaiting();

src/sw/to-cache.ts

@@ -0,0 +1,213 @@
+import { threads, simd } from 'wasm-feature-detect';
+import webpDataUrl from 'data-url:./tiny.webp';
+import avifDataUrl from 'data-url:./tiny.avif';
+
+// Give TypeScript the correct global.
+declare var self: ServiceWorkerGlobalScope;
+
+function subtractSets<T extends any>(set1: Set<T>, set2: Set<T>): Set<T> {
+  const result = new Set(set1);
+  for (const item of set2) result.delete(item);
+  return result;
+}
+
+// Initial app stuff
+import * as initialApp from 'entry-data:client/initial-app';
+import * as compress from 'entry-data:client/lazy-app/Compress';
+import * as swBridge from 'entry-data:client/lazy-app/sw-bridge';
+import * as blobAnim from 'entry-data:shared/prerendered-app/Intro/blob-anim';
+import logo from 'url:shared/prerendered-app/Intro/imgs/logo.svg';
+import githubLogo from 'url:shared/prerendered-app/Intro/imgs/github-logo.svg';
+import largePhotoIcon from 'url:shared/prerendered-app/Intro/imgs/demos/icon-demo-large-photo.jpg';
+import artworkIcon from 'url:shared/prerendered-app/Intro/imgs/demos/icon-demo-artwork.jpg';
+import deviceScreenIcon from 'url:shared/prerendered-app/Intro/imgs/demos/icon-demo-device-screen.jpg';
+import logoIcon from 'url:shared/prerendered-app/Intro/imgs/demos/icon-demo-logo.png';
+import logoWithText from 'url:shared/prerendered-app/Intro/imgs/logo-with-text.svg';
+
+let initalJs = new Set([
+  compress.main,
+  ...compress.deps,
+  swBridge.main,
+  ...swBridge.deps,
+  blobAnim.main,
+  ...blobAnim.deps,
+]);
+// But initial app and any deps have already been inlined, so we don't need them:
+initalJs = subtractSets(
+  initalJs,
+  new Set([initialApp.main, ...initialApp.deps]),
+);
+
+export const initial = [
+  '/',
+  ...initalJs,
+  logo,
+  githubLogo,
+  largePhotoIcon,
+  artworkIcon,
+  deviceScreenIcon,
+  logoIcon,
+  logoWithText,
+];
+
+// The processors and codecs
+// Simple stuff everyone gets:
+import * as featuresWorker from 'entry-data:../features-worker';
+import rotateWasm from 'url:codecs/rotate/rotate.wasm';
+import quantWasm from 'url:codecs/imagequant/imagequant.wasm';
+import resizeWasm from 'url:codecs/resize/pkg/squoosh_resize_bg.wasm';
+import hqxWasm from 'url:codecs/hqx/pkg/squooshhqx_bg.wasm';
+import mozjpegWasm from 'url:codecs/mozjpeg/enc/mozjpeg_enc.wasm';
+
+// Decoders (some are feature detected)
+import * as avifDec from 'entry-data:codecs/avif/dec/avif_dec';
+import avifDecWasm from 'url:codecs/avif/dec/avif_dec.wasm';
+import jxlDecWasm from 'url:codecs/jxl/dec/jxl_dec.wasm';
+import * as webpDec from 'entry-data:codecs/webp/dec/webp_dec';
+import webpDecWasm from 'url:codecs/webp/dec/webp_dec.wasm';
+import wp2DecWasm from 'url:codecs/wp2/dec/wp2_dec.wasm';
+
+// AVIF
+import * as avifEncMtWorker from 'entry-data:codecs/avif/enc/avif_enc_mt.worker.js';
+import * as avifEncMt from 'entry-data:codecs/avif/enc/avif_enc_mt';
+import avifEncMtWasm from 'url:codecs/avif/enc/avif_enc_mt.wasm';
+import avifEncWasm from 'url:codecs/avif/enc/avif_enc.wasm';
+import * as avifEnc from 'entry-data:codecs/avif/enc/avif_enc.js';
+
+// JXL
+import * as jxlEncMtSimdWorker from 'entry-data:codecs/jxl/enc/jxl_enc_mt_simd.worker.js';
+import * as jxlEncMtSimd from 'entry-data:codecs/jxl/enc/jxl_enc_mt_simd';
+import jxlEncMtSimdWasm from 'url:codecs/jxl/enc/jxl_enc_mt_simd.wasm';
+import * as jxlEncMtWorker from 'entry-data:codecs/jxl/enc/jxl_enc_mt.worker.js';
+import * as jxlEncMt from 'entry-data:codecs/jxl/enc/jxl_enc_mt';
+import jxlEncMtWasm from 'url:codecs/jxl/enc/jxl_enc_mt.wasm';
+import jxlEncWasm from 'url:codecs/jxl/enc/jxl_enc.wasm';
+import * as jxlEnc from 'entry-data:codecs/jxl/enc/jxl_enc';
+
+// OXI
+import * as oxiMtWorker from 'entry-data:features/encoders/oxiPNG/worker/sub-worker';
+import oxiMtWasm from 'url:codecs/oxipng/pkg-parallel/squoosh_oxipng_bg.wasm';
+import oxiWasm from 'url:codecs/oxipng/pkg/squoosh_oxipng_bg.wasm';
+
+// WebP
+import * as webpEncSimd from 'entry-data:codecs/webp/enc/webp_enc_simd';
+import webpEncSimdWasm from 'url:codecs/webp/enc/webp_enc_simd.wasm';
+import * as webpEnc from 'entry-data:codecs/webp/enc/webp_enc';
+import webpEncWasm from 'url:codecs/webp/enc/webp_enc.wasm';
+
+// WP2
+import * as wp2EncMtSimdWorker from 'entry-data:codecs/wp2/enc/wp2_enc_mt_simd.worker.js';
+import * as wp2EncMtSimd from 'entry-data:codecs/wp2/enc/wp2_enc_mt_simd';
+import wp2EncMtSimdWasm from 'url:codecs/wp2/enc/wp2_enc_mt_simd.wasm';
+import * as wp2EncMtWorker from 'entry-data:codecs/wp2/enc/wp2_enc_mt.worker.js';
+import * as wp2EncMt from 'entry-data:codecs/wp2/enc/wp2_enc_mt';
+import wp2EncMtWasm from 'url:codecs/wp2/enc/wp2_enc_mt.wasm';
+import * as wp2Enc from 'entry-data:codecs/wp2/enc/wp2_enc';
+import wp2EncWasm from 'url:codecs/wp2/enc/wp2_enc.wasm';
+
+export const theRest = (async () => {
+  const [
+    supportsThreads,
+    supportsSimd,
+    supportsWebP,
+    supportsAvif,
+  ] = await Promise.all([
+    threads(),
+    simd(),
+    ...[webpDataUrl, avifDataUrl].map(async (dataUrl) => {
+      if (!self.createImageBitmap) return false;
+      const response = await fetch(dataUrl);
+      const blob = await response.blob();
+      return createImageBitmap(blob).then(
+        () => true,
+        () => false,
+      );
+    }),
+  ]);
+
+  const items = [
+    featuresWorker.main,
+    ...featuresWorker.deps,
+    rotateWasm,
+    quantWasm,
+    resizeWasm,
+    hqxWasm,
+    mozjpegWasm,
+    jxlDecWasm,
+    wp2DecWasm,
+  ];
+
+  if (!supportsAvif) items.push(avifDec.main, ...avifDec.deps, avifDecWasm);
+  if (!supportsWebP) items.push(webpDec.main, ...webpDec.deps, webpDecWasm);
+
+  // AVIF
+  if (supportsThreads) {
+    items.push(
+      avifEncMtWorker.main,
+      ...avifEncMtWorker.deps,
+      avifEncMt.main,
+      ...avifEncMt.deps,
+      avifEncMtWasm,
+    );
+  } else {
+    items.push(avifEnc.main, ...avifEnc.deps, avifEncWasm);
+  }
+
+  // JXL
+  if (supportsThreads && supportsSimd) {
+    items.push(
+      jxlEncMtSimdWorker.main,
+      ...jxlEncMtSimdWorker.deps,
+      jxlEncMtSimd.main,
+      ...jxlEncMtSimd.deps,
+      jxlEncMtSimdWasm,
+    );
+  } else if (supportsThreads) {
+    items.push(
+      jxlEncMtWorker.main,
+      ...jxlEncMtWorker.deps,
+      jxlEncMt.main,
+      ...jxlEncMt.deps,
+      jxlEncMtWasm,
+    );
+  } else {
+    items.push(jxlEnc.main, ...jxlEnc.deps, jxlEncWasm);
+  }
+
+  // OXI
+  if (supportsThreads) {
+    items.push(oxiMtWorker.main, ...oxiMtWorker.deps, oxiMtWasm);
+  } else {
+    items.push(oxiWasm);
+  }
+
+  // WebP
+  if (supportsSimd) {
+    items.push(webpEncSimd.main, ...webpEncSimd.deps, webpEncSimdWasm);
+  } else {
+    items.push(webpEnc.main, ...webpEnc.deps, webpEncWasm);
+  }
+
+  // WP2
+  if (supportsThreads && supportsSimd) {
+    items.push(
+      wp2EncMtSimdWorker.main,
+      ...wp2EncMtSimdWorker.deps,
+      wp2EncMtSimd.main,
+      ...wp2EncMtSimd.deps,
+      wp2EncMtSimdWasm,
+    );
+  } else if (supportsThreads) {
+    items.push(
+      wp2EncMtWorker.main,
+      ...wp2EncMtWorker.deps,
+      wp2EncMt.main,
+      ...wp2EncMt.deps,
+      wp2EncMtWasm,
+    );
+  } else {
+    items.push(wp2Enc.main, ...wp2Enc.deps, wp2EncWasm);
+  }
+
+  return [...new Set(items)];
+})();

src/sw/util.ts

@@ -1,5 +1,4 @@
-import webpDataUrl from 'data-url:./tiny.webp';
+import { initial, theRest } from './to-cache';
-import avifDataUrl from 'data-url:./tiny.avif';
 
 // Give TypeScript the correct global.
 declare var self: ServiceWorkerGlobalScope;
@@ -85,76 +84,14 @@ export function cleanupCache(
   );
 }
 
-function getAssetsWithPrefix(assets: string[], prefixes: string[]) {
+export async function cacheBasics(cacheName: string) {
-  return assets.filter((asset) =>
+  const cache = await caches.open(cacheName);
-    prefixes.some((prefix) => asset.startsWith(prefix)),
+  return cache.addAll(initial);
-  );
 }
 
-export async function cacheBasics(cacheName: string, buildAssets: string[]) {
+export async function cacheAdditionalProcessors(cacheName: string) {
-  const toCache = ['/'];
-
-  const prefixesToCache = [
-    // TODO: this is likely incomplete
-    // Main app JS & CSS:
-    'c/initial-app-',
-    // Service worker handler:
-    'c/sw-bridge-',
-    // Little icons for the demo images on the homescreen:
-    'c/icon-demo-',
-    // Site logo:
-    'c/logo-',
-  ];
-
-  const prefixMatches = getAssetsWithPrefix(buildAssets, prefixesToCache);
-
-  toCache.push(...prefixMatches);
-
   const cache = await caches.open(cacheName);
-  await cache.addAll(toCache);
+  return cache.addAll(await theRest);
-}
-
-export async function cacheAdditionalProcessors(
-  cacheName: string,
-  buildAssets: string[],
-) {
-  let toCache = [];
-
-  const prefixesToCache = [
-    // TODO: these will need to change
-    // Worker which handles image processing:
-    'processor-worker.',
-    // processor-worker imports:
-    'process-',
-  ];
-
-  const prefixMatches = getAssetsWithPrefix(buildAssets, prefixesToCache);
-  const wasm = buildAssets.filter((asset) => asset.endsWith('.wasm'));
-
-  toCache.push(...prefixMatches, ...wasm);
-
-  const [supportsWebP, supportsAvif] = await Promise.all(
-    [webpDataUrl, avifDataUrl].map(async (dataUrl) => {
-      if (!self.createImageBitmap) return false;
-      const response = await fetch(dataUrl);
-      const blob = await response.blob();
-      return createImageBitmap(blob).then(
-        () => true,
-        () => false,
-      );
-    }),
-  );
-
-  // TODO: this is likely wrong
-  // No point caching decoders the browser already supports:
-  toCache = toCache.filter(
-    (asset) =>
-      (supportsWebP ? !/webp[\-_]dec/.test(asset) : true) &&
-      (supportsAvif ? !/avif[\-_]dec/.test(asset) : true),
-  );
-
-  const cache = await caches.open(cacheName);
-  await cache.addAll(toCache);
 }
 
 const nextMessageResolveMap = new Map<string, (() => void)[]>();