From 733b470f1f3bf7d1ab8b86316f5b05e53603e692 Mon Sep 17 00:00:00 2001 From: DetachHead Date: Sat, 26 Dec 2020 22:56:23 +1000 Subject: [PATCH 1/5] allow passing entire directories as input --- cli/src/index.js | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/cli/src/index.js b/cli/src/index.js index d7c97eda..ba1107ee 100644 --- a/cli/src/index.js +++ b/cli/src/index.js @@ -183,31 +183,37 @@ function progressTracker(results) { return tracker; } -async function checkInputFilesValid(files) { +async function getInputFiles(paths) { const validFiles = []; - for (const file of files) { - try { - await fsp.stat(file); - } catch (err) { - if (err.code === 'ENOENT') { - console.warn( - `Warning: Input file does not exist: ${resolvePath(file)}`, - ); - continue; - } else { - throw err; + for (const path of paths) { + //allow paths ending in / of \ to get all files in that directory + const files = path.endsWith('/') || path.endsWith('\\') + ? await fs.readdir(paths) + : [path]; + for (const file of files) { + try { + await fsp.stat(file); + } catch (err) { + if (err.code === 'ENOENT') { + console.warn( + `Warning: Input file does not exist: ${resolvePath(file)}`, + ); + continue; + } else { + throw err; + } } - } - validFiles.push(file); + validFiles.push(file); + } } return validFiles; } async function processFiles(files) { - files = await checkInputFilesValid(files); + files = await getInputFiles(files); const parallelism = cpus().length; From 1e64e52298be93f46bee98291c350a2c3f6f1ff4 Mon Sep 17 00:00:00 2001 From: DetachHead Date: Sat, 26 Dec 2020 23:26:27 +1000 Subject: [PATCH 2/5] fix reading directories --- cli/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/index.js b/cli/src/index.js index ba1107ee..c58d99c2 100644 --- a/cli/src/index.js +++ b/cli/src/index.js @@ -189,7 +189,7 @@ async function getInputFiles(paths) { for (const path of paths) { //allow paths ending in / of \ to get all files in that directory const files = path.endsWith('/') || path.endsWith('\\') - ? await fs.readdir(paths) + ? (await fsp.readdir(path)).map(file => join(path, file)) : [path]; for (const file of files) { try { From 10d648c28d159fc45b3a7685ce552d670c2f0c9b Mon Sep 17 00:00:00 2001 From: DetachHead Date: Sun, 27 Dec 2020 15:59:56 +1000 Subject: [PATCH 3/5] check if path is directory using lstat instead of requiring trailing slash --- cli/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/index.js b/cli/src/index.js index c58d99c2..6098949a 100644 --- a/cli/src/index.js +++ b/cli/src/index.js @@ -188,7 +188,7 @@ async function getInputFiles(paths) { for (const path of paths) { //allow paths ending in / of \ to get all files in that directory - const files = path.endsWith('/') || path.endsWith('\\') + const files = (await fsp.lstat(path)).isDirectory() ? (await fsp.readdir(path)).map(file => join(path, file)) : [path]; for (const file of files) { From 935256985231aafce71929d388d57afe0e9e22ba Mon Sep 17 00:00:00 2001 From: DetachHead Date: Sun, 27 Dec 2020 16:01:02 +1000 Subject: [PATCH 4/5] remove outdated comment --- cli/src/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/src/index.js b/cli/src/index.js index 6098949a..390dc43d 100644 --- a/cli/src/index.js +++ b/cli/src/index.js @@ -187,7 +187,6 @@ async function getInputFiles(paths) { const validFiles = []; for (const path of paths) { - //allow paths ending in / of \ to get all files in that directory const files = (await fsp.lstat(path)).isDirectory() ? (await fsp.readdir(path)).map(file => join(path, file)) : [path]; From e7e205c326dfcc8c14760f42179013f91aa3ccb9 Mon Sep 17 00:00:00 2001 From: Surma Date: Tue, 5 Jan 2021 14:26:26 +0000 Subject: [PATCH 5/5] Simplify WorkerPool joining (closes #925) --- cli/src/worker_pool.js | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/cli/src/worker_pool.js b/cli/src/worker_pool.js index 90830f5d..57ebbfa7 100644 --- a/cli/src/worker_pool.js +++ b/cli/src/worker_pool.js @@ -24,7 +24,7 @@ function jobPromise(worker, msg) { export default class WorkerPool { constructor(numWorkers, workerFile) { - this.closing = false; + this.numWorkers = numWorkers; this.jobQueue = new TransformStream(); this.workerQueue = new TransformStream(); @@ -42,7 +42,6 @@ export default class WorkerPool { while (true) { const { value, done } = await reader.read(); if (done) { - this.workerQueue.writable.close(); await this._terminateAll(); return; } @@ -50,12 +49,6 @@ export default class WorkerPool { const worker = await this._nextWorker(); jobPromise(worker, msg).then((result) => { resolve(result); - // If we are in the process of closing, `workerQueue` is - // already closed and we can’t requeue the worker. - if (this.closing) { - worker.terminate(); - return; - } const writer = this.workerQueue.writable.getWriter(); writer.write(worker); writer.releaseLock(); @@ -71,18 +64,15 @@ export default class WorkerPool { } async _terminateAll() { - while (true) { + for (let n = 0; n < this.numWorkers; n++) { const worker = await this._nextWorker(); - if (!worker) { - return; - } worker.terminate(); } + this.workerQueue.writable.close(); } async join() { - this.closing = true; - this.jobQueue.writable.close(); + this.jobQueue.writable.getWriter().close(); await this.done; }