From 50fe5f1a4b657161bd336c38ec8025ad6fd747da Mon Sep 17 00:00:00 2001 From: merceyz Date: Fri, 6 Jan 2023 17:49:41 +0100 Subject: [PATCH 1/2] chore: use `os.availableParallelism` when available --- .yarn/versions/cad178bf.yml | 34 +++++++++++++++++++ .../sources/commands/foreach.ts | 13 +++++-- packages/yarnpkg-core/sources/WorkerPool.ts | 13 +++++-- 3 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 .yarn/versions/cad178bf.yml diff --git a/.yarn/versions/cad178bf.yml b/.yarn/versions/cad178bf.yml new file mode 100644 index 000000000000..80389a446795 --- /dev/null +++ b/.yarn/versions/cad178bf.yml @@ -0,0 +1,34 @@ +releases: + "@yarnpkg/cli": patch + "@yarnpkg/core": patch + "@yarnpkg/plugin-workspace-tools": patch + +declined: + - "@yarnpkg/plugin-compat" + - "@yarnpkg/plugin-constraints" + - "@yarnpkg/plugin-dlx" + - "@yarnpkg/plugin-essentials" + - "@yarnpkg/plugin-exec" + - "@yarnpkg/plugin-file" + - "@yarnpkg/plugin-git" + - "@yarnpkg/plugin-github" + - "@yarnpkg/plugin-http" + - "@yarnpkg/plugin-init" + - "@yarnpkg/plugin-interactive-tools" + - "@yarnpkg/plugin-link" + - "@yarnpkg/plugin-nm" + - "@yarnpkg/plugin-npm" + - "@yarnpkg/plugin-npm-cli" + - "@yarnpkg/plugin-pack" + - "@yarnpkg/plugin-patch" + - "@yarnpkg/plugin-pnp" + - "@yarnpkg/plugin-pnpm" + - "@yarnpkg/plugin-stage" + - "@yarnpkg/plugin-typescript" + - "@yarnpkg/plugin-version" + - "@yarnpkg/builder" + - "@yarnpkg/doctor" + - "@yarnpkg/extensions" + - "@yarnpkg/nm" + - "@yarnpkg/pnpify" + - "@yarnpkg/sdks" diff --git a/packages/plugin-workspace-tools/sources/commands/foreach.ts b/packages/plugin-workspace-tools/sources/commands/foreach.ts index 93fcb31b4ef9..1cda28e38065 100644 --- a/packages/plugin-workspace-tools/sources/commands/foreach.ts +++ b/packages/plugin-workspace-tools/sources/commands/foreach.ts @@ -5,12 +5,21 @@ import {formatUtils, miscUtils, structUtils} from '@yarn import {gitUtils} from '@yarnpkg/plugin-git'; import {Command, Option, Usage, UsageError} from 'clipanion'; import micromatch from 'micromatch'; -import {cpus} from 'os'; +import os from 'os'; import pLimit from 'p-limit'; import {Writable} from 'stream'; import {WriteStream} from 'tty'; import * as t from 'typanion'; +function availableParallelism() { + // TODO: Use os.availableParallelism directly when dropping support for Node.js < 19.4.0 + if (`availableParallelism` in os) + // @ts-expect-error - No types yet + return os.availableParallelism(); + + return Math.max(1, os.cpus().length); +} + // eslint-disable-next-line arca/no-default-export export default class WorkspacesForeachCommand extends BaseCommand { static paths = [ @@ -200,7 +209,7 @@ export default class WorkspacesForeachCommand extends BaseCommand { const concurrency = this.parallel ? (this.jobs === `unlimited` ? Infinity - : Number(this.jobs) || Math.max(1, cpus().length / 2)) + : Number(this.jobs) || Math.ceil(availableParallelism() / 2)) : 1; // No need to parallelize if we were explicitly asked for one job diff --git a/packages/yarnpkg-core/sources/WorkerPool.ts b/packages/yarnpkg-core/sources/WorkerPool.ts index 8b8a07a84360..c6300d9d2e8b 100644 --- a/packages/yarnpkg-core/sources/WorkerPool.ts +++ b/packages/yarnpkg-core/sources/WorkerPool.ts @@ -1,4 +1,4 @@ -import {cpus} from 'os'; +import os from 'os'; import PLimit from 'p-limit'; import {Worker} from 'worker_threads'; @@ -8,9 +8,18 @@ type PoolWorker = Worker & { [kTaskInfo]: null | { resolve: (value: TOut) => void, reject: (reason?: any) => void }; }; +function availableParallelism(): number { + // TODO: Use os.availableParallelism directly when dropping support for Node.js < 19.4.0 + if (`availableParallelism` in os) + // @ts-expect-error - No types yet + return os.availableParallelism(); + + return Math.max(1, os.cpus().length); +} + export class WorkerPool { private workers: Array> = []; - private limit = PLimit(Math.max(1, cpus().length)); + private limit = PLimit(availableParallelism()); private cleanupInterval: ReturnType; constructor(private source: string) { From cd49ca6c3df7918525c1a6cf06ad73d9f94b31b5 Mon Sep 17 00:00:00 2001 From: merceyz Date: Sun, 8 Jan 2023 00:24:31 +0100 Subject: [PATCH 2/2] refactor: move to nodeUtils --- .../sources/commands/foreach.ts | 14 ++------------ packages/yarnpkg-core/sources/WorkerPool.ts | 18 +++++------------- packages/yarnpkg-core/sources/nodeUtils.ts | 10 ++++++++++ 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/packages/plugin-workspace-tools/sources/commands/foreach.ts b/packages/plugin-workspace-tools/sources/commands/foreach.ts index 1cda28e38065..182fb6d7fd18 100644 --- a/packages/plugin-workspace-tools/sources/commands/foreach.ts +++ b/packages/plugin-workspace-tools/sources/commands/foreach.ts @@ -1,25 +1,15 @@ import {BaseCommand, WorkspaceRequiredError} from '@yarnpkg/cli'; import {Configuration, LocatorHash, Project, scriptUtils, Workspace} from '@yarnpkg/core'; import {DescriptorHash, MessageName, Report, StreamReport} from '@yarnpkg/core'; -import {formatUtils, miscUtils, structUtils} from '@yarnpkg/core'; +import {formatUtils, miscUtils, structUtils, nodeUtils} from '@yarnpkg/core'; import {gitUtils} from '@yarnpkg/plugin-git'; import {Command, Option, Usage, UsageError} from 'clipanion'; import micromatch from 'micromatch'; -import os from 'os'; import pLimit from 'p-limit'; import {Writable} from 'stream'; import {WriteStream} from 'tty'; import * as t from 'typanion'; -function availableParallelism() { - // TODO: Use os.availableParallelism directly when dropping support for Node.js < 19.4.0 - if (`availableParallelism` in os) - // @ts-expect-error - No types yet - return os.availableParallelism(); - - return Math.max(1, os.cpus().length); -} - // eslint-disable-next-line arca/no-default-export export default class WorkspacesForeachCommand extends BaseCommand { static paths = [ @@ -209,7 +199,7 @@ export default class WorkspacesForeachCommand extends BaseCommand { const concurrency = this.parallel ? (this.jobs === `unlimited` ? Infinity - : Number(this.jobs) || Math.ceil(availableParallelism() / 2)) + : Number(this.jobs) || Math.ceil(nodeUtils.availableParallelism() / 2)) : 1; // No need to parallelize if we were explicitly asked for one job diff --git a/packages/yarnpkg-core/sources/WorkerPool.ts b/packages/yarnpkg-core/sources/WorkerPool.ts index c6300d9d2e8b..61a0bd772ba2 100644 --- a/packages/yarnpkg-core/sources/WorkerPool.ts +++ b/packages/yarnpkg-core/sources/WorkerPool.ts @@ -1,6 +1,7 @@ -import os from 'os'; -import PLimit from 'p-limit'; -import {Worker} from 'worker_threads'; +import PLimit from 'p-limit'; +import {Worker} from 'worker_threads'; + +import * as nodeUtils from './nodeUtils'; const kTaskInfo = Symbol(`kTaskInfo`); @@ -8,18 +9,9 @@ type PoolWorker = Worker & { [kTaskInfo]: null | { resolve: (value: TOut) => void, reject: (reason?: any) => void }; }; -function availableParallelism(): number { - // TODO: Use os.availableParallelism directly when dropping support for Node.js < 19.4.0 - if (`availableParallelism` in os) - // @ts-expect-error - No types yet - return os.availableParallelism(); - - return Math.max(1, os.cpus().length); -} - export class WorkerPool { private workers: Array> = []; - private limit = PLimit(availableParallelism()); + private limit = PLimit(nodeUtils.availableParallelism()); private cleanupInterval: ReturnType; constructor(private source: string) { diff --git a/packages/yarnpkg-core/sources/nodeUtils.ts b/packages/yarnpkg-core/sources/nodeUtils.ts index cf2657dd3fbf..343f14d19b65 100644 --- a/packages/yarnpkg-core/sources/nodeUtils.ts +++ b/packages/yarnpkg-core/sources/nodeUtils.ts @@ -1,5 +1,6 @@ import {ppath} from '@yarnpkg/fslib'; import Module from 'module'; +import os from 'os'; import * as execUtils from './execUtils'; import * as miscUtils from './miscUtils'; @@ -136,3 +137,12 @@ export function getCaller() { return parseStackLine(line); } + +export function availableParallelism() { + // TODO: Use os.availableParallelism directly when dropping support for Node.js < 19.4.0 + if (`availableParallelism` in os) + // @ts-expect-error - No types yet + return os.availableParallelism(); + + return Math.max(1, os.cpus().length); +}