Implements .env file support (#5531)

arcanis · web-flow · commit c0cb6a503a98 · 2023-06-26T14:37:19.000+02:00
**What's the problem this PR addresses?** A common need is to provide environment values into the environment via `.env` files. There's been a couple of issues and attempts at implementation already, which all were decently upvoted. I myself could have used it once or twice 😄 Props to @jj811208 for his implementation in #4835 - I wanted to make the configuration a little more generic (allowing to have multiple environment files, and to possibly disable it altogether), but it was a appreciated start. Fixes #4718 Closes #4835 (Supercedes it) **How did you fix it?** A new setting, `injectEnvironmentFiles`, lets you define files that Yarn will load and inject into all scripts. It only affects subprocesses - Yarn itself still uses `process.env` for its checks, so you can't for example set `YARN_*` values and expect them to be applied to the current process (use the yarnrc file for that instead). The `injectEnvironmentFiles` setting has a few properties: - It defaults to `.env` - Nothing will be injected if it's set to an empty array or null - The paths inside may be suffixed by `?` - in that case, Yarn won't throw if the file doesn't exist The idea with this last property is to allow for simple user configuration (imagine, with the example below, that the project also has a gitignore with `.env.*`): ``` injectEnvironmentFiles: - .env - .env.${USER}? ``` **Checklist**    - [x] I have read the [Contributing Guide](https://yarnpkg.com/advanced/contributing).   - [x] I have set the packages that need to be released for my changes to be effective.   - [x] I will check that all automated PR checks pass before the PR gets reviewed.
diff --git a/.pnp.cjs b/.pnp.cjs
diff --git a/.yarn/cache/dotenv-npm-16.3.1-e6d380a398-dbb778237e.zip b/.yarn/cache/dotenv-npm-16.3.1-e6d380a398-dbb778237e.zip
diff --git a/.yarn/versions/8ccfe176.yml b/.yarn/versions/8ccfe176.yml
@@ -0,0 +1,39 @@
+releases:
+  "@yarnpkg/cli": major
+  "@yarnpkg/core": major
+  "@yarnpkg/fslib": major
+  "@yarnpkg/plugin-essentials": major
+  "@yarnpkg/plugin-npm-cli": major
+  "@yarnpkg/plugin-workspace-tools": major
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-nm"
+  - "@yarnpkg/plugin-npm"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-pnpm"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - vscode-zipfs
+  - "@yarnpkg/builder"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/extensions"
+  - "@yarnpkg/libzip"
+  - "@yarnpkg/nm"
+  - "@yarnpkg/pnp"
+  - "@yarnpkg/pnpify"
+  - "@yarnpkg/sdks"
+  - "@yarnpkg/shell"
diff --git a/packages/acceptance-tests/pkg-tests-specs/sources/features/dotEnvFiles.test.ts b/packages/acceptance-tests/pkg-tests-specs/sources/features/dotEnvFiles.test.ts
@@ -0,0 +1,113 @@
+import {Filename, ppath, xfs} from '@yarnpkg/fslib';
+
+describe(`DotEnv files`, () => {
+  it(`should automatically inject a .env file in the environment`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await run(`install`);
+
+    await xfs.writeFilePromise(ppath.join(path, `.env`), [
+      `INJECTED_FROM_ENV_FILE=hello\n`,
+    ].join(``));
+
+    await expect(run(`exec`, `env`)).resolves.toMatchObject({
+      stdout: expect.stringMatching(/^INJECTED_FROM_ENV_FILE=hello$/m),
+    });
+  }));
+
+  it(`should allow .env variables to be interpolated`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await run(`install`);
+
+    await xfs.writeFilePromise(ppath.join(path, `.env`), [
+      `INJECTED_FROM_ENV_FILE=\${FOO}\n`,
+    ].join(``));
+
+    await expect(run(`exec`, `env`, {env: {FOO: `foo`}})).resolves.toMatchObject({
+      stdout: expect.stringMatching(/^INJECTED_FROM_ENV_FILE=foo$/m),
+    });
+  }));
+
+  it(`should allow .env variables to be used in the next ones`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await run(`install`);
+
+    await xfs.writeFilePromise(ppath.join(path, `.env`), [
+      `INJECTED_FROM_ENV_FILE_1=hello\n`,
+      `INJECTED_FROM_ENV_FILE_2=\${INJECTED_FROM_ENV_FILE_1} world\n`,
+    ].join(``));
+
+    await expect(run(`exec`, `env`, {env: {FOO: `foo`}})).resolves.toMatchObject({
+      stdout: expect.stringMatching(/^INJECTED_FROM_ENV_FILE_2=hello world$/m),
+    });
+  }));
+
+  it(`shouldn't read the .env if the injectEnvironmentFiles setting is defined`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await xfs.writeJsonPromise(ppath.join(path, Filename.rc), {
+      injectEnvironmentFiles: [],
+    });
+
+    await xfs.writeFilePromise(ppath.join(path, `.my-env`), [
+      `INJECTED_FROM_ENV_FILE=hello\n`,
+    ].join(``));
+
+    await run(`install`);
+
+    await expect(run(`exec`, `env`)).resolves.toMatchObject({
+      stdout: expect.not.stringMatching(/^INJECTED_FROM_ENV_FILE=/m),
+    });
+  }));
+
+  it(`should allow multiple environment files to be defined`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await xfs.writeJsonPromise(ppath.join(path, Filename.rc), {
+      injectEnvironmentFiles: [`.my-env`, `.my-other-env`],
+    });
+
+    await xfs.writeFilePromise(ppath.join(path, `.my-env`), [
+      `INJECTED_FROM_ENV_FILE_1=hello\n`,
+    ].join(``));
+
+    await xfs.writeFilePromise(ppath.join(path, `.my-other-env`), [
+      `INJECTED_FROM_ENV_FILE_2=world\n`,
+    ].join(``));
+
+    await run(`install`);
+
+    const {stdout} = await run(`exec`, `env`);
+
+    expect(stdout).toMatch(/^INJECTED_FROM_ENV_FILE_1=hello$/m);
+    expect(stdout).toMatch(/^INJECTED_FROM_ENV_FILE_2=world$/m);
+  }));
+
+  it(`should let the last environment file override the first`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await xfs.writeJsonPromise(ppath.join(path, Filename.rc), {
+      injectEnvironmentFiles: [`.my-env`, `.my-other-env`],
+    });
+
+    await xfs.writeFilePromise(ppath.join(path, `.my-env`), [
+      `INJECTED_FROM_ENV_FILE=hello\n`,
+    ].join(``));
+
+    await xfs.writeFilePromise(ppath.join(path, `.my-other-env`), [
+      `INJECTED_FROM_ENV_FILE=world\n`,
+    ].join(``));
+
+    await run(`install`);
+
+    await expect(run(`exec`, `env`)).resolves.toMatchObject({
+      stdout: expect.stringMatching(/^INJECTED_FROM_ENV_FILE=world$/m),
+    });
+  }));
+
+  it(`should throw an error if the settings reference a non-existing file`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await xfs.writeJsonPromise(ppath.join(path, Filename.rc), {
+      injectEnvironmentFiles: [`.my-env`],
+    });
+
+    await expect(run(`install`)).rejects.toThrow();
+  }));
+
+  it(`shouldn't throw an error if the settings reference a non-existing file with a ?-suffixed path`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await xfs.writeJsonPromise(ppath.join(path, Filename.rc), {
+      injectEnvironmentFiles: [`.my-env?`],
+    });
+
+    await run(`install`);
+  }));
+});
diff --git a/packages/plugin-essentials/sources/commands/set/version.ts b/packages/plugin-essentials/sources/commands/set/version.ts
@@ -192,7 +192,7 @@ export async function setVersion(configuration: Configuration, bundleVersion: st
 
       const {stdout} = await execUtils.execvp(process.execPath, [npath.fromPortablePath(temporaryPath), `--version`], {
         cwd: tmpDir,
-        env: {...process.env, YARN_IGNORE_PATH: `1`},
+        env: {...configuration.env, YARN_IGNORE_PATH: `1`},
       });
 
       bundleVersion = stdout.trim();
diff --git a/packages/plugin-npm-cli/sources/commands/npm/login.ts b/packages/plugin-npm-cli/sources/commands/npm/login.ts
@@ -140,10 +140,10 @@ async function getCredentials({configuration, registry, report, stdin, stdout}:
 
   report.reportSeparator();
 
-  if (process.env.YARN_IS_TEST_ENV) {
+  if (configuration.env.YARN_IS_TEST_ENV) {
     return {
-      name: process.env.YARN_INJECT_NPM_USER || ``,
-      password: process.env.YARN_INJECT_NPM_PASSWORD || ``,
+      name: configuration.env.YARN_INJECT_NPM_USER || ``,
+      password: configuration.env.YARN_INJECT_NPM_PASSWORD || ``,
     };
   }
 
diff --git a/packages/plugin-workspace-tools/sources/commands/foreach.ts b/packages/plugin-workspace-tools/sources/commands/foreach.ts
@@ -178,7 +178,7 @@ export default class WorkspacesForeachCommand extends BaseCommand {
 
       // Prevents infinite loop in the case of configuring a script as such:
       // "lint": "yarn workspaces foreach --all lint"
-      if (scriptName === process.env.npm_lifecycle_event && workspace.cwd === cwdWorkspace!.cwd)
+      if (scriptName === configuration.env.npm_lifecycle_event && workspace.cwd === cwdWorkspace!.cwd)
         continue;
 
       if (this.include.length > 0 && !micromatch.isMatch(structUtils.stringifyIdent(workspace.locator), this.include) && !micromatch.isMatch(workspace.relativeCwd, this.include))
diff --git a/packages/yarnpkg-core/package.json b/packages/yarnpkg-core/package.json
@@ -23,6 +23,7 @@
     "clipanion": "^3.2.1",
     "cross-spawn": "7.0.3",
     "diff": "^5.1.0",
+    "dotenv": "^16.3.1",
     "globby": "^11.0.1",
     "got": "^11.7.0",
     "lodash": "^4.17.15",
diff --git a/packages/yarnpkg-core/sources/Configuration.ts b/packages/yarnpkg-core/sources/Configuration.ts
@@ -3,6 +3,7 @@ import {parseSyml, stringifySyml}
 import camelcase                                                                                                 from 'camelcase';
 import {isCI, isPR, GITHUB_ACTIONS}                                                                              from 'ci-info';
 import {UsageError}                                                                                              from 'clipanion';
+import {parse as parseDotEnv}                                                                                    from 'dotenv';
 import pLimit, {Limit}                                                                                           from 'p-limit';
 import {PassThrough, Writable}                                                                                   from 'stream';
 
@@ -529,6 +530,14 @@ export const coreDefinitions: {[coreSettingName: string]: SettingsDefinition} =
     default: `throw`,
   },
 
+  // Miscellaneous settings
+  injectEnvironmentFiles: {
+    description: `List of all the environment files that Yarn should inject inside the process when it starts`,
+    type: SettingsType.ABSOLUTE_PATH,
+    default: [`.env?`],
+    isArray: true,
+  },
+
   // Package patching - to fix incorrect definitions
   packageExtensions: {
     description: `Map of package corrections to apply on the dependency tree`,
@@ -640,6 +649,9 @@ export interface ConfigurationValueMap {
   enableImmutableCache: boolean;
   checksumBehavior: string;
 
+  // Miscellaneous settings
+  injectEnvironmentFiles: Array<PortablePath>;
+
   // Package patching - to fix incorrect definitions
   packageExtensions: Map<string, miscUtils.ToMapValue<{
     dependencies?: Map<string, string>;
@@ -841,7 +853,9 @@ function getDefaultValue(configuration: Configuration, definition: SettingsDefin
         return null;
 
       if (configuration.projectCwd === null) {
-        if (ppath.isAbsolute(definition.default)) {
+        if (Array.isArray(definition.default)) {
+          return definition.default.map((entry: string) => ppath.normalize(entry as PortablePath));
+        } else if (ppath.isAbsolute(definition.default)) {
           return ppath.normalize(definition.default);
         } else if (definition.isNullable) {
           return null;
@@ -966,6 +980,7 @@ export class Configuration {
 
   public invalid: Map<string, string> = new Map();
 
+  public env: Record<string, string | undefined> = {};
   public packageExtensions: Map<IdentHash, Array<[string, Array<PackageExtension>]>> = new Map();
 
   public limits: Map<string, Limit> = new Map();
@@ -1052,8 +1067,8 @@ export class Configuration {
 
     const allCoreFieldKeys = new Set(Object.keys(coreDefinitions));
 
-    const pickPrimaryCoreFields = ({ignoreCwd, yarnPath, ignorePath, lockfileFilename}: CoreFields) => ({ignoreCwd, yarnPath, ignorePath, lockfileFilename});
-    const pickSecondaryCoreFields = ({ignoreCwd, yarnPath, ignorePath, lockfileFilename, ...rest}: CoreFields) => {
+    const pickPrimaryCoreFields = ({ignoreCwd, yarnPath, ignorePath, lockfileFilename, injectEnvironmentFiles}: CoreFields) => ({ignoreCwd, yarnPath, ignorePath, lockfileFilename, injectEnvironmentFiles});
+    const pickSecondaryCoreFields = ({ignoreCwd, yarnPath, ignorePath, lockfileFilename, injectEnvironmentFiles, ...rest}: CoreFields) => {
       const secondaryCoreFields: CoreFields = {};
       for (const [key, value] of Object.entries(rest))
         if (allCoreFieldKeys.has(key))
@@ -1120,6 +1135,22 @@ export class Configuration {
     configuration.startingCwd = startingCwd;
     configuration.projectCwd = projectCwd;
 
+    const env = Object.assign(Object.create(null), process.env);
+    configuration.env = env;
+
+    // load the environment files
+    const environmentFiles = await Promise.all(configuration.get(`injectEnvironmentFiles`).map(async p => {
+      const content = p.endsWith(`?`)
+        ? await xfs.readFilePromise(p.slice(0, -1) as PortablePath, `utf8`).catch(() => ``)
+        : await xfs.readFilePromise(p as PortablePath, `utf8`);
+
+      return parseDotEnv(content);
+    }));
+
+    for (const environmentEntries of environmentFiles)
+      for (const [key, value] of Object.entries(environmentEntries))
+        configuration.env[key] = miscUtils.replaceEnvVariables(value, {env});
+
     // load all fields of the core definitions
     configuration.importSettings(pickSecondaryCoreFields(coreDefinitions));
     configuration.useWithSource(`<environment>`, pickSecondaryCoreFields(environmentSettings), startingCwd, {strict});
diff --git a/packages/yarnpkg-core/sources/scriptUtils.ts b/packages/yarnpkg-core/sources/scriptUtils.ts
@@ -107,9 +107,11 @@ export async function detectPackageManager(location: PortablePath): Promise<Pack
   return null;
 }
 
-export async function makeScriptEnv({project, locator, binFolder, ignoreCorepack, lifecycleScript}: {project?: Project, locator?: Locator, binFolder: PortablePath, ignoreCorepack?: boolean, lifecycleScript?: string}) {
+export async function makeScriptEnv({project, locator, binFolder, ignoreCorepack, lifecycleScript, baseEnv = project?.configuration.env ?? process.env}: {project?: Project, locator?: Locator, binFolder: PortablePath, ignoreCorepack?: boolean, lifecycleScript?: string, baseEnv?: Record<string, string | undefined>}) {
   const scriptEnv: {[key: string]: string} = {};
-  for (const [key, value] of Object.entries(process.env))
+
+  // Ensure that the PATH environment variable is properly capitalized (Windows)
+  for (const [key, value] of Object.entries(baseEnv))
     if (typeof value !== `undefined`)
       scriptEnv[key.toLowerCase() !== `path` ? key : `PATH`] = value;
 
diff --git a/packages/yarnpkg-fslib/sources/path.ts b/packages/yarnpkg-fslib/sources/path.ts
@@ -32,6 +32,7 @@ export const Filename = {
   pnpData: `.pnp.data.json` as Filename,
   pnpEsmLoader: `.pnp.loader.mjs` as Filename,
   rc: `.yarnrc.yml` as Filename,
+  env: `.env` as Filename,
 };
 
 export type TolerateLiterals<T> = {
diff --git a/yarn.lock b/yarn.lock
@@ -7081,6 +7081,7 @@ __metadata:
     comment-json: "npm:^2.2.0"
     cross-spawn: "npm:7.0.3"
     diff: "npm:^5.1.0"
+    dotenv: "npm:^16.3.1"
     esbuild: "npm:esbuild-wasm@^0.15.15"
     globby: "npm:^11.0.1"
     got: "npm:^11.7.0"
@@ -12537,6 +12538,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"dotenv@npm:^16.3.1":
+  version: 16.3.1
+  resolution: "dotenv@npm:16.3.1"
+  checksum: dbb778237ef8750e9e3cd1473d3c8eaa9cc3600e33a75c0e36415d0fa0848197f56c3800f77924c70e7828f0b03896818cd52f785b07b9ad4d88dba73fbba83f
+  languageName: node
+  linkType: hard
+
 "dotenv@npm:^8.2.0":
   version: 8.2.0
   resolution: "dotenv@npm:8.2.0"
