Implements .env file support (#5531)

**What's the problem this PR addresses?**

A common need is to provide environment values into the environment via
`.env` files. There's been a couple of issues and attempts at
implementation already, which all were decently upvoted. I myself could
have used it once or twice 😄

Props to @jj811208 for his implementation in #4835 - I wanted to make
the configuration a little more generic (allowing to have multiple
environment files, and to possibly disable it altogether), but it was a
appreciated start.

Fixes #4718 
Closes #4835 (Supercedes it)

**How did you fix it?**

A new setting, `injectEnvironmentFiles`, lets you define files that Yarn
will load and inject into all scripts. It only affects subprocesses -
Yarn itself still uses `process.env` for its checks, so you can't for
example set `YARN_*` values and expect them to be applied to the current
process (use the yarnrc file for that instead).

The `injectEnvironmentFiles` setting has a few properties:

- It defaults to `.env`
- Nothing will be injected if it's set to an empty array or null
- The paths inside may be suffixed by `?` - in that case, Yarn won't
throw if the file doesn't exist

The idea with this last property is to allow for simple user
configuration (imagine, with the example below, that the project also
has a gitignore with `.env.*`):

```
injectEnvironmentFiles:
  - .env
  - .env.${USER}?
```

**Checklist**
<!--- Don't worry if you miss something, chores are automatically
tested. -->
<!--- This checklist exists to help you remember doing the chores when
you submit a PR. -->
<!--- Put an `x` in all the boxes that apply. -->
- [x] I have read the [Contributing
Guide](https://yarnpkg.com/advanced/contributing).

<!-- See
https://yarnpkg.com/advanced/contributing#preparing-your-pr-to-be-released
for more details. -->
<!-- Check with `yarn version check` and fix with `yarn version check
-i` -->
- [x] I have set the packages that need to be released for my changes to
be effective.

<!-- The "Testing chores" workflow validates that your PR follows our
guidelines. -->
<!-- If it doesn't pass, click on it to see details as to what your PR
might be missing. -->
- [x] I will check that all automated PR checks pass before the PR gets
reviewed.

.yarn/versions/8ccfe176.yml

@@ -0,0 +1,39 @@
+releases:
+  "@yarnpkg/cli": major
+  "@yarnpkg/core": major
+  "@yarnpkg/fslib": major
+  "@yarnpkg/plugin-essentials": major
+  "@yarnpkg/plugin-npm-cli": major
+  "@yarnpkg/plugin-workspace-tools": major
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-nm"
+  - "@yarnpkg/plugin-npm"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-pnpm"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - vscode-zipfs
+  - "@yarnpkg/builder"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/extensions"
+  - "@yarnpkg/libzip"
+  - "@yarnpkg/nm"
+  - "@yarnpkg/pnp"
+  - "@yarnpkg/pnpify"
+  - "@yarnpkg/sdks"
+  - "@yarnpkg/shell"

packages/acceptance-tests/pkg-tests-specs/sources/features/dotEnvFiles.test.ts

@@ -0,0 +1,113 @@
+import {Filename, ppath, xfs} from '@yarnpkg/fslib';
+
+describe(`DotEnv files`, () => {
+  it(`should automatically inject a .env file in the environment`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await run(`install`);
+
+    await xfs.writeFilePromise(ppath.join(path, `.env`), [
+      `INJECTED_FROM_ENV_FILE=hello\n`,
+    ].join(``));
+
+    await expect(run(`exec`, `env`)).resolves.toMatchObject({
+      stdout: expect.stringMatching(/^INJECTED_FROM_ENV_FILE=hello$/m),
+    });
+  }));
+
+  it(`should allow .env variables to be interpolated`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await run(`install`);
+
+    await xfs.writeFilePromise(ppath.join(path, `.env`), [
+      `INJECTED_FROM_ENV_FILE=\${FOO}\n`,
+    ].join(``));
+
+    await expect(run(`exec`, `env`, {env: {FOO: `foo`}})).resolves.toMatchObject({
+      stdout: expect.stringMatching(/^INJECTED_FROM_ENV_FILE=foo$/m),
+    });
+  }));
+
+  it(`should allow .env variables to be used in the next ones`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await run(`install`);
+
+    await xfs.writeFilePromise(ppath.join(path, `.env`), [
+      `INJECTED_FROM_ENV_FILE_1=hello\n`,
+      `INJECTED_FROM_ENV_FILE_2=\${INJECTED_FROM_ENV_FILE_1} world\n`,
+    ].join(``));
+
+    await expect(run(`exec`, `env`, {env: {FOO: `foo`}})).resolves.toMatchObject({
+      stdout: expect.stringMatching(/^INJECTED_FROM_ENV_FILE_2=hello world$/m),
+    });
+  }));
+
+  it(`shouldn't read the .env if the injectEnvironmentFiles setting is defined`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await xfs.writeJsonPromise(ppath.join(path, Filename.rc), {
+      injectEnvironmentFiles: [],
+    });
+
+    await xfs.writeFilePromise(ppath.join(path, `.my-env`), [
+      `INJECTED_FROM_ENV_FILE=hello\n`,
+    ].join(``));
+
+    await run(`install`);
+
+    await expect(run(`exec`, `env`)).resolves.toMatchObject({
+      stdout: expect.not.stringMatching(/^INJECTED_FROM_ENV_FILE=/m),
+    });
+  }));
+
+  it(`should allow multiple environment files to be defined`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await xfs.writeJsonPromise(ppath.join(path, Filename.rc), {
+      injectEnvironmentFiles: [`.my-env`, `.my-other-env`],
+    });
+
+    await xfs.writeFilePromise(ppath.join(path, `.my-env`), [
+      `INJECTED_FROM_ENV_FILE_1=hello\n`,
+    ].join(``));
+
+    await xfs.writeFilePromise(ppath.join(path, `.my-other-env`), [
+      `INJECTED_FROM_ENV_FILE_2=world\n`,
+    ].join(``));
+
+    await run(`install`);
+
+    const {stdout} = await run(`exec`, `env`);
+
+    expect(stdout).toMatch(/^INJECTED_FROM_ENV_FILE_1=hello$/m);
+    expect(stdout).toMatch(/^INJECTED_FROM_ENV_FILE_2=world$/m);
+  }));
+
+  it(`should let the last environment file override the first`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await xfs.writeJsonPromise(ppath.join(path, Filename.rc), {
+      injectEnvironmentFiles: [`.my-env`, `.my-other-env`],
+    });
+
+    await xfs.writeFilePromise(ppath.join(path, `.my-env`), [
+      `INJECTED_FROM_ENV_FILE=hello\n`,
+    ].join(``));
+
+    await xfs.writeFilePromise(ppath.join(path, `.my-other-env`), [
+      `INJECTED_FROM_ENV_FILE=world\n`,
+    ].join(``));
+
+    await run(`install`);
+
+    await expect(run(`exec`, `env`)).resolves.toMatchObject({
+      stdout: expect.stringMatching(/^INJECTED_FROM_ENV_FILE=world$/m),
+    });
+  }));
+
+  it(`should throw an error if the settings reference a non-existing file`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await xfs.writeJsonPromise(ppath.join(path, Filename.rc), {
+      injectEnvironmentFiles: [`.my-env`],
+    });
+
+    await expect(run(`install`)).rejects.toThrow();
+  }));
+
+  it(`shouldn't throw an error if the settings reference a non-existing file with a ?-suffixed path`, makeTemporaryEnv({}, async ({path, run, source}) => {
+    await xfs.writeJsonPromise(ppath.join(path, Filename.rc), {
+      injectEnvironmentFiles: [`.my-env?`],
+    });
+
+    await run(`install`);
+  }));
+});

packages/plugin-essentials/sources/commands/set/version.ts

@@ -192,7 +192,7 @@ export async function setVersion(configuration: Configuration, bundleVersion: st
 
       const {stdout} = await execUtils.execvp(process.execPath, [npath.fromPortablePath(temporaryPath), `--version`], {
         cwd: tmpDir,
-        env: {...process.env, YARN_IGNORE_PATH: `1`},
+        env: {...configuration.env, YARN_IGNORE_PATH: `1`},
       });
 
       bundleVersion = stdout.trim();

packages/plugin-npm-cli/sources/commands/npm/login.ts

@@ -140,10 +140,10 @@ async function getCredentials({configuration, registry, report, stdin, stdout}:
 
   report.reportSeparator();
 
-  if (process.env.YARN_IS_TEST_ENV) {
+  if (configuration.env.YARN_IS_TEST_ENV) {
     return {
-      name: process.env.YARN_INJECT_NPM_USER || ``,
-      password: process.env.YARN_INJECT_NPM_PASSWORD || ``,
+      name: configuration.env.YARN_INJECT_NPM_USER || ``,
+      password: configuration.env.YARN_INJECT_NPM_PASSWORD || ``,
     };
   }
 

packages/plugin-workspace-tools/sources/commands/foreach.ts

@@ -178,7 +178,7 @@ export default class WorkspacesForeachCommand extends BaseCommand {
 
       // Prevents infinite loop in the case of configuring a script as such:
       // "lint": "yarn workspaces foreach --all lint"
-      if (scriptName === process.env.npm_lifecycle_event && workspace.cwd === cwdWorkspace!.cwd)
+      if (scriptName === configuration.env.npm_lifecycle_event && workspace.cwd === cwdWorkspace!.cwd)
         continue;
 
       if (this.include.length > 0 && !micromatch.isMatch(structUtils.stringifyIdent(workspace.locator), this.include) && !micromatch.isMatch(workspace.relativeCwd, this.include))

packages/yarnpkg-core/package.json

@@ -23,6 +23,7 @@
     "clipanion": "^3.2.1",
     "cross-spawn": "7.0.3",
     "diff": "^5.1.0",
+    "dotenv": "^16.3.1",
     "globby": "^11.0.1",
     "got": "^11.7.0",
     "lodash": "^4.17.15",

packages/yarnpkg-core/sources/Configuration.ts

@@ -3,6 +3,7 @@ import {parseSyml, stringifySyml}
 import camelcase                                                                                                 from 'camelcase';
 import {isCI, isPR, GITHUB_ACTIONS}                                                                              from 'ci-info';
 import {UsageError}                                                                                              from 'clipanion';
+import {parse as parseDotEnv}                                                                                    from 'dotenv';
 import pLimit, {Limit}                                                                                           from 'p-limit';
 import {PassThrough, Writable}                                                                                   from 'stream';
 
@@ -529,6 +530,14 @@ export const coreDefinitions: {[coreSettingName: string]: SettingsDefinition} =
     default: `throw`,
   },
 
+  // Miscellaneous settings
+  injectEnvironmentFiles: {
+    description: `List of all the environment files that Yarn should inject inside the process when it starts`,
+    type: SettingsType.ABSOLUTE_PATH,
+    default: [`.env?`],
+    isArray: true,
+  },
+
   // Package patching - to fix incorrect definitions
   packageExtensions: {
     description: `Map of package corrections to apply on the dependency tree`,
@@ -640,6 +649,9 @@ export interface ConfigurationValueMap {
   enableImmutableCache: boolean;
   checksumBehavior: string;
 
+  // Miscellaneous settings
+  injectEnvironmentFiles: Array<PortablePath>;
+
   // Package patching - to fix incorrect definitions
   packageExtensions: Map<string, miscUtils.ToMapValue<{
     dependencies?: Map<string, string>;
@@ -841,7 +853,9 @@ function getDefaultValue(configuration: Configuration, definition: SettingsDefin
         return null;
 
       if (configuration.projectCwd === null) {
-        if (ppath.isAbsolute(definition.default)) {
+        if (Array.isArray(definition.default)) {
+          return definition.default.map((entry: string) => ppath.normalize(entry as PortablePath));
+        } else if (ppath.isAbsolute(definition.default)) {
           return ppath.normalize(definition.default);
         } else if (definition.isNullable) {
           return null;
@@ -966,6 +980,7 @@ export class Configuration {
 
   public invalid: Map<string, string> = new Map();
 
+  public env: Record<string, string | undefined> = {};
   public packageExtensions: Map<IdentHash, Array<[string, Array<PackageExtension>]>> = new Map();
 
   public limits: Map<string, Limit> = new Map();
@@ -1052,8 +1067,8 @@ export class Configuration {
 
     const allCoreFieldKeys = new Set(Object.keys(coreDefinitions));
 
-    const pickPrimaryCoreFields = ({ignoreCwd, yarnPath, ignorePath, lockfileFilename}: CoreFields) => ({ignoreCwd, yarnPath, ignorePath, lockfileFilename});
-    const pickSecondaryCoreFields = ({ignoreCwd, yarnPath, ignorePath, lockfileFilename, ...rest}: CoreFields) => {
+    const pickPrimaryCoreFields = ({ignoreCwd, yarnPath, ignorePath, lockfileFilename, injectEnvironmentFiles}: CoreFields) => ({ignoreCwd, yarnPath, ignorePath, lockfileFilename, injectEnvironmentFiles});
+    const pickSecondaryCoreFields = ({ignoreCwd, yarnPath, ignorePath, lockfileFilename, injectEnvironmentFiles, ...rest}: CoreFields) => {
       const secondaryCoreFields: CoreFields = {};
       for (const [key, value] of Object.entries(rest))
         if (allCoreFieldKeys.has(key))
@@ -1120,6 +1135,22 @@ export class Configuration {
     configuration.startingCwd = startingCwd;
     configuration.projectCwd = projectCwd;
 
+    const env = Object.assign(Object.create(null), process.env);
+    configuration.env = env;
+
+    // load the environment files
+    const environmentFiles = await Promise.all(configuration.get(`injectEnvironmentFiles`).map(async p => {
+      const content = p.endsWith(`?`)
+        ? await xfs.readFilePromise(p.slice(0, -1) as PortablePath, `utf8`).catch(() => ``)
+        : await xfs.readFilePromise(p as PortablePath, `utf8`);
+
+      return parseDotEnv(content);
+    }));
+
+    for (const environmentEntries of environmentFiles)
+      for (const [key, value] of Object.entries(environmentEntries))
+        configuration.env[key] = miscUtils.replaceEnvVariables(value, {env});
+
     // load all fields of the core definitions
     configuration.importSettings(pickSecondaryCoreFields(coreDefinitions));
     configuration.useWithSource(`<environment>`, pickSecondaryCoreFields(environmentSettings), startingCwd, {strict});

packages/yarnpkg-core/sources/scriptUtils.ts

@@ -107,9 +107,11 @@ export async function detectPackageManager(location: PortablePath): Promise<Pack
   return null;
 }
 
-export async function makeScriptEnv({project, locator, binFolder, ignoreCorepack, lifecycleScript}: {project?: Project, locator?: Locator, binFolder: PortablePath, ignoreCorepack?: boolean, lifecycleScript?: string}) {
+export async function makeScriptEnv({project, locator, binFolder, ignoreCorepack, lifecycleScript, baseEnv = project?.configuration.env ?? process.env}: {project?: Project, locator?: Locator, binFolder: PortablePath, ignoreCorepack?: boolean, lifecycleScript?: string, baseEnv?: Record<string, string | undefined>}) {
   const scriptEnv: {[key: string]: string} = {};
-  for (const [key, value] of Object.entries(process.env))
+
+  // Ensure that the PATH environment variable is properly capitalized (Windows)
+  for (const [key, value] of Object.entries(baseEnv))
     if (typeof value !== `undefined`)
       scriptEnv[key.toLowerCase() !== `path` ? key : `PATH`] = value;
 

packages/yarnpkg-fslib/sources/path.ts

@@ -32,6 +32,7 @@ export const Filename = {
   pnpData: `.pnp.data.json` as Filename,
   pnpEsmLoader: `.pnp.loader.mjs` as Filename,
   rc: `.yarnrc.yml` as Filename,
+  env: `.env` as Filename,
 };
 
 export type TolerateLiterals<T> = {

yarn.lock

@@ -7081,6 +7081,7 @@ __metadata:
     comment-json: "npm:^2.2.0"
     cross-spawn: "npm:7.0.3"
     diff: "npm:^5.1.0"
+    dotenv: "npm:^16.3.1"
     esbuild: "npm:esbuild-wasm@^0.15.15"
     globby: "npm:^11.0.1"
     got: "npm:^11.7.0"
@@ -12537,6 +12538,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"dotenv@npm:^16.3.1":
+  version: 16.3.1
+  resolution: "dotenv@npm:16.3.1"
+  checksum: dbb778237ef8750e9e3cd1473d3c8eaa9cc3600e33a75c0e36415d0fa0848197f56c3800f77924c70e7828f0b03896818cd52f785b07b9ad4d88dba73fbba83f
+  languageName: node
+  linkType: hard
+
 "dotenv@npm:^8.2.0":
   version: 8.2.0
   resolution: "dotenv@npm:8.2.0"