feat(exec): improve plugin-exec (#1159)

* feat(exec): automatically update cache when using the `exec:` protocol

* feat(exec): expose more information to `exec` script using an `execEnv` global variable

* chore(release-workflow): set releases

* docs: update `exec` documentation

* refactor: small changes

* test: add test for `execEnv` global variable injection

* test: add test for cache updating

* docs: change `NativePath` to `string`

* docs: move `exec` documentation to the `plugin-exec` readme

* refactor: change locator to stringified locator

* fix: fix `generatorFs` implementation

* feat(exec): expose the built-in modules as global variables

* refactor: change `envFile` folder

* refactor: remove the log variables

* refactor: remove `generatorPath` from `execEnv`

* refactor: change `tempDir` and add `buildDir`

* docs/test: update documentation and tests

* Update README.md

* refactor(exec): create folders before spawning the process

Co-authored-by: Maël Nison <[email protected]>

.yarn/versions/1e9c3b66.yml

@@ -0,0 +1,2 @@
+releases:
+  "@yarnpkg/plugin-exec": prerelease

packages/acceptance-tests/pkg-tests-specs/sources/protocols/execs.test.js

@@ -15,16 +15,100 @@ describe(`Protocols`, () => {
         }));
 
         await xfs.writeFilePromise(`${path}/genpkg.js`, `
-          const fs = require('fs');
-          fs.mkdirSync('build');
-          fs.writeFileSync('build/index.js', 'module.exports = 42;');
-          fs.writeFileSync('build/package.json', '{}');
+          const {buildDir} = execEnv;
+          fs.writeFileSync(path.join(buildDir, 'index.js'), 'module.exports = 42;');
+          fs.writeFileSync(path.join(buildDir, 'package.json'), '{}');
         `);
 
         await run(`install`);
 
         await expect(source(`require('dynamic-pkg')`)).resolves.toEqual(42);
       }),
     );
+
+    test(
+      `it should correctly inject the built-in modules as global variables`,
+      makeTemporaryEnv({
+        dependencies: {
+          [`dynamic-pkg`]: `exec:./genpkg.js`,
+        },
+      }, async ({path, run, source}) => {
+        await xfs.writeFilePromise(`${path}/.yarnrc.yml`, stringifySyml({
+          plugins: [require.resolve(`@yarnpkg/monorepo/scripts/plugin-exec.js`)],
+        }));
+
+        await xfs.writeFilePromise(`${path}/genpkg.js`, `
+          const {buildDir} = execEnv;
+          fs.writeFileSync(path.join(buildDir, 'index.js'), \`module.exports = \${JSON.stringify(Object.getOwnPropertyNames(global))};\`);
+          fs.writeFileSync(path.join(buildDir, 'package.json'), '{}');
+        `);
+
+        await run(`install`);
+
+        await expect(source(`require('dynamic-pkg')`)).resolves.toEqual(
+          expect.arrayContaining(
+            require(`module`).builtinModules.filter((name) => name !== `module` && !name.startsWith(`_`)).concat([`Module`]),
+          )
+        );
+      }),
+    );
+
+    test(
+      `it should correctly inject the \`execEnv\` global variable`,
+      makeTemporaryEnv({
+        dependencies: {
+          [`dynamic-pkg`]: `exec:./genpkg.js`,
+        },
+      }, async ({path, run, source}) => {
+        await xfs.writeFilePromise(`${path}/.yarnrc.yml`, stringifySyml({
+          plugins: [require.resolve(`@yarnpkg/monorepo/scripts/plugin-exec.js`)],
+        }));
+
+        await xfs.writeFilePromise(`${path}/genpkg.js`, `
+          const {buildDir} = execEnv;
+          fs.writeFileSync(path.join(buildDir, 'index.js'), \`module.exports = \${JSON.stringify(execEnv)};\`);
+          fs.writeFileSync(path.join(buildDir, 'package.json'), '{}');
+        `);
+
+        await run(`install`);
+
+        await expect(source(`require('dynamic-pkg')`)).resolves.toMatchObject({
+          tempDir: expect.any(String),
+          buildDir: expect.any(String),
+          locator: expect.any(String),
+        });
+      }),
+    );
+
+    test(
+      `it should update the cache`,
+      makeTemporaryEnv({
+        dependencies: {
+          [`dynamic-pkg`]: `exec:./genpkg.js`,
+        },
+      }, async ({path, run, source}) => {
+        await xfs.writeFilePromise(`${path}/.yarnrc.yml`, stringifySyml({
+          plugins: [require.resolve(`@yarnpkg/monorepo/scripts/plugin-exec.js`)],
+        }));
+
+        await xfs.writeFilePromise(`${path}/genpkg.js`, `
+          const {buildDir} = execEnv;
+          fs.writeFileSync(path.join(buildDir, 'index.js'), 'module.exports = 42;');
+          fs.writeFileSync(path.join(buildDir, 'package.json'), '{}');
+        `);
+
+        await run(`install`);
+
+        await xfs.writeFilePromise(`${path}/genpkg.js`, `
+          const {buildDir} = execEnv;
+          fs.writeFileSync(path.join(buildDir, 'index.js'), 'module.exports = 100;');
+          fs.writeFileSync(path.join(buildDir, 'package.json'), '{}');
+        `);
+
+        await run(`install`);
+
+        await expect(source(`require('dynamic-pkg')`)).resolves.toEqual(100);
+      }),
+    );
   });
 });

packages/gatsby/content/features/protocols.md

@@ -4,27 +4,50 @@ path: /features/protocols
 title: "Protocols"
 ---
 
+```toc
+# This code block gets replaced with the Table of Contents
+```
+
+## Table
+
 The following protocols can be used by any dependency entry listed in the `dependencies` or `devDependencies` fields. While they work regardless of the context we strongly recommend you to only use semver ranges on published packages as they are the one common protocol whose semantic is clearly defined across all package managers.
 
-| Name | Example | Description |
-| --- | --- | --- |
-| Semver | `^1.2.3` | Resolves from the default registry |
-| Tag | `latest` | Resolves from the default registry |
-| Npm alias | `npm:name@...` | Resolves from the npm registry |
-| GitHub | `foo/bar` | Alias for the `github:` protocol |
-| GitHub | `github:foo/bar` | Downloads a **public** package from GitHub |
-| File | `file:./my-package` | Copies the target location into the cache |
-| Link | `link:./my-folder` | Creates a link to the `./my-folder` folder (ignore dependencies) |
-| Patch | `patch:[email protected]#./my-patch.patch` | Creates a patched copy of the original package |
-| Portal | `portal:./my-folder` | Creates a link to the `./my-folder` folder (follow dependencies) |
+| Name          | Example                                 | Description                                                                                                                     |
+| ------------- | --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
+| Semver        | `^1.2.3`                                | Resolves from the default registry                                                                                              |
+| Tag           | `latest`                                | Resolves from the default registry                                                                                              |
+| Npm alias     | `npm:name@...`                          | Resolves from the npm registry                                                                                                  |
+| GitHub        | `foo/bar`                               | Alias for the `github:` protocol                                                                                                |
+| GitHub        | `github:foo/bar`                        | Downloads a **public** package from GitHub                                                                                      |
+| File          | `file:./my-package`                     | Copies the target location into the cache                                                                                       |
+| Link          | `link:./my-folder`                      | Creates a link to the `./my-folder` folder (ignore dependencies)                                                                |
+| Patch         | `patch:[email protected]#./my-patch.patch` | Creates a patched copy of the original package                                                                                  |
+| Portal        | `portal:./my-folder`                    | Creates a link to the `./my-folder` folder (follow dependencies)                                                                |
+| [Exec](#exec) | `exec:./my-generator-package`           | <sup>*Experimental & Plugin*</sup><br>Instructs Yarn to execute the specified Node script and use its output as package content |
+
+## Details
+
+### Exec
+
+> **Experimental**
+>
+> This feature is still incubating, and we'll likely be improving it based on your feedback.
+
+> **Plugin**
+>
+> To use this protocol, first install the `exec` plugin: `yarn plugin import exec`
+
+The documentation and usage can be found on GitHub: [yarnpkg/berry/blob/master/packages/plugin-exec/README.md](https://github.com/yarnpkg/berry/blob/master/packages/plugin-exec/README.md).
+
+## Questions & Answers
 
-## Why can't I add dependencies through the `patch:` protocol?
+### Why can't I add dependencies through the `patch:` protocol?
 
 A Yarn install is split across multiple steps (described [here](/advanced/architecture#install-architecture)). Most importantly, we first fully resolve the dependency tree, and only then do we download the packages from their remote sources. Since patches occur during this second step, by the time we inject the new dependencies it's already too late as the dependency tree has already been frozen.
 
 In order to add dependencies to a package, either fork it (and reference it through the Git protocol, for example), or use the [`packageExtensions`](/configuration/yarnrc#packageExtensions) mechanism which is specifically made to add new runtime dependencies to packages.
 
-## What's the difference between `link:` and `portal:`?
+### What's the difference between `link:` and `portal:`?
 
 The `link:` protocol is meant to link a package name to a folder on the disk - any folder. For example one perfect use case for the `link:` protocol is to map your `src` folder to a clearer name that you can then use from your Node applications without having to use relative paths (for example you could link `my-app` to `link:./src` so that you can call `require('my-app')` from any file within your application).
 

packages/plugin-exec/README.md

@@ -1,5 +1,9 @@
 # `@yarnpkg/plugin-exec`
 
+> **Experimental**
+>
+> This feature is still incubating, and we'll likely be improving it based on your feedback.
+
 This plugin will add support to Yarn for the `exec:` protocol. This protocol is special in that it'll instruct Yarn to execute the specified Node script and use its output as package content (thus treating the script as a kind of package factory).
 
 ## Install
@@ -23,21 +27,74 @@ yarn plugin import exec
 **gen-pkg.js**
 
 ```js
-const {mkdirSync, writeFileSync} = require(`fs`);
-const generatorPath = process.argv[2];
-
-mkdirSync(`${generatorPath}/build`);
+const {buildDir} = execEnv;
 
-writeFileSync(`${generatorPath}/build/package.json`, JSON.stringify({
+fs.writeFileSync(path.join(buildDir, `package.json`), JSON.stringify({
   name: `pkg`,
   version: `1.0.0`,
 }));
 
-writeFileSync(`${generatorPath}/build/index.js`, `module.exports = ${Date.now()};\n`);
+fs.writeFileSync(path.join(buildDir, `index.js`), `module.exports = ${Date.now()};\n`);
 ```
 
+## Rational
+
+Typical Yarn fetchers download packages from the internet - this works fine if the project you want to use got packaged beforehand, but fails short as soon as you need to bundle it yourself. Yarn's builtin mechanism allows you to run the `prepare` script on compatible git repositories and use the result as final package, but even that isn't always enough - you may need to clone a specific branch, go into a specific directory, run a specific build script ... all things that makes it hard for us to support every single use case.
+
+The `exec:` protocol represents a way to define yourself how the specified package should be fetched. In a sense, it can be seen as a more high-level version of the [Fetcher API](/advanced/lexicon#fetcher) that Yarn provides.
+
 ## Documentation
 
-The script will be invoked with one parameter which is a temporary directory. You're free to do whatever you want inside, but at the end of the execution Yarn will expect a `build` directory to have been created inside it that will then be compressed into an archive and stored within the cache.
+The JavaScript file targeted by the `exec:` protocol will be invoked inside a temporary directory at fetch-time with a preconfigured runtime environment. The script is then expected to populate a special directory defined in the environment, and exit once the generation has finished.
+
+### Generator scripts & `require`
+
+Because the generator will be called in a very special context (before any package has been installed on the disk), it won't be able to call the `require` function (not even with relative paths). Should you need very complex generators, just bundle them up beforehand in a single script using tools such as Webpack or Rollup.
+
+Because of this restriction, and because generators will pretty much always need to use the Node builtin modules, those are made available in the global scope - in a very similar way to what the Node REPL already does. As a result, no need to manually require the `fs` module: it's available through the global `fs` variable!
+
+### Runtime environment
+
+In order to let the script knows about the various predefined folders involved in the generation process, Yarn will inject a special `execEnv` global variable available to the script. This object's [interface](/api/interfaces/plugin_exec.execenv.html) is defined as such:
+
+| Property   | Type     | Description                                                                                                                                                     |
+| ---------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `tempDir`  | `string` | The absolute path of the empty temporary directory. It is created before the script is invoked.                                                                 |
+| `buildDir` | `string` | The absolute path of the empty build directory that will be compressed into an archive and stored within the cache. It is created before the script is invoked. |
+| `locator`  | `string` | The stringified `Locator` identifying the generator package.                                                                                                    |
+You're free to do whatever you want inside `execEnv.tempDir` but, at the end of the execution, Yarn will expect `execEnv.buildDir` to contain the files that can be compressed into an archive and stored within the cache.
+
+### Example
 
-Busting the cache isn't currently supported - you'll need to manually remove the relevant archives from your cache each time you want to update the content of the package. Help welcome!
+Generate an hello world package:
+
+```ts
+fs.writeFileSync(path.join(execEnv.buildDir, 'package.json'), JSON.stringify({
+  name: 'hello-world',
+  version: '1.0.0',
+}));
+
+fs.writeFileSync(path.join(execEnv.buildDir, 'index.js'), `
+  module.exports = 'hello world!';
+`);
+```
+
+Clone a monorepo and build a specific package:
+
+```ts
+const pathToRepo = path.join(execEnv.tempDir, 'repo');
+const pathToArchive = path.join(execEnv.tempDir, 'archive.tgz');
+const pathToSubpackage = path.join(pathToRepo, 'packages/foobar');
+
+# Clone the repository
+child_process.execFileSync(`git`, [`clone`, `[email protected]:foo/bar`, pathToRepo]);
+
+# Install the dependencies
+child_process.execFileSync(`yarn`, [`install`], {cwd: pathToRepo});
+
+# Pack a specific workspace
+child_process.execFileSync(`yarn`, [`pack`, `--out`, pathToArchive], {cwd: pathToSubpackage});
+
+# Send the package content into the build directory
+child_process.execFileSync(`tar`, [`xfz`, `--strip-components=1`, pathToArchive, `-C`, execEnv.buildDir]);
+```