From 846d351130d0f7cc65b7d4a9e8986b6fa29ddfa7 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Tue, 11 Jan 2022 20:43:16 -0800 Subject: [PATCH] refactor: move yarn fetch to a separate external repo Previously every @node_some_platform had a yarn program in it, which doesn't make sense since yarn is platform-independent JS. Instead make a @yarn repo and use that for getting our yarn binary. Trivially allows a vendored yarn by passing the yarn attribute to yarn_install overriding the default. BREAKING CHANGE: vendored_yarn attribute is removed --- .bazelci/presubmit.yml | 6 +- .circleci/config.yml | 2 +- docs/Built-ins.md | 12 +- docs/Core.md | 188 +++++++++------ docs/repositories.md | 9 +- e2e/nodejs_host/BUILD.bazel | 6 +- examples/vendored_node_and_yarn/WORKSPACE | 2 +- internal/node/node_repositories.bzl | 17 +- internal/npm_install/npm_install.bzl | 36 ++- nodejs/index.for_docs.bzl | 2 + .../node => nodejs/private}/node_labels.bzl | 5 +- nodejs/private/nodejs_repo_host_os_alias.bzl | 3 - nodejs/repositories.bzl | 208 +---------------- nodejs/yarn_repositories.bzl | 217 ++++++++++++++++++ 14 files changed, 395 insertions(+), 318 deletions(-) rename {internal/node => nodejs/private}/node_labels.bzl (87%) create mode 100644 nodejs/yarn_repositories.bzl diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 811b9eeba8..5305b13801 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -14,7 +14,7 @@ tasks: name: ubuntu1804 platform: ubuntu1804 run_targets: - - "@node16_linux_amd64//:yarn" + - "@yarn//:yarn" # Regression test for #1493 - "//packages/create:npm_package.pack" - "//internal/node/test:no_deps" @@ -129,7 +129,7 @@ tasks: name: macos platform: macos run_targets: - - "@node16_darwin_amd64//:yarn" + - "@yarn//:yarn" # Regression test for #1493 - "//packages/create:npm_package.pack" - "//internal/node/test:no_deps" @@ -219,7 +219,7 @@ tasks: name: windows platform: windows run_targets: - - "@node16_windows_amd64//:yarn" + - "@yarn//:yarn" - "//internal/node/test:no_deps" - "//internal/node/test:has_deps_legacy" - "//internal/node/test:has_deps" diff --git a/.circleci/config.yml b/.circleci/config.yml index 6fa6750349..bb6e3b7173 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -101,7 +101,7 @@ jobs: - *init_bazel # Run yarn - - run: bazel run @node16_linux_amd64//:yarn + - run: bazel run @yarn//:yarn # Save all node_modules to the cache - save_cache: diff --git a/docs/Built-ins.md b/docs/Built-ins.md index 84c0f4d355..bc63a1ea6b 100755 --- a/docs/Built-ins.md +++ b/docs/Built-ins.md @@ -728,7 +728,7 @@ Defaults to `""`

node_repository

-(*String*): The basename for nodejs toolchains. +(*String*): The basename for a nodejs toolchain to use for running npm. Usually this is the value of the `name` attribute given to a nodejs_register_toolchains call in WORKSPACE Defaults to `"nodejs"` @@ -1182,7 +1182,7 @@ yarn_install(name, generate_local_modules_build_files, included_files, links, manual_build_file_contents, node_repository, package_json, package_json_remove, package_json_replace, package_path, patch_args, patch_tool, post_install_patches, pre_install_patches, quiet, repo_mapping, - strict_visibility, symlink_node_modules, timeout, use_global_yarn_cache, yarn_lock) + strict_visibility, symlink_node_modules, timeout, use_global_yarn_cache, yarn, yarn_lock) Runs yarn install during workspace setup. @@ -1415,7 +1415,7 @@ Defaults to `""`

node_repository

-(*String*): The basename for nodejs toolchains. +(*String*): The basename for a nodejs toolchain to use for running npm. Usually this is the value of the `name` attribute given to a nodejs_register_toolchains call in WORKSPACE Defaults to `"nodejs"` @@ -1619,6 +1619,12 @@ to yarn so that the local cache is contained within the external repository. Defaults to `True` +

yarn

+ +(*Label*): The yarn.js entry point to execute + +Defaults to `@yarn//:bin/yarn` +

yarn_lock

(*Label, mandatory*) diff --git a/docs/Core.md b/docs/Core.md index de1d8c236f..5df86e7774 100644 --- a/docs/Core.md +++ b/docs/Core.md @@ -27,29 +27,28 @@ We plan to clean these up and port into `rules_nodejs` in a future major release 
 node_repositories(name, node_download_auth, node_repositories, node_urls, node_version, platform,
-                  repo_mapping, use_nvmrc, vendored_node, vendored_yarn, yarn_download_auth,
-                  yarn_repositories, yarn_urls, yarn_version)
+                  repo_mapping, use_nvmrc, vendored_node)
To be run in user's WORKSPACE to install rules_nodejs dependencies. -This rule sets up node, npm, and yarn. The versions of these tools can be specified in one of three ways +This rule sets up node, npm, and npx. The versions of these tools can be specified in one of three ways ### Simplest Usage -Specify no explicit versions. This will download and use the latest NodeJS & Yarn that were available when the +Specify no explicit versions. This will download and use the latest NodeJS that was available when the version of rules_nodejs you're using was released. Note that you can skip calling `node_repositories` in your WORKSPACE file - if you later try to `yarn_install` or `npm_install`, we'll automatically select this simple usage for you. ### Forced version(s) -You can select the version of NodeJS and/or Yarn to download & use by specifying it when you call node_repositories, +You can select the version of NodeJS to download & use by specifying it when you call node_repositories, using a value that matches a known version (see the default values) ### Using a custom version -You can pass in a custom list of NodeJS and/or Yarn repositories and URLs for node_resositories to use. +You can pass in a custom list of NodeJS repositories and URLs for node_repositories to use. #### Custom NodeJS versions @@ -78,40 +77,10 @@ node_repositories( A Mac client will try to download node from `https://mycorpproxy/mirror/node/v10.10.0/node-v10.10.0-darwin-x64.tar.gz` and expect that file to have sha256sum `00b7a8426e076e9bf9d12ba2d571312e833fe962c70afafd10ad3682fdeeaa5e` -#### Custom Yarn versions - -To specify custom Yarn versions, use the `yarn_repositories` attribute - -```python -node_repositories( - yarn_repositories = { - "1.12.1": ("yarn-v1.12.1.tar.gz", "yarn-v1.12.1", "09bea8f4ec41e9079fa03093d3b2db7ac5c5331852236d63815f8df42b3ba88d"), - }, -) -``` - -Like `node_urls`, the `yarn_urls` attribute can be used to provide a list of custom URLs to use to download yarn - -```python -node_repositories( - yarn_repositories = { - "1.12.1": ("yarn-v1.12.1.tar.gz", "yarn-v1.12.1", "09bea8f4ec41e9079fa03093d3b2db7ac5c5331852236d63815f8df42b3ba88d"), - }, - yarn_version = "1.12.1", - yarn_urls = [ - "https://github.com/yarnpkg/yarn/releases/download/v{version}/{filename}", - ], -) -``` - -Will download yarn from https://github.com/yarnpkg/yarn/releases/download/v1.2.1/yarn-v1.12.1.tar.gz -and expect the file to have sha256sum `09bea8f4ec41e9079fa03093d3b2db7ac5c5331852236d63815f8df42b3ba88d`. - -If you don't use Yarn at all, you can skip downloading it by setting `yarn_urls = []`. ### Using a local version -To avoid downloads, you can check in vendored copies of NodeJS and/or Yarn and set vendored_node and or vendored_yarn +To avoid downloads, you can check in vendored copies of NodeJS and set vendored_node to point to those before calling node_repositories. You can also point to a location where node is installed on your computer, but we don't recommend this because it leads to version skew between you, your coworkers, and your Continuous Integration environment. It also ties your build to a single platform, preventing you from cross-compiling into a Linux docker image on Mac for example. @@ -189,45 +158,6 @@ If set then also set node_version to the version that of node that is vendored. Defaults to `None` -

vendored_yarn

- -(*Label*): the local path to a pre-installed yarn tool - -Defaults to `None` - -

yarn_download_auth

- -(*Dictionary: String -> String*): auth to use for all url requests -Example: {"type": "basic", "login": "", "password": "" } - -Defaults to `{}` - -

yarn_repositories

- -(*Dictionary: String -> List of strings*): Custom list of yarn repositories to use. - -Dictionary mapping Yarn versions to their corresponding (filename, strip_prefix, sha256) tuples. - -By default, if this attribute has no items, we'll use a list of all public NodeJS releases. - -Defaults to `{}` - -

yarn_urls

- -(*List of strings*): custom list of URLs to use to download Yarn - -Each entry is a template, similar to the `node_urls` attribute, using `yarn_version` and `yarn_repositories` in the substitutions. - -If this list is empty, we won't download yarn at all. - -Defaults to `["https://github.com/yarnpkg/yarn/releases/download/v{version}/{filename}"]` - -

yarn_version

- -(*String*): the specific version of Yarn to install - -Defaults to `"1.22.11"` - ## node_toolchain @@ -299,6 +229,112 @@ Defaults to `None` Defaults to `""` +## yarn_repositories + +**USAGE** + +
+yarn_repositories(name, node_repository, repo_mapping, yarn_download_auth, yarn_releases, yarn_urls,
+                  yarn_version)
+
+ +Repository rule to fetch the yarnpkg.com package manager. + +Note, the recommended name is "yarn". If you choose a different name, you'll have to override the +`yarn` attribute in your `yarn_install` rule to point to your `yarn.js` file. + +## Custom Yarn versions + +To specify custom Yarn versions, use the `yarn_releases` attribute + +```python +yarn_repositories( + yarn_releases = { + "1.12.1": ("yarn-v1.12.1.tar.gz", "yarn-v1.12.1", "09bea8f4ec41e9079fa03093d3b2db7ac5c5331852236d63815f8df42b3ba88d"), + }, +) +``` + +Like `node_urls`, the `yarn_urls` attribute can be used to provide a list of custom URLs to use to download yarn + +```python +yarn_repositories( + yarn_releases = { + "1.12.1": ("yarn-v1.12.1.tar.gz", "yarn-v1.12.1", "09bea8f4ec41e9079fa03093d3b2db7ac5c5331852236d63815f8df42b3ba88d"), + }, + yarn_version = "1.12.1", + yarn_urls = [ + "https://github.com/yarnpkg/yarn/releases/download/v{version}/{filename}", + ], +) +``` + +Will download yarn from https://github.com/yarnpkg/yarn/releases/download/v1.2.1/yarn-v1.12.1.tar.gz +and expect the file to have sha256sum `09bea8f4ec41e9079fa03093d3b2db7ac5c5331852236d63815f8df42b3ba88d`. + +If you don't use Yarn at all, you can skip downloading it by setting `yarn_urls = []`. + +## Vendored yarn + +You can vendor the `yarn.js` file into your repo. In this case, don't call `yarn_repositories` at all. +Just pass the label of your vendored file to the `yarn` attribute of `yarn_install`. + + +**ATTRIBUTES** + + +

name

+ +(*Name, mandatory*): A unique name for this repository. + + +

node_repository

+ +(*String*): The basename for a nodejs toolchain to use for running yarn. + Usually this is the value of the `name` attribute given to a nodejs_register_toolchains call in WORKSPACE + +Defaults to `"nodejs"` + +

repo_mapping

+ +(*Dictionary: String -> String, mandatory*): A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.

For example, an entry `"@foo": "@bar"` declares that, for any time this repository depends on `@foo` (such as a dependency on `@foo//some:target`, it should actually resolve that dependency within globally-declared `@bar` (`@bar//some:target`). + + +

yarn_download_auth

+ +(*Dictionary: String -> String*): auth to use for all url requests + Example: {"type": "basic", "login": "", "password": "" } + +Defaults to `{}` + +

yarn_releases

+ +(*Dictionary: String -> List of strings*): Custom list of yarn releases to use. + + Dictionary mapping Yarn versions to their corresponding (filename, strip_prefix, sha256) tuples. + + By default, if this attribute has no items, we'll use a list of all public releases which + is periodically mirrored to rules_nodejs. + +Defaults to `{}` + +

yarn_urls

+ +(*List of strings*): custom list of URLs to use to download Yarn + + Each entry is a template using `yarn_version` and `yarn_releases` in the substitutions. + + If this list is empty, we won't download yarn at all. + +Defaults to `["https://github.com/yarnpkg/yarn/releases/download/v{version}/{filename}"]` + +

yarn_version

+ +(*String*): the specific version of Yarn to install + +Defaults to `"1.22.11"` + + ## DeclarationInfo **USAGE** diff --git a/docs/repositories.md b/docs/repositories.md index fd4fbd7bbb..8df63061d1 100644 --- a/docs/repositories.md +++ b/docs/repositories.md @@ -7,7 +7,7 @@ These are referenced with the `@repo//` syntax in your BUILD files. ## @nodejs_host This repository is created by calling the `node_repositories` function in your `WORKSPACE` file. -It contains the node, npm, and yarn programs. +It contains the node, npm, and npx programs. As always, `bazel query` is useful for learning about what targets are available. @@ -23,7 +23,12 @@ Some ways you can use this: - Run the Bazel-managed version of node: `bazel run @nodejs_host//:node path/to/program.js` - Run the Bazel-managed version of npm: `bazel run @nodejs_host//:npm` -- Run the Bazel-managed version of yarn: `bazel run @nodejs_host//:yarn` + +## @yarn + +This repository is created by calling the `yarn_repositories` function in your `WORKSPACE` file. + +- Run the Bazel-managed version of yarn: `bazel run @yarn//:yarn` ## @npm diff --git a/e2e/nodejs_host/BUILD.bazel b/e2e/nodejs_host/BUILD.bazel index ac5afc1b7a..794cd6dcd8 100644 --- a/e2e/nodejs_host/BUILD.bazel +++ b/e2e/nodejs_host/BUILD.bazel @@ -14,9 +14,9 @@ nodejs_test( "@nodejs_host//:npm_bin", "@nodejs_host//:npm_files", "@nodejs_host//:npx_bin", - "@nodejs_host//:yarn", - "@nodejs_host//:yarn_bin", - "@nodejs_host//:yarn_files", + "@yarn", + "@yarn//:yarn_bin", + "@yarn//:yarn_files", ], entry_point = ":index.spec.js", ) diff --git a/examples/vendored_node_and_yarn/WORKSPACE b/examples/vendored_node_and_yarn/WORKSPACE index 1008b2f84f..642c2ff5c5 100644 --- a/examples/vendored_node_and_yarn/WORKSPACE +++ b/examples/vendored_node_and_yarn/WORKSPACE @@ -48,7 +48,6 @@ load("@build_bazel_rules_nodejs//:index.bzl", "node_repositories", "yarn_install node_repositories( node_version = "10.12.0", vendored_node = "@vendored_node_10_12_0//:node-v10.12.0-linux-x64", - vendored_yarn = "@vendored_yarn_1_10_0//:yarn-v1.10.0", ) yarn_install( @@ -59,5 +58,6 @@ yarn_install( ], exports_directories_only = False, package_json = "//:package.json", + yarn = "@vendored_yarn_1_10_0//:yarn-v1.10.0/bin/yarn.js", yarn_lock = "//:yarn.lock", ) diff --git a/internal/node/node_repositories.bzl b/internal/node/node_repositories.bzl index 74f350dc52..a2136d4d89 100644 --- a/internal/node/node_repositories.bzl +++ b/internal/node/node_repositories.bzl @@ -21,6 +21,7 @@ See https://docs.bazel.build/versions/main/skylark/repository_rules.html load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load("@rules_nodejs//nodejs/private:os_name.bzl", "OS_ARCH_NAMES", "node_exists_for_os", "os_name") load("@rules_nodejs//nodejs:repositories.bzl", "DEFAULT_NODE_VERSION", "nodejs_register_toolchains", node_repositories_rule = "node_repositories") +load("@rules_nodejs//nodejs:yarn_repositories.bzl", "yarn_repositories") load("//internal/common:check_bazel_version.bzl", "check_bazel_version") def node_repositories(**kwargs): @@ -41,13 +42,15 @@ def node_repositories(**kwargs): minimum_bazel_version = "4.0.0", ) - # buildifier: disable=print - print("""WARN: node_repositories is deprecated, please instead use: - load("@rules_nodejs//nodejs:repositories.bzl", "nodejs_register_toolchains") - nodejs_register_toolchains(name = "nodejs") - - See https://github.com/bazelbuild/rules_nodejs/wiki/Migrating-to-5.0 - """) + # Back-compat: allow yarn_repositories args to be provided to node_repositories + yarn_args = {} + for k, v in kwargs.items(): + if k.startswith("yarn_"): + yarn_args[k] = kwargs.pop(k) + yarn_repositories( + name = "yarn", + **yarn_args + ) # This needs to be setup so toolchains can access nodejs for all different versions node_version = kwargs.get("node_version", DEFAULT_NODE_VERSION) diff --git a/internal/npm_install/npm_install.bzl b/internal/npm_install/npm_install.bzl index 0e829540af..3f61bc04be 100644 --- a/internal/npm_install/npm_install.bzl +++ b/internal/npm_install/npm_install.bzl @@ -22,9 +22,9 @@ See discussion in the README. """ load("@rules_nodejs//nodejs/private:os_name.bzl", "is_windows_os", "os_name") +load("@rules_nodejs//nodejs/private:node_labels.bzl", "get_node_label", "get_npm_label") load("//:version.bzl", "VERSION") load("//internal/common:check_bazel_version.bzl", "check_bazel_version") -load("//internal/node:node_labels.bzl", "get_node_label", "get_npm_label", "get_yarn_label") COMMON_ATTRIBUTES = dict(dict(), **{ "data": attr.label_list( @@ -202,7 +202,7 @@ fine grained npm dependencies. ), "node_repository": attr.string( default = "nodejs", - doc = """The basename for nodejs toolchains. + doc = """The basename for a nodejs toolchain to use for running npm. Usually this is the value of the `name` attribute given to a nodejs_register_toolchains call in WORKSPACE""", ), "package_json": attr.label( @@ -609,17 +609,20 @@ def _copy_data_dependencies(repository_ctx): # files as npm file:// packages _copy_file(repository_ctx, f) -def _add_node_repositories_info_deps(repository_ctx): +def _add_node_repositories_info_deps(repository_ctx, yarn = None): # Add a dep to the node_info & yarn_info files from node_repositories # so that if the node or yarn versions change we re-run the repository rule repository_ctx.symlink( Label("@{}_{}//:node_info".format(repository_ctx.attr.node_repository, os_name(repository_ctx))), repository_ctx.path("_node_info"), ) - repository_ctx.symlink( - Label("@{}_{}//:yarn_info".format(repository_ctx.attr.node_repository, os_name(repository_ctx))), - repository_ctx.path("_yarn_info"), - ) + + # A custom yarn might be vendored, and not have a yarn_info file in the repo. + if str(yarn) == _DEFAULT_YARN: + repository_ctx.symlink( + Label("@{}//:yarn_info".format(yarn.workspace_name)), + repository_ctx.path("_yarn_info"), + ) def _symlink_node_modules(repository_ctx): package_json_dir = repository_ctx.path(repository_ctx.attr.package_json).dirname @@ -791,7 +794,7 @@ check if yarn is being run by the `npm_install` repository rule.""", ) def _detect_yarn_version(rctx, yarn): - result = rctx.execute([rctx.path(yarn), "--version"]) + result = rctx.execute([yarn, "--version"]) if result.return_code: fail("yarn --version failed: %s (%s)" % (result.stdout, result.stderr)) if result.stdout.startswith("1."): @@ -805,7 +808,10 @@ def _yarn_install_impl(repository_ctx): is_windows_host = is_windows_os(repository_ctx) node = repository_ctx.path(get_node_label(repository_ctx)) - yarn = get_yarn_label(repository_ctx) + yarn_label = repository_ctx.attr.yarn + if is_windows_host: + yarn_label = yarn_label.relative(":bin/yarn.cmd") + yarn = repository_ctx.path(yarn_label) yarn_version = _detect_yarn_version(repository_ctx, yarn) yarn_args = [] @@ -878,7 +884,7 @@ unset npm_config_registry (cd "{root}"; "{yarn}" {yarn_args}) """.format( root = root, - yarn = repository_ctx.path(yarn), + yarn = yarn, yarn_args = " ".join(yarn_args), ), executable = True, @@ -893,7 +899,7 @@ set “npm_config_registry=” cd /D "{root}" && "{yarn}" {yarn_args} """.format( root = root, - yarn = repository_ctx.path(yarn), + yarn = yarn, yarn_args = " ".join(yarn_args), ), executable = True, @@ -903,7 +909,7 @@ cd /D "{root}" && "{yarn}" {yarn_args} _copy_file(repository_ctx, repository_ctx.attr.package_json) _copy_data_dependencies(repository_ctx) _add_scripts(repository_ctx) - _add_node_repositories_info_deps(repository_ctx) + _add_node_repositories_info_deps(repository_ctx, yarn = repository_ctx.attr.yarn) _apply_pre_install_patches(repository_ctx) # _package_json_changes should be called _after_ _apply_pre_install_patches (as per docstring) @@ -937,6 +943,8 @@ cd /D "{root}" && "{yarn}" {yarn_args} _create_build_files(repository_ctx, "yarn_install", node, repository_ctx.attr.yarn_lock, repository_ctx.attr.generate_local_modules_build_files) +_DEFAULT_YARN = "@yarn//:bin/yarn" + yarn_install = repository_rule( attrs = dict(COMMON_ATTRIBUTES, **{ "args": attr.string_list( @@ -987,6 +995,10 @@ If False, this rule will pass `--cache-folder /path/to/external/repository/__yar to yarn so that the local cache is contained within the external repository. """, ), + "yarn": attr.label( + default = _DEFAULT_YARN, + doc = "The yarn.js entry point to execute", + ), "yarn_lock": attr.label( mandatory = True, allow_single_file = True, diff --git a/nodejs/index.for_docs.bzl b/nodejs/index.for_docs.bzl index 4024b15ba5..268c92e68b 100644 --- a/nodejs/index.for_docs.bzl +++ b/nodejs/index.for_docs.bzl @@ -26,6 +26,7 @@ load( _js_module_info = "js_module_info", ) load(":repositories.bzl", _node_repositories = "node_repositories") +load(":yarn_repositories.bzl", _yarn_repositories = "yarn_repositories") load(":toolchain.bzl", _node_toolchain = "node_toolchain") DeclarationInfo = _DeclarationInfo @@ -37,3 +38,4 @@ DirectoryFilePathInfo = _DirectoryFilePathInfo UserBuildSettingInfo = _UserBuildSettingInfo node_repositories = _node_repositories node_toolchain = _node_toolchain +yarn_repositories = _yarn_repositories diff --git a/internal/node/node_labels.bzl b/nodejs/private/node_labels.bzl similarity index 87% rename from internal/node/node_labels.bzl rename to nodejs/private/node_labels.bzl index 3446299661..2a2f74b727 100644 --- a/internal/node/node_labels.bzl +++ b/nodejs/private/node_labels.bzl @@ -17,7 +17,7 @@ Labels are different on windows and linux/OSX. """ -load("@rules_nodejs//nodejs/private:os_name.bzl", "is_windows_os", "os_name") +load(":os_name.bzl", "is_windows_os", "os_name") def _get_label(rctx, tool): ext = ".cmd" if is_windows_os(rctx) else "" @@ -28,6 +28,3 @@ def get_node_label(rctx): def get_npm_label(rctx): return _get_label(rctx, "bin/npm") - -def get_yarn_label(rctx): - return _get_label(rctx, "bin/yarn") diff --git a/nodejs/private/nodejs_repo_host_os_alias.bzl b/nodejs/private/nodejs_repo_host_os_alias.bzl index 95b33a334e..2e6b781b05 100644 --- a/nodejs/private/nodejs_repo_host_os_alias.bzl +++ b/nodejs/private/nodejs_repo_host_os_alias.bzl @@ -14,12 +14,9 @@ alias(name = "run_npm.bat.template", actual = "@{node_repository}_{os_name}// alias(name = "node_bin", actual = "@{node_repository}_{os_name}//:node_bin") alias(name = "npm_bin", actual = "@{node_repository}_{os_name}//:npm_bin") alias(name = "npx_bin", actual = "@{node_repository}_{os_name}//:npx_bin") -alias(name = "yarn_bin", actual = "@{node_repository}_{os_name}//:yarn_bin") alias(name = "node", actual = "@{node_repository}_{os_name}//:node") alias(name = "npm", actual = "@{node_repository}_{os_name}//:npm") -alias(name = "yarn", actual = "@{node_repository}_{os_name}//:yarn") alias(name = "node_files", actual = "@{node_repository}_{os_name}//:node_files") -alias(name = "yarn_files", actual = "@{node_repository}_{os_name}//:yarn_files") alias(name = "npm_files", actual = "@{node_repository}_{os_name}//:npm_files") exports_files(["index.bzl"]) """.format( diff --git a/nodejs/repositories.bzl b/nodejs/repositories.bzl index 2a3663b1e2..646db0448c 100644 --- a/nodejs/repositories.bzl +++ b/nodejs/repositories.bzl @@ -4,7 +4,6 @@ load("//nodejs/private:os_name.bzl", "assert_node_exists_for_host", "node_exists load("//nodejs/private:node_versions.bzl", "NODE_VERSIONS") load("//nodejs/private:nodejs_repo_host_os_alias.bzl", "nodejs_repo_host_os_alias") load("//nodejs/private:toolchains_repo.bzl", "PLATFORMS", "toolchains_repo") -load("//nodejs/private:yarn_versions.bzl", "YARN_VERSIONS") load("//third_party/github.com/bazelbuild/bazel-skylib:lib/paths.bzl", "paths") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") @@ -15,23 +14,23 @@ BUILT_IN_NODE_PLATFORMS = PLATFORMS.keys() _DOC = """To be run in user's WORKSPACE to install rules_nodejs dependencies. -This rule sets up node, npm, and yarn. The versions of these tools can be specified in one of three ways +This rule sets up node, npm, and npx. The versions of these tools can be specified in one of three ways ### Simplest Usage -Specify no explicit versions. This will download and use the latest NodeJS & Yarn that were available when the +Specify no explicit versions. This will download and use the latest NodeJS that was available when the version of rules_nodejs you're using was released. Note that you can skip calling `node_repositories` in your WORKSPACE file - if you later try to `yarn_install` or `npm_install`, we'll automatically select this simple usage for you. ### Forced version(s) -You can select the version of NodeJS and/or Yarn to download & use by specifying it when you call node_repositories, +You can select the version of NodeJS to download & use by specifying it when you call node_repositories, using a value that matches a known version (see the default values) ### Using a custom version -You can pass in a custom list of NodeJS and/or Yarn repositories and URLs for node_resositories to use. +You can pass in a custom list of NodeJS repositories and URLs for node_repositories to use. #### Custom NodeJS versions @@ -60,40 +59,10 @@ node_repositories( A Mac client will try to download node from `https://mycorpproxy/mirror/node/v10.10.0/node-v10.10.0-darwin-x64.tar.gz` and expect that file to have sha256sum `00b7a8426e076e9bf9d12ba2d571312e833fe962c70afafd10ad3682fdeeaa5e` -#### Custom Yarn versions - -To specify custom Yarn versions, use the `yarn_repositories` attribute - -```python -node_repositories( - yarn_repositories = { - "1.12.1": ("yarn-v1.12.1.tar.gz", "yarn-v1.12.1", "09bea8f4ec41e9079fa03093d3b2db7ac5c5331852236d63815f8df42b3ba88d"), - }, -) -``` - -Like `node_urls`, the `yarn_urls` attribute can be used to provide a list of custom URLs to use to download yarn - -```python -node_repositories( - yarn_repositories = { - "1.12.1": ("yarn-v1.12.1.tar.gz", "yarn-v1.12.1", "09bea8f4ec41e9079fa03093d3b2db7ac5c5331852236d63815f8df42b3ba88d"), - }, - yarn_version = "1.12.1", - yarn_urls = [ - "https://github.com/yarnpkg/yarn/releases/download/v{version}/{filename}", - ], -) -``` - -Will download yarn from https://github.com/yarnpkg/yarn/releases/download/v1.2.1/yarn-v1.12.1.tar.gz -and expect the file to have sha256sum `09bea8f4ec41e9079fa03093d3b2db7ac5c5331852236d63815f8df42b3ba88d`. - -If you don't use Yarn at all, you can skip downloading it by setting `yarn_urls = []`. ### Using a local version -To avoid downloads, you can check in vendored copies of NodeJS and/or Yarn and set vendored_node and or vendored_yarn +To avoid downloads, you can check in vendored copies of NodeJS and set vendored_node to point to those before calling node_repositories. You can also point to a location where node is installed on your computer, but we don't recommend this because it leads to version skew between you, your coworkers, and your Continuous Integration environment. It also ties your build to a single platform, preventing you from cross-compiling into a Linux docker image on Mac for example. @@ -150,43 +119,9 @@ If set then also set node_version to the version found in the .nvmrc file.""", If set then also set node_version to the version that of node that is vendored.""", ), - "vendored_yarn": attr.label( - allow_single_file = True, - doc = "the local path to a pre-installed yarn tool", - ), - "yarn_download_auth": attr.string_dict( - default = {}, - doc = """auth to use for all url requests -Example: {\"type\": \"basic\", \"login\": \"\", \"password\": \"\" } -""", - ), - "yarn_repositories": attr.string_list_dict( - doc = """Custom list of yarn repositories to use. - -Dictionary mapping Yarn versions to their corresponding (filename, strip_prefix, sha256) tuples. - -By default, if this attribute has no items, we'll use a list of all public NodeJS releases. -""", - ), - "yarn_urls": attr.string_list( - default = [ - "https://github.com/yarnpkg/yarn/releases/download/v{version}/{filename}", - ], - doc = """custom list of URLs to use to download Yarn - -Each entry is a template, similar to the `node_urls` attribute, using `yarn_version` and `yarn_repositories` in the substitutions. - -If this list is empty, we won't download yarn at all. -""", - ), - "yarn_version": attr.string( - doc = "the specific version of Yarn to install", - default = "1.22.11", - ), } NODE_EXTRACT_DIR = "bin/nodejs" -YARN_EXTRACT_DIR = "bin/yarnpkg" GET_SCRIPT_DIR = """ # From stackoverflow.com @@ -264,62 +199,6 @@ def _download_node(repository_ctx): sha256 = sha256, )) -def _download_yarn(repository_ctx): - """Used to download a yarn tool package. - - Args: - repository_ctx: The repository rule context - """ - yarn_urls = repository_ctx.attr.yarn_urls - - # If there are no URLs to download yarn, skip the download - if not len(yarn_urls): - repository_ctx.file("yarn_info", content = "# no yarn urls") - return - - # If yarn is vendored locally, we still need the info file but can skip downloading - if repository_ctx.attr.vendored_yarn: - repository_ctx.file("yarn_info", content = "# vendored_yarn: {vendored_yarn}".format( - vendored_yarn = repository_ctx.attr.vendored_yarn, - )) - return - - yarn_version = repository_ctx.attr.yarn_version - yarn_repositories = repository_ctx.attr.yarn_repositories - - # We insert our default value here, not on the attribute's default, so it isn't documented. - # The size of YARN_VERSIONS constant is huge and not useful to document. - if not yarn_repositories.items(): - yarn_repositories = YARN_VERSIONS - - if yarn_version in yarn_repositories: - filename, strip_prefix, sha256 = yarn_repositories[yarn_version] - else: - fail("Unknown Yarn version %s" % yarn_version) - - urls = [url.format(version = yarn_version, filename = filename) for url in yarn_urls] - - auth = {} - for url in urls: - auth[url] = repository_ctx.attr.yarn_download_auth - - repository_ctx.download_and_extract( - auth = auth, - url = urls, - output = YARN_EXTRACT_DIR, - stripPrefix = strip_prefix, - sha256 = sha256, - ) - - repository_ctx.file("yarn_info", content = """# filename: {filename} -# strip_prefix: {strip_prefix} -# sha256: {sha256} -""".format( - filename = filename, - strip_prefix = strip_prefix, - sha256 = sha256, - )) - def _prepare_node(repository_ctx): """Sets up BUILD files and shell wrappers for the versions of NodeJS, npm & yarn just set up. @@ -353,22 +232,6 @@ def _prepare_node(repository_ctx): node_path = NODE_EXTRACT_DIR node_package = NODE_EXTRACT_DIR - if repository_ctx.attr.vendored_yarn: - yarn_path = "/".join([f for f in [ - "../../..", - repository_ctx.attr.vendored_yarn.workspace_root, - repository_ctx.attr.vendored_yarn.package, - repository_ctx.attr.vendored_yarn.name, - ] if f]) - yarn_package = "@%s//%s:%s" % ( - repository_ctx.attr.vendored_yarn.workspace_name, - repository_ctx.attr.vendored_yarn.package, - repository_ctx.attr.vendored_yarn.name, - ) - else: - yarn_path = YARN_EXTRACT_DIR - yarn_package = YARN_EXTRACT_DIR - node_bin = ("%s/bin/node" % node_path) if not is_windows else ("%s/node.exe" % node_path) node_bin_label = ("%s/bin/node" % node_package) if not is_windows else ("%s/node.exe" % node_package) @@ -381,26 +244,17 @@ def _prepare_node(repository_ctx): npx_bin = ("%s/lib/node_modules/npm/bin/npx-cli.js" % node_path) if not is_windows else ("%s/npx.cmd" % node_path) npx_bin_label = ("%s/lib/node_modules/npm/bin/npx-cli.js" % node_package) if not is_windows else ("%s/npx.cmd" % node_package) - # Use the yarn.js script as the bin for osx & linux so there are no symlink issues with `%s/bin/npm` - yarn_bin = ("%s/bin/yarn.js" % yarn_path) if not is_windows else ("%s/bin/yarn.cmd" % yarn_path) - yarn_bin_label = ("%s/bin/yarn.js" % yarn_package) if not is_windows else ("%s/bin/yarn.cmd" % yarn_package) - yarn_script = "%s/bin/yarn.js" % yarn_path - # Ensure that the "vendored" binaries are resolved # Just requesting their path from the repository context is enough to eager-load them if repository_ctx.attr.vendored_node: repository_ctx.path(Label(node_bin_label)) - if repository_ctx.attr.vendored_yarn: - repository_ctx.path(Label(yarn_bin_label)) entry_ext = ".cmd" if is_windows else "" node_entry = "bin/node%s" % entry_ext npm_entry = "bin/npm%s" % entry_ext - yarn_entry = "bin/yarn%s" % entry_ext node_bin_relative = node_bin if repository_ctx.attr.vendored_node else paths.relativize(node_bin, "bin") npm_script_relative = npm_script if repository_ctx.attr.vendored_node else paths.relativize(npm_script, "bin") - yarn_script_relative = yarn_script if repository_ctx.attr.vendored_yarn else paths.relativize(yarn_script, "bin") # The entry points for node for osx/linux and windows if not is_windows: @@ -477,46 +331,6 @@ SET SCRIPT_DIR=%~dp0 script = repository_ctx.path(npm_script), )) - # The entry points for yarn for osx/linux and windows. - # Runs yarn using appropriate node entry point. - # Unset YARN_IGNORE_PATH before calling yarn incase it is set so that - # .yarnrc yarn-path is followed if set. This is for the case when calling - # bazel from yarn with `yarn bazel ...` and yarn follows yarn-path in - # .yarnrc it will set YARN_IGNORE_PATH=1 which will prevent the bazel - # call into yarn from also following the yarn-path as desired. - if not is_windows: - # Yarn entry point - repository_ctx.file( - "bin/yarn", - content = """#!/usr/bin/env bash -# Generated by node_repositories.bzl -# Immediately exit if any command fails. -set -e -unset YARN_IGNORE_PATH -{get_script_dir} -"$SCRIPT_DIR/{node}" "$SCRIPT_DIR/{script}" "$@" -""".format( - get_script_dir = GET_SCRIPT_DIR, - node = paths.relativize(node_entry, "bin"), - script = yarn_script_relative, - ), - executable = True, - ) - else: - # Yarn entry point - repository_ctx.file( - "bin/yarn.cmd", - content = """@echo off -SET SCRIPT_DIR=%~dp0 -SET "YARN_IGNORE_PATH=" -"%SCRIPT_DIR%\\{node}" "%SCRIPT_DIR%\\{script}" %* -""".format( - node = paths.relativize(node_entry, "bin"), - script = yarn_script_relative, - ), - executable = True, - ) - # Base BUILD file for this repository build_content = """# Generated by node_repositories.bzl package(default_visibility = ["//visibility:public"]) @@ -524,23 +338,16 @@ exports_files([ "run_npm.template", "{node_entry}", "{npm_entry}", - "{yarn_entry}", ]) alias(name = "node_bin", actual = "{node_bin_label}") alias(name = "npm_bin", actual = "{npm_bin_label}") alias(name = "npx_bin", actual = "{npx_bin_label}") -alias(name = "yarn_bin", actual = "{yarn_bin_label}") alias(name = "node", actual = "{node_entry}") alias(name = "npm", actual = "{npm_entry}") -alias(name = "yarn", actual = "{yarn_entry}") filegroup( name = "node_files", srcs = [":node", ":node_bin"], ) -filegroup( - name = "yarn_files", - srcs = {yarn_files_glob}[":node_files"], -) filegroup( name = "npm_files", srcs = {npm_files_glob}[":node_files"], @@ -550,15 +357,11 @@ filegroup( npm_bin_export = "" if repository_ctx.attr.vendored_node else ("\n \"%s\"," % npm_bin), npx_bin_export = "" if repository_ctx.attr.vendored_node else ("\n \"%s\"," % npx_bin), npm_files_glob = "" if repository_ctx.attr.vendored_node else "glob([\"bin/nodejs/**\"]) + ", - yarn_bin_export = "" if repository_ctx.attr.vendored_yarn else ("\n \"%s\"," % yarn_bin), - yarn_files_glob = "" if repository_ctx.attr.vendored_yarn else "glob([\"bin/yarnpkg/**\"]) + ", node_bin_label = node_bin_label, npm_bin_label = npm_bin_label, npx_bin_label = npx_bin_label, - yarn_bin_label = yarn_bin_label, node_entry = node_entry, npm_entry = npm_entry, - yarn_entry = yarn_entry, ) # the platform attribute is only set when used from this file, not from build_bazel_rules_nodejs @@ -581,7 +384,6 @@ def _verify_version_is_valid(version): def _nodejs_repo_impl(repository_ctx): assert_node_exists_for_host(repository_ctx) _download_node(repository_ctx) - _download_yarn(repository_ctx) _prepare_node(repository_ctx) node_repositories = repository_rule( diff --git a/nodejs/yarn_repositories.bzl b/nodejs/yarn_repositories.bzl new file mode 100644 index 0000000000..f104bd9a33 --- /dev/null +++ b/nodejs/yarn_repositories.bzl @@ -0,0 +1,217 @@ +"Repository rule to fetch the Yarn package manager" + +load("//nodejs/private:os_name.bzl", "is_windows_os") +load("//nodejs/private:node_labels.bzl", "get_node_label") +load("//nodejs/private:yarn_versions.bzl", "YARN_VERSIONS") +load("//nodejs:repositories.bzl", "GET_SCRIPT_DIR") +load("//third_party/github.com/bazelbuild/bazel-skylib:lib/paths.bzl", "paths") + +_DOC = """Repository rule to fetch the yarnpkg.com package manager. + +Note, the recommended name is "yarn". If you choose a different name, you'll have to override the +`yarn` attribute in your `yarn_install` rule to point to your `yarn.js` file. + +## Custom Yarn versions + +To specify custom Yarn versions, use the `yarn_releases` attribute + +```python +yarn_repositories( + yarn_releases = { + "1.12.1": ("yarn-v1.12.1.tar.gz", "yarn-v1.12.1", "09bea8f4ec41e9079fa03093d3b2db7ac5c5331852236d63815f8df42b3ba88d"), + }, +) +``` + +Like `node_urls`, the `yarn_urls` attribute can be used to provide a list of custom URLs to use to download yarn + +```python +yarn_repositories( + yarn_releases = { + "1.12.1": ("yarn-v1.12.1.tar.gz", "yarn-v1.12.1", "09bea8f4ec41e9079fa03093d3b2db7ac5c5331852236d63815f8df42b3ba88d"), + }, + yarn_version = "1.12.1", + yarn_urls = [ + "https://github.com/yarnpkg/yarn/releases/download/v{version}/{filename}", + ], +) +``` + +Will download yarn from https://github.com/yarnpkg/yarn/releases/download/v1.2.1/yarn-v1.12.1.tar.gz +and expect the file to have sha256sum `09bea8f4ec41e9079fa03093d3b2db7ac5c5331852236d63815f8df42b3ba88d`. + +If you don't use Yarn at all, you can skip downloading it by setting `yarn_urls = []`. + +## Vendored yarn + +You can vendor the `yarn.js` file into your repo. In this case, don't call `yarn_repositories` at all. +Just pass the label of your vendored file to the `yarn` attribute of `yarn_install`. +""" + +YARN_EXTRACT_DIR = "bin/yarnpkg" + +def _download_yarn(repository_ctx): + """Used to download a yarn tool package. + + Args: + repository_ctx: The repository rule context + """ + yarn_urls = repository_ctx.attr.yarn_urls + + # If there are no URLs to download yarn, skip the download + if not len(yarn_urls): + repository_ctx.file("yarn_info", content = "# no yarn urls") + return + + yarn_version = repository_ctx.attr.yarn_version + yarn_releases = repository_ctx.attr.yarn_releases + + # We insert our default value here, not on the attribute's default, so it isn't documented. + # The size of YARN_VERSIONS constant is huge and not useful to document. + if not yarn_releases.items(): + yarn_releases = YARN_VERSIONS + + if yarn_version in yarn_releases: + filename, strip_prefix, sha256 = yarn_releases[yarn_version] + else: + fail("Unknown Yarn version %s" % yarn_version) + + urls = [url.format(version = yarn_version, filename = filename) for url in yarn_urls] + + auth = {} + for url in urls: + auth[url] = repository_ctx.attr.yarn_download_auth + + repository_ctx.download_and_extract( + auth = auth, + url = urls, + output = YARN_EXTRACT_DIR, + stripPrefix = strip_prefix, + sha256 = sha256, + ) + + repository_ctx.file("yarn_info", content = """# filename: {filename} +# strip_prefix: {strip_prefix} +# sha256: {sha256} +""".format( + filename = filename, + strip_prefix = strip_prefix, + sha256 = sha256, + )) + +def _yarn_repositories_impl(repository_ctx): + _download_yarn(repository_ctx) + is_windows = is_windows_os(repository_ctx) + entry_ext = ".cmd" if is_windows else "" + + yarn_path = YARN_EXTRACT_DIR + yarn_package = YARN_EXTRACT_DIR + + # Use the yarn.js script as the bin for osx & linux so there are no symlink issues with `%s/bin/npm` + yarn_bin = ("%s/bin/yarn.js" % yarn_path) if not is_windows else ("%s/bin/yarn.cmd" % yarn_path) + yarn_bin_label = ("%s/bin/yarn.js" % yarn_package) if not is_windows else ("%s/bin/yarn.cmd" % yarn_package) + yarn_script = "%s/bin/yarn.js" % yarn_path + yarn_entry = "bin/yarn%s" % entry_ext + yarn_script_relative = paths.relativize(yarn_script, "bin") + node_label = get_node_label(repository_ctx) + + # The entry points for yarn for osx/linux and windows. + # Runs yarn using appropriate node entry point. + # Unset YARN_IGNORE_PATH before calling yarn incase it is set so that + # .yarnrc yarn-path is followed if set. This is for the case when calling + # bazel from yarn with `yarn bazel ...` and yarn follows yarn-path in + # .yarnrc it will set YARN_IGNORE_PATH=1 which will prevent the bazel + # call into yarn from also following the yarn-path as desired. + if not is_windows: + # Yarn entry point + repository_ctx.file( + "bin/yarn", + content = """#!/usr/bin/env bash +# Generated by node_repositories.bzl +# Immediately exit if any command fails. +set -e +unset YARN_IGNORE_PATH +{get_script_dir} +"{node}" "$SCRIPT_DIR/{script}" "$@" +""".format( + get_script_dir = GET_SCRIPT_DIR, + node = repository_ctx.path(node_label), + script = yarn_script_relative, + ), + executable = True, + ) + else: + # Yarn entry point + repository_ctx.file( + "bin/yarn.cmd", + content = """@echo off +SET SCRIPT_DIR=%~dp0 +SET "YARN_IGNORE_PATH=" +"{node}" "%SCRIPT_DIR%\\{script}" %* +""".format( + node = repository_ctx.path(node_label), + script = yarn_script_relative, + ), + executable = True, + ) + + # Base BUILD file for this repository + build_content = """# Generated by node_repositories.bzl +package(default_visibility = ["//visibility:public"]) +exports_files(["{yarn_entry}"]) +alias(name = "yarn_bin", actual = "{yarn_bin_label}") +alias(name = "yarn", actual = "{yarn_entry}") +filegroup( + name = "yarn_files", + srcs = {yarn_files_glob}, +) +""".format( + yarn_bin_export = ("\n \"%s\"," % yarn_bin), + yarn_files_glob = "glob([\"bin/yarnpkg/**\"])", + yarn_bin_label = yarn_bin_label, + yarn_entry = yarn_entry, + ) + + repository_ctx.file("BUILD.bazel", content = build_content) + +yarn_repositories = repository_rule( + implementation = _yarn_repositories_impl, + attrs = { + "node_repository": attr.string( + default = "nodejs", + doc = """The basename for a nodejs toolchain to use for running yarn. + Usually this is the value of the `name` attribute given to a nodejs_register_toolchains call in WORKSPACE""", + ), + "yarn_download_auth": attr.string_dict( + default = {}, + doc = """auth to use for all url requests + Example: {\"type\": \"basic\", \"login\": \"\", \"password\": \"\" } + """, + ), + "yarn_releases": attr.string_list_dict( + doc = """Custom list of yarn releases to use. + + Dictionary mapping Yarn versions to their corresponding (filename, strip_prefix, sha256) tuples. + + By default, if this attribute has no items, we'll use a list of all public releases which + is periodically mirrored to rules_nodejs. + """, + ), + "yarn_urls": attr.string_list( + default = [ + "https://github.com/yarnpkg/yarn/releases/download/v{version}/{filename}", + ], + doc = """custom list of URLs to use to download Yarn + + Each entry is a template using `yarn_version` and `yarn_releases` in the substitutions. + + If this list is empty, we won't download yarn at all. + """, + ), + "yarn_version": attr.string( + doc = "the specific version of Yarn to install", + default = "1.22.11", + ), + }, + doc = _DOC, +)