feat(builtin): first experimental rules for npm tarballs

This is not fully designed yet, so it's not included in public API. May be deleted at any time.
The newly added README explains what's going on.
Based on design: https://hackmd.io/gu2Nj0TKS068LKAf8KanuA

WORKSPACE

@@ -79,6 +79,19 @@ npm_install(
     package_lock_json = "//packages/node-patches:package-lock.json",
 )
 
+load("@build_bazel_rules_nodejs//internal/npm_tarballs:translate_package_lock.bzl", "translate_package_lock")
+
+# Translate our package.lock file from JSON to Starlark
+translate_package_lock(
+    name = "npm_node_patches_lock",
+    package_lock = "//packages/node-patches:package-lock.json",
+)
+
+load("@npm_node_patches_lock//:index.bzl", _npm_patches_repositories = "npm_repositories")
+
+# # Declare an external repository for each npm package fetchable by the lock file
+_npm_patches_repositories()
+
 npm_install(
     name = "angular_deps",
     package_json = "//packages/angular:package.json",

internal/common/download.bzl

@@ -0,0 +1,43 @@
+"Repository rule wrapper around Bazel's downloader"
+
+def _bazel_download(repository_ctx):
+    repository_ctx.file("BUILD.bazel", repository_ctx.attr.build_file_content)
+    repository_ctx.download(
+        output = repository_ctx.attr.output,
+        url = repository_ctx.attr.url,
+        integrity = repository_ctx.attr.integrity,
+    )
+
+bazel_download = repository_rule(
+    doc = """Utility to call Bazel downloader.
+
+    This is a simple pass-thru wrapper for Bazel's
+    [repository_ctx#download](https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#download)
+    function.
+    """,
+    implementation = _bazel_download,
+    attrs = {
+        "build_file_content": attr.string(
+            doc = "Content for the generated BUILD file.",
+            mandatory = True,
+        ),
+        "integrity": attr.string(
+            doc = """
+            Expected checksum of the file downloaded, in Subresource Integrity format.
+            This must match the checksum of the file downloaded.
+            It is a security risk to omit the checksum as remote files can change.
+            At best omitting this field will make your build non-hermetic.
+            It is optional to make development easier but should be set before shipping.
+            """,
+            mandatory = True,
+        ),
+        "output": attr.string(
+            doc = "path to the output file, relative to the repository directory",
+            mandatory = True,
+        ),
+        "url": attr.string_list(
+            doc = "List of mirror URLs referencing the same file.",
+            mandatory = True,
+        ),
+    },
+)

internal/npm_tarballs/BUILD.bazel

@@ -0,0 +1 @@
+# No bazel targets in this package

internal/npm_tarballs/README.md

@@ -0,0 +1,68 @@
+# npm_tarballs
+
+This is an expermental feature inspired by external package fetching in rules_go and others.
+
+See the design doc: https://hackmd.io/gu2Nj0TKS068LKAf8KanuA
+
+## Rules
+
+`translate_package_lock.bzl` takes a package-lock.json file and produces a Starlark representation of downloader rules for each package listed.
+
+Currently this is implemented only for npm v7 produced lockfiles (version 2 of the spec) but it could be ported to any other lockfile format.
+
+For example, for https://github.com/bazelbuild/rules_nodejs/blob/stable/packages/node-patches/package-lock.json we produce an `index.bzl` file like:
+
+```
+"Generated by package_lock.bzl from //packages/node-patches:package-lock.json"
+
+load("@build_bazel_rules_nodejs//internal/common:download.bzl", "bazel_download")
+
+def npm_repositories():
+    """Define external repositories to fetch each tarball individually from npm on-demand.
+    """
+
+# [...]
+
+    bazel_download(
+        name = "npm_typescript-3.5.3",
+        output = "typescript-3.5.3.tgz",
+        integrity = "sha512-ACzBtm/PhXBDId6a6sDJfroT2pOWt/oOnk4/dElG5G33ZL776N3Y6/6bKZJBFpd+b05F3Ct9qDjMeJmRWtE2/g==",
+        url = ["https://registry.npmjs.org/typescript/-/typescript-3.5.3.tgz"],
+        build_file_content = """"Generated by package_lock.bzl"
+
+load("@build_bazel_rules_nodejs//internal/npm_tarballs:npm_tarball.bzl", "npm_tarball")
+
+npm_tarball(
+    name = "npm_typescript-3.5.3",
+    src = "typescript-3.5.3.tgz",
+    package_name = "typescript",
+    deps = [],
+    visibility = ["//visibility:public"],
+)
+
+"""
+    )
+
+# [...]
+```
+
+This generated index.bzl can then be loaded in the WORKSPACE and the `npm_repositories` macro called.
+This then declares `bazel_download` rules that are themselves able to fetch packages on-demand.
+We also supply a BUILD file content for each of these packages, using a minimal `npm_tarball` rule that
+represents the location and dependencies of the downloaded .tgz file.
+
+In addition, we give some syntax sugar.
+In the repo produced by `translate_package_lock` we provide "catch-all" targets
+`//:dependencies` and `//:devDependencies` that depend on all tarballs so listed in the package-lock.json.
+For direct dependencies, we also produce a `//somepackage` target that aliases the version of `somepackage` depended on.
+In the above example, that means the user can dep on `@npm_repositories//typescript` rather than
+`@npm_typescript-3.5.3` because we know the package depends on version 3.5.3.
+
+## Future work
+
+So far the resulting tarballs aren't used by anything in rules_nodejs (nothing consumes `NpmTarballInfo`).
+In later work we'll explore what other rules might want to use the tarballs,
+such as a pnpm_install rule that uses pnpm semantics to just symlink things into a tree.
+Or maybe an npm_install rule, one for each package, that unpacks the tarballs and runs the postinstall logic on each.
+We believe some experimentation will be required to find a good path forward that uses the download-as-needed semantics here,
+while keeping most existing semantics of rules_nodejs rules working.

internal/npm_tarballs/npm_tarball.bzl

@@ -0,0 +1,64 @@
+"Info about npm tarball files"
+
+NpmTarballInfo = provider(
+    doc = "Describe tarballs downloaded from npm registry",
+    fields = {
+        "tarballs": "depset of needed tarballs to be able to npm install",
+    },
+)
+
+_DOC = """This rule is a simple reference to a file downloaded from npm.
+
+It is not meant to be used on its own, rather it is generated into BUILD files in external repos
+and its provider can then be referenced in actions by tools like pnpm that need to find the .tgz files.
+"""
+
+_ATTRS = {
+    "deps": attr.label_list(
+        doc = "Other npm_tarball rules for packages this one depends on",
+        providers = [NpmTarballInfo],
+    ),
+    "package_name": attr.string(
+        doc = "the name field from the package.json of the package this tarball contains",
+    ),
+    "src": attr.label(
+        doc = "The downloaded tarball",
+        allow_single_file = [".tgz"],
+    ),
+}
+
+def _npm_tarball(ctx):
+    # Allow aggregate rules like "all_dependencies" to have only deps but no tarball
+    if ctx.attr.src and not ctx.attr.package_name:
+        fail("when given a src, must also tell the package_name for it")
+    direct = []
+    direct_files = []
+    if ctx.attr.src:
+        direct = [struct(
+            package_name = ctx.attr.package_name,
+            tarball = ctx.file.src,
+        )]
+        direct_files = [ctx.file.src]
+
+    transitive = [d[NpmTarballInfo].tarballs for d in ctx.attr.deps]
+    transitive_files = []
+    for dset in transitive:
+        for info in dset.to_list():
+            transitive_files.append(info.tarball)
+    return [
+        NpmTarballInfo(tarballs = depset(
+            direct,
+            transitive = transitive,
+        )),
+        # For testing
+        OutputGroupInfo(
+            direct = direct_files,
+            transitive = transitive_files,
+        ),
+    ]
+
+npm_tarball = rule(
+    implementation = _npm_tarball,
+    attrs = _ATTRS,
+    doc = _DOC,
+)

internal/npm_tarballs/test/BUILD.bazel

@@ -0,0 +1,58 @@
+load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_test")
+
+# In normal usage, some other rule would extract the NpmTarballInfo
+# For testing we simply want to grab the file
+filegroup(
+    name = "get_typescript",
+    # the internal/node-patches package-lock.json depends on this version of typescript
+    srcs = ["@npm_typescript-3.5.3"],
+    output_group = "direct",
+)
+
+nodejs_test(
+    name = "test_some_package_fetched",
+    data = [":get_typescript"],
+    entry_point = "test_some_package_fetched.js",
+)
+
+filegroup(
+    name = "get_typescript_alias",
+    # Since typescript is a direct dependency, we can point to the version used by this package
+    # without having to specify (it's an alias)
+    srcs = ["@npm_node_patches_lock//typescript"],
+    output_group = "direct",
+)
+
+# Run the same test again but point to this filegroup to be sure the same typescript was there
+nodejs_test(
+    name = "test_alias",
+    data = [":get_typescript_alias"],
+    entry_point = "test_some_package_fetched.js",
+)
+
+filegroup(
+    name = "get_all_devdeps",
+    # Check that there's also a syntax-sugar for "all the devDependencies listed"
+    srcs = ["@npm_node_patches_lock//:devDependencies"],
+    output_group = "transitive",
+)
+
+# Run that same test again, typescript should be in here
+nodejs_test(
+    name = "test_alldevdeps",
+    data = [":get_all_devdeps"],
+    entry_point = "test_some_package_fetched.js",
+)
+
+filegroup(
+    name = "get_ansi-align",
+    # According to package-lock.json, it depends on [email protected]
+    srcs = ["@npm_ansi-align-3.0.0"],
+    output_group = "transitive",
+)
+
+nodejs_test(
+    name = "test_dependencies_available",
+    data = [":get_ansi-align"],
+    entry_point = "test_dependencies_available.js",
+)

internal/npm_tarballs/test/test_dependencies_available.js

@@ -0,0 +1,6 @@
+const assert = require('assert');
+const {existsSync} = require('fs');
+const runfiles = require(process.env['BAZEL_NODE_RUNFILES_HELPER']);
+const tarPath = runfiles.resolve('npm_string-width-3.1.0/string-width-3.1.0.tgz');
+
+assert.ok(existsSync(tarPath));

internal/npm_tarballs/test/test_some_package_fetched.js

@@ -0,0 +1,12 @@
+const assert = require('assert');
+const {existsSync, statSync} = require('fs');
+const runfiles = require(process.env['BAZEL_NODE_RUNFILES_HELPER']);
+const tarPath = runfiles.resolve('npm_typescript-3.5.3/typescript-3.5.3.tgz');
+
+assert.ok(existsSync(tarPath));
+
+// The size of https://www.npmjs.com/package/typescript/v/3.5.3
+expectedSize = 7960741;
+assert.strictEqual(
+    statSync(tarPath).size, expectedSize,
+    `Expected to download the typescript 3.5.3 release which is ${expectedSize} bytes`);