Skip to content

Commit

Permalink
feat(builtin): expose @npm//foo__all_files filegroup that includes …
Browse files Browse the repository at this point in the history 
…all files in the npm package (#1600)

Including files with spaces & unicode characters & files that have been filtered out by the `included_files` attribute. This filegroup cannot be used in runfiles because of Bazel issue bazelbuild/bazel#4327, but it can be used for other purposes such as the srcs input to a pkg_tar for generating a tar of an npm package pulled down with yarn_install or npm_install.
  • Loading branch information
@gregmagolan
gregmagolan authored Feb 5, 2020
1 parent 42bdb9c commit 8d77827
Show file tree
Hide file tree
Showing 19 changed files with 362 additions and 261 deletions.
10 changes: 10 additions & 0 deletions WORKSPACE
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,16 @@ yarn_install(

yarn_install(
name = "fine_grained_goldens",
included_files = [
"",
".js",
".jst",
".ts",
".map",
".d.ts",
".json",
".proto",
],
manual_build_file_contents = """
filegroup(
name = "golden_files",
Expand Down
2 changes: 2 additions & 0 deletions internal/node/node.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,8 @@ def _nodejs_binary_impl(ctx):
node_modules_manifest = write_node_modules_manifest(ctx)
node_modules_depsets = []
node_modules_depsets.append(depset(ctx.files.node_modules))
if NpmPackageInfo in ctx.attr.node_modules:
node_modules_depsets.append(ctx.attr.node_modules[NpmPackageInfo].sources)

# Also include files from npm fine grained deps as inputs.
# These deps are identified by the NpmPackageInfo provider.
Expand Down
168 changes: 85 additions & 83 deletions internal/npm_install/generate_build_file.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,17 +161,12 @@ for installation instructions.`);
* Generates the root BUILD file.
*/
function generateRootBuildFile(pkgs) {
let exportsStarlark = '';
pkgs.forEach(pkg => {
pkg._files.forEach(f => {
exportsStarlark += ` "node_modules/${pkg._dir}/${f}",
`;
});
});
let filesStarlark = '';
let pkgFilesStarlark = '';
if (pkgs.length) {
const list = pkgs.map(pkg => `"//${pkg._dir}:${pkg._name}__all_files",`).join('\n ');
filesStarlark = `
const list = pkgs.map(pkg => `"//${pkg._dir}:${pkg._name}__files",
"//${pkg._dir}:${pkg._name}__nested_node_modules",`)
.join('\n ');
pkgFilesStarlark = `
# direct sources listed for strict deps support
srcs = [
${list}
Expand All @@ -186,6 +181,13 @@ for installation instructions.`);
${list}
],`;
}
let exportsStarlark = '';
pkgs.forEach(pkg => {
pkg._files.forEach(f => {
exportsStarlark += ` "node_modules/${pkg._dir}/${f}",
`;
});
});
let buildFile = BUILD_FILE_HEADER +
`load("@build_bazel_rules_nodejs//internal/npm_install:node_module_library.bzl", "node_module_library")
Expand All @@ -197,7 +199,7 @@ ${exportsStarlark}])
# there are many files in target.
# See https://github.com/bazelbuild/bazel/issues/5153.
node_module_library(
name = "node_modules",${filesStarlark}${depsStarlark}
name = "node_modules",${pkgFilesStarlark}${depsStarlark}
)
`;
Expand Down Expand Up @@ -390,9 +392,6 @@ def _maybe(repo_rule, name, **kwargs):
}
return isDirectory ? files.concat(listFiles(rootDir, relPath)) : files.concat(relPath);
}, [])
// Files with spaces (\x20) or unicode characters (<\x20 && >\x7E) are not allowed in
// Bazel runfiles. See https://github.com/bazelbuild/bazel/issues/4327
.filter(f => !/[^\x21-\x7E]/.test(f))
// We return a sorted array so that the order of files
// is the same regardless of platform
.sort();
Expand Down Expand Up @@ -480,6 +479,10 @@ def _maybe(repo_rule, name, **kwargs):
pkg._isNested = /\/node_modules\//.test(p);
// List all the files in the npm package for later use
pkg._files = listFiles(p);
// The subset of files that are valid in runfiles.
// Files with spaces (\x20) or unicode characters (<\x20 && >\x7E) are not allowed in
// Bazel runfiles. See https://github.com/bazelbuild/bazel/issues/4327
pkg._runfiles = pkg._files.filter((f) => !/[^\x21-\x7E]/.test(f));
// Initialize _dependencies to an empty array
// which is later filled with the flattened dependency list
pkg._dependencies = [];
Expand Down Expand Up @@ -686,10 +689,11 @@ def _maybe(repo_rule, name, **kwargs):
*/
function printJson(pkg) {
// Clone and modify _dependencies to avoid circular issues when JSONifying
// & delete _files array
// & delete _files & _runfiles arrays
const cloned = Object.assign({}, pkg);
cloned._dependencies = pkg._dependencies.map(dep => dep._dir);
delete cloned._files;
delete cloned._runfiles;
return JSON.stringify(cloned, null, 2).split('\n').map(line => `# ${line}`).join('\n');
}
/**
Expand Down Expand Up @@ -748,15 +752,6 @@ def _maybe(repo_rule, name, **kwargs):
return false;
});
}
/**
* If the package is in the Angular package format returns list
* of package files that end with `.umd.js`, `.ngfactory.js` and `.ngsummary.js`.
*/
function getNgApfScripts(pkg) {
return isNgApfPackage(pkg) ?
filterFiles(pkg._files, ['.umd.js', '.ngfactory.js', '.ngsummary.js']) :
[];
}
/**
* Looks for a file within a package and returns it if found.
*/
Expand All @@ -773,91 +768,98 @@ def _maybe(repo_rule, name, **kwargs):
* Given a pkg, return the skylark `node_module_library` targets for the package.
*/
function printPackage(pkg) {
const sources = filterFiles(pkg._files, INCLUDED_FILES);
const files = sources.filter((f) => !f.startsWith('node_modules/'));
const nestedNodeModules = sources.filter((f) => f.startsWith('node_modules/'));
const dtsSources = filterFiles(pkg._files, ['.d.ts']);
// TODO(gmagolan): add UMD & AMD scripts to scripts even if not an APF package _but_ only if they
// are named?
const namedSources = getNgApfScripts(pkg);
const deps = [pkg].concat(pkg._dependencies.filter(dep => dep !== pkg && !dep._isNested));
let namedSourcesStarlark = '';
if (namedSources.length) {
namedSourcesStarlark = `
# subset of srcs that are javascript named-UMD or named-AMD scripts
named_module_srcs = [
${namedSources.map((f) => `"//:node_modules/${pkg._dir}/${f}",`).join('\n ')}
],`;
}
let filesStarlark = '';
if (files.length) {
filesStarlark = `
# ${pkg._dir} package files (and files in nested node_modules)
srcs = [
function starlarkFiles(attr, files, comment = '') {
return `
${comment ? comment + '\n ' : ''}${attr} = [
${files.map((f) => `"//:node_modules/${pkg._dir}/${f}",`).join('\n ')}
],`;
}
let nestedNodeModulesStarlark = '';
if (nestedNodeModules.length) {
nestedNodeModulesStarlark = `
# ${pkg._dir} package files (and files in nested node_modules)
srcs = [
${nestedNodeModules.map((f) => `"//:node_modules/${pkg._dir}/${f}",`)
.join('\n ')}
],`;
}
let depsStarlark = '';
if (deps.length) {
const list = deps.map(dep => `"//${dep._dir}:${dep._name}__contents",`).join('\n ');
depsStarlark = `
# flattened list of direct and transitive dependencies hoisted to root by the package manager
deps = [
${list}
],`;
}
let dtsStarlark = '';
if (dtsSources.length) {
dtsStarlark = `
# ${pkg._dir} package declaration files (and declaration files in nested node_modules)
srcs = [
${dtsSources.map(f => `"//:node_modules/${pkg._dir}/${f}",`).join('\n ')}
],`;
}
const includedRunfiles = filterFiles(pkg._runfiles, INCLUDED_FILES);
// Files that are part of the npm package not including its nested node_modules
// (filtered by the 'included_files' attribute)
const pkgFiles = includedRunfiles.filter((f) => !f.startsWith('node_modules/'));
const pkgFilesStarlark = pkgFiles.length ? starlarkFiles('srcs', pkgFiles) : '';
// Files that are in the npm package's nested node_modules
// (filtered by the 'included_files' attribute)
const nestedNodeModules = includedRunfiles.filter((f) => f.startsWith('node_modules/'));
const nestedNodeModulesStarlark = nestedNodeModules.length ? starlarkFiles('srcs', nestedNodeModules) : '';
// Files that have been excluded from the ${pkg._name}__files target above because
// they are filtered out by 'included_files' or because they are not valid runfiles
// See https://github.com/bazelbuild/bazel/issues/4327.
const notPkgFiles = pkg._files.filter((f) => !f.startsWith('node_modules/') && !includedRunfiles.includes(f));
const notPkgFilesStarlark = notPkgFiles.length ? starlarkFiles('srcs', notPkgFiles) : '';
// If the package is in the Angular package format returns list
// of package files that end with `.umd.js`, `.ngfactory.js` and `.ngsummary.js`.
// TODO(gmagolan): add UMD & AMD scripts to scripts even if not an APF package _but_ only if they
// are named?
const namedSources = isNgApfPackage(pkg) ?
filterFiles(pkg._runfiles, ['.umd.js', '.ngfactory.js', '.ngsummary.js']) :
[];
const namedSourcesStarlark = namedSources.length ?
starlarkFiles('named_module_srcs', namedSources, '# subset of srcs that are javascript named-UMD or named-AMD scripts') :
'';
// Typings files that are part of the npm package not including nested node_modules
const dtsSources = filterFiles(pkg._runfiles, ['.d.ts']).filter((f) => !f.startsWith('node_modules/'));
const dtsStarlark = dtsSources.length ?
starlarkFiles('srcs', dtsSources, `# ${pkg._dir} package declaration files (and declaration files in nested node_modules)`) :
'';
// Flattened list of direct and transitive dependencies hoisted to root by the package manager
const deps = [pkg].concat(pkg._dependencies.filter(dep => dep !== pkg && !dep._isNested));
const depsStarlark = deps.map(dep => `"//${dep._dir}:${dep._name}__contents",`).join('\n ');
let result = `load("@build_bazel_rules_nodejs//internal/npm_install:node_module_library.bzl", "node_module_library")
# Generated targets for npm package "${pkg._dir}"
${printJson(pkg)}
# Files that are part of the npm package not including its nested node_modules
# (filtered by the 'included_files' attribute)
filegroup(
name = "${pkg._name}__files",${filesStarlark}
name = "${pkg._name}__files",${pkgFilesStarlark}
)
# Files that are in the npm package's nested node_modules
# (filtered by the 'included_files' attribute)
filegroup(
name = "${pkg._name}__nested_node_modules",${nestedNodeModulesStarlark}
visibility = ["//:__subpackages__"],
)
# Files that have been excluded from the ${pkg._name}__files target above because
# they are filtered out by 'included_files' or because they are not valid runfiles
# See https://github.com/bazelbuild/bazel/issues/4327.
filegroup(
name = "${pkg._name}__not_files",${notPkgFilesStarlark}
visibility = ["//visibility:private"],
)
# All of the files in the npm package including files that have been
# filtered out by 'included_files' or because they are not valid runfiles
# but not including nested node_modules.
filegroup(
name = "${pkg._name}__all_files",
srcs = [":${pkg._name}__files", ":${pkg._name}__nested_node_modules"],
visibility = ["//:__subpackages__"],
srcs = [":${pkg._name}__files", ":${pkg._name}__not_files"],
)
# The primary target for this package for use in rule deps
node_module_library(
name = "${pkg._name}",
# direct sources listed for strict deps support
srcs = [":${pkg._name}__all_files"],${depsStarlark}
srcs = [":${pkg._name}__files"],
# nested node_modules for this package plus flattened list of direct and transitive dependencies
# hoisted to root by the package manager
deps = [
${depsStarlark}
],
)
# ${pkg._name}__contents target is used as dep for main targets to prevent
# circular dependencies errors
# Target is used as dep for main targets to prevent circular dependencies errors
node_module_library(
name = "${pkg._name}__contents",
srcs = [":${pkg._name}__all_files"],${namedSourcesStarlark}
srcs = [":${pkg._name}__files", ":${pkg._name}__nested_node_modules"],${namedSourcesStarlark}
visibility = ["//:__subpackages__"],
)
# ${pkg._name}__typings is the subset of ${pkg._name}__contents that are declarations
# Typings files that are part of the npm package not including nested node_modules
node_module_library(
name = "${pkg._name}__typings",${dtsStarlark}
)
Expand Down Expand Up @@ -1013,10 +1015,10 @@ def ${name.replace(/-/g, '_')}_test(**kwargs):
});
// filter out duplicate deps
deps = [...pkgs, ...new Set(deps)];
let filesStarlark = '';
let pkgFilesStarlark = '';
if (deps.length) {
const list = deps.map(dep => `"//${dep._dir}:${dep._name}__all_files",`).join('\n ');
filesStarlark = `
const list = deps.map(dep => `"//${dep._dir}:${dep._name}__files",`).join('\n ');
pkgFilesStarlark = `
# direct sources listed for strict deps support
srcs = [
${list}
Expand All @@ -1035,7 +1037,7 @@ def ${name.replace(/-/g, '_')}_test(**kwargs):
# Generated target for npm scope ${scope}
node_module_library(
name = "${scope}",${filesStarlark}${depsStarlark}
name = "${scope}",${pkgFilesStarlark}${depsStarlark}
)
`;
Expand Down
Loading

0 comments on commit 8d77827

Please sign in to comment.