diff --git a/.circleci/config.yml b/.circleci/config.yml index d977c425bd..d1bc7c3758 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -204,7 +204,7 @@ jobs: - persist_to_workspace: root: ~/ paths: - - ./rules_nodejs/dist/npm_bazel_typescript + - ./rules_nodejs/dist/npm_bazel_typescript$* build_karma_package: <<: *job_defaults @@ -217,7 +217,7 @@ jobs: - persist_to_workspace: root: ~/ paths: - - ./rules_nodejs/dist/npm_bazel_karma + - ./rules_nodejs/dist/npm_bazel_karma$* build_jasmine_package: <<: *job_defaults @@ -230,7 +230,7 @@ jobs: - persist_to_workspace: root: ~/ paths: - - ./rules_nodejs/dist/npm_bazel_jasmine + - ./rules_nodejs/dist/npm_bazel_jasmine$* build_labs_package: <<: *job_defaults @@ -243,7 +243,7 @@ jobs: - persist_to_workspace: root: ~/ paths: - - ./rules_nodejs/dist/npm_bazel_labs + - ./rules_nodejs/dist/npm_bazel_labs$* test_packages: <<: *job_defaults diff --git a/e2e/define_var/WORKSPACE b/e2e/define_var/WORKSPACE index bda44fd738..b9e04773e9 100644 --- a/e2e/define_var/WORKSPACE +++ b/e2e/define_var/WORKSPACE @@ -1,8 +1,19 @@ -workspace(name = "examples_define_var") +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +workspace(name = "e2e_define_var") -# In your code, you'd fetch this repository with an `http_archive` call. -# We do this local repository only because this example lives in the same -# repository with the rules_nodejs code and we want to test them together. local_repository( name = "build_bazel_rules_nodejs", path = "../../dist/build_bazel_rules_nodejs/release", diff --git a/e2e/tsconfig_extends/WORKSPACE b/e2e/tsconfig_extends/WORKSPACE index 13bf4fd320..027cba44aa 100644 --- a/e2e/tsconfig_extends/WORKSPACE +++ b/e2e/tsconfig_extends/WORKSPACE @@ -1,8 +1,19 @@ -workspace(name = "examples_tsconfig_extends") +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +workspace(name = "e2e_tsconfig_extends") -# In your code, you'd fetch this repository with an `http_archive` call. -# We do this local repository only because this example lives in the same -# repository with the rules_nodejs code and we want to test them together. local_repository( name = "build_bazel_rules_nodejs", path = "../../dist/build_bazel_rules_nodejs/release", @@ -10,10 +21,6 @@ local_repository( load("@build_bazel_rules_nodejs//:defs.bzl", "yarn_install") -# This runs yarn install, then our generate_build_file.js to create BUILD files -# inside the resulting node_modules directory. -# The name "npm" here means the resulting modules are referenced like -# @npm//jasmine yarn_install( name = "npm", package_json = "//:package.json", diff --git a/examples/webapp/package.json b/examples/webapp/package.json index d584594456..ada87a9b9b 100644 --- a/examples/webapp/package.json +++ b/examples/webapp/package.json @@ -4,4 +4,4 @@ "scripts": { "test": "bazel build :all" } -} \ No newline at end of file +} diff --git a/internal/npm_install/npm_install.bzl b/internal/npm_install/npm_install.bzl index 71132f844f..9287b5cfa0 100644 --- a/internal/npm_install/npm_install.bzl +++ b/internal/npm_install/npm_install.bzl @@ -178,7 +178,10 @@ cd "{root}" && "{npm}" {npm_args} _add_data_dependencies(repository_ctx) _add_scripts(repository_ctx) - result = repository_ctx.execute([node, "process_package_json.js", ",".join(repository_ctx.attr.exclude_packages)]) + result = repository_ctx.execute( + [node, "process_package_json.js", "npm", ",".join(repository_ctx.attr.exclude_packages)], + quiet = repository_ctx.attr.quiet, + ) if result.return_code: fail("node failed: \nSTDOUT:\n%s\nSTDERR:\n%s" % (result.stdout, result.stderr)) @@ -249,7 +252,10 @@ def _yarn_install_impl(repository_ctx): _add_data_dependencies(repository_ctx) _add_scripts(repository_ctx) - result = repository_ctx.execute([node, "process_package_json.js", ",".join(repository_ctx.attr.exclude_packages)]) + result = repository_ctx.execute( + [node, "process_package_json.js", "yarn", ",".join(repository_ctx.attr.exclude_packages)], + quiet = repository_ctx.attr.quiet, + ) if result.return_code: fail("node failed: \nSTDOUT:\n%s\nSTDERR:\n%s" % (result.stdout, result.stderr)) diff --git a/internal/npm_install/process_package_json.js b/internal/npm_install/process_package_json.js index f4f38de19e..86d0575416 100644 --- a/internal/npm_install/process_package_json.js +++ b/internal/npm_install/process_package_json.js @@ -23,10 +23,13 @@ 'use strict'; const fs = require('fs'); -const path = require('path'); +const child_process = require('child_process'); + +const DEBUG = false; const args = process.argv.slice(2); -const removePackages = args[0] ? args[0].split(',') : []; +const packageManager = args[0]; +const excludePackages = args[1] ? args[1].split(',') : []; if (require.main === module) { main(); @@ -36,9 +39,26 @@ if (require.main === module) { * Main entrypoint. */ function main() { + const isYarn = (packageManager === 'yarn'); + const pkg = JSON.parse(fs.readFileSync('_package.json', {encoding: 'utf8'})); - removePackages.forEach(p => { + if (DEBUG) console.error(`Pre-processing package.json`); + + removeExcludedPackages(pkg); + + if (isYarn) { + // Work-around for https://github.com/yarnpkg/yarn/issues/2165 + // Note: there is no equivalent npm functionality to clean out individual packages + // from the npm cache. + clearYarnFilePathCaches(pkg); + } + + fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2)); +} + +function removeExcludedPackages(pkg) { + excludePackages.forEach(p => { if (pkg.dependencies) { delete pkg.dependencies[p]; } @@ -52,8 +72,45 @@ function main() { delete pkg.optionalDependencies[p]; } }); +} - fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2)); +/** + * Runs `yarn cache clean` for all packages that have `file://` URIs. + * Work-around for https://github.com/yarnpkg/yarn/issues/2165. + */ +function clearYarnFilePathCaches(pkg) { + const fileRegex = /^file\:\/\//i; + const clearPackages = []; + + if (pkg.dependencies) { + Object.keys(pkg.dependencies).forEach(p => { + if (pkg.dependencies[p].match(fileRegex)) { + clearPackages.push(p); + } + }); + } + if (pkg.devDependencies) { + Object.keys(pkg.devDependencies).forEach(p => { + if (pkg.devDependencies[p].match(fileRegex)) { + clearPackages.push(p); + } + }); + } + if (pkg.optionalDependencies) { + Object.keys(pkg.optionalDependencies).forEach(p => { + if (pkg.optionalDependencies[p].match(fileRegex)) { + clearPackages.push(p); + } + }); + } + + if (clearPackages.length) { + if (DEBUG) console.error(`Cleaning packages from yarn cache: ${clearPackages.join(' ')}`); + + child_process.execFileSync( + 'yarn', ['cache', 'clean'].concat(clearPackages), + {stdio: [process.stdin, process.stdout, process.stderr]}); + } } module.exports = {main}; diff --git a/scripts/build_all.sh b/scripts/build_all.sh new file mode 100755 index 0000000000..34df084fa1 --- /dev/null +++ b/scripts/build_all.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -eu -o pipefail +# -e: exits if a command fails +# -u: errors if an variable is referenced before being set +# -o pipefail: causes a pipeline to produce a failure return code if any command errors + +readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd) + +${RULES_NODEJS_DIR}/scripts/build_release.sh +${RULES_NODEJS_DIR}/scripts/build_packages_all.sh diff --git a/scripts/build_packages.sh b/scripts/build_packages.sh index 0f943b55a2..b763a3c752 100755 --- a/scripts/build_packages.sh +++ b/scripts/build_packages.sh @@ -15,7 +15,8 @@ echo_and_run() { echo "+ $@" ; "$@" ; } for package in ${PACKAGES[@]} ; do ( - readonly DEST_DIR="${DIST_DIR}/npm_bazel_${package}" + readonly DEST_DIR_BASE="${DIST_DIR}/npm_bazel_${package}" + readonly DEST_DIR="${DEST_DIR_BASE}\$${RANDOM}" # Build npm package cd "${PACKAGES_DIR}/${package}" @@ -25,7 +26,7 @@ for package in ${PACKAGES[@]} ; do # Copy the npm_package to /dist echo "Copying npm package to ${DEST_DIR}" - rm -rf ${DEST_DIR} + rm -rf ${DEST_DIR_BASE}\$* mkdir -p ${DIST_DIR} readonly BAZEL_BIN=$(bazel info bazel-bin) echo_and_run cp -R "${BAZEL_BIN}/npm_package" ${DEST_DIR} diff --git a/scripts/build_packages_all.sh b/scripts/build_packages_all.sh index 7ed475e024..97bd4d0ac2 100755 --- a/scripts/build_packages_all.sh +++ b/scripts/build_packages_all.sh @@ -8,7 +8,5 @@ set -eu -o pipefail readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd) source "${RULES_NODEJS_DIR}/scripts/packages.sh" -echo_and_run() { echo "+ $@" ; "$@" ; } - ${RULES_NODEJS_DIR}/scripts/build_packages.sh ${PACKAGES[@]} diff --git a/scripts/check_deps.sh b/scripts/check_deps.sh index 9e3c95cedd..09a2d929c4 100755 --- a/scripts/check_deps.sh +++ b/scripts/check_deps.sh @@ -23,7 +23,8 @@ for dep in ${DEPS} ; do ALL_GOOD=0 fi else - if [[ ! -d "${RULES_NODEJS_DIR}/dist/npm_bazel_${dep}" ]] ; then + results=$(ls -d ${RULES_NODEJS_DIR}/dist/npm_bazel_${dep}\$* 2> /dev/null || :) + if [[ -z "${results}" ]] ; then echo "ERROR: You must first run 'yarn build_packages ${dep}' or 'yarn build_packages_all'"; ALL_GOOD=0 fi diff --git a/scripts/clean_all.sh b/scripts/clean_all.sh index 195926571a..0dd4d40241 100755 --- a/scripts/clean_all.sh +++ b/scripts/clean_all.sh @@ -17,19 +17,21 @@ echo_and_run rm -rf ./internal/npm_install/test/package/node_modules echo_and_run bazel clean --expunge -for rootDir in examples e2e internal/e2e packages ; do - ( - cd ${rootDir} - for subDir in $(ls) ; do - [[ -d "${subDir}" ]] || continue - ( - cd ${subDir} - if [[ -e 'WORKSPACE' ]] ; then - printf "\n\nCleaning /${rootDir}/${subDir}\n" - echo_and_run bazel clean --expunge - echo_and_run rm -rf node_modules - fi - ) - done - ) -done +${RULES_NODEJS_DIR}/scripts/clean_e2e_all.sh +${RULES_NODEJS_DIR}/scripts/clean_examples_all.sh +${RULES_NODEJS_DIR}/scripts/clean_packages_all.sh + +( + cd internal/e2e + for subDir in $(ls) ; do + [[ -d "${subDir}" ]] || continue + ( + cd ${subDir} + if [[ -e 'WORKSPACE' ]] ; then + printf "\n\nCleaning /internal/e2e/${subDir}\n" + echo_and_run bazel clean --expunge + echo_and_run rm -rf node_modules + fi + ) + done +) diff --git a/scripts/clean_e2e.sh b/scripts/clean_e2e.sh index ff53cbc4bc..a5bc104caa 100755 --- a/scripts/clean_e2e.sh +++ b/scripts/clean_e2e.sh @@ -17,6 +17,7 @@ for e2eTest in ${E2E_TESTS[@]} ; do # Clean e2e test cd "${E2E_DIR}/${e2eTest}" printf "\n\nCleaning e2e test ${e2eTest}\n" + ${RULES_NODEJS_DIR}/scripts/unlink_deps.sh echo_and_run bazel clean --expunge echo_and_run rm -rf node_modules ) diff --git a/scripts/clean_e2e_all.sh b/scripts/clean_e2e_all.sh index 02e244325f..117f4e9ab0 100755 --- a/scripts/clean_e2e_all.sh +++ b/scripts/clean_e2e_all.sh @@ -8,6 +8,6 @@ set -eu -o pipefail readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd) readonly E2E_DIR="${RULES_NODEJS_DIR}/e2e" -readonly E2E=$(ls ${E2E_DIR}) +readonly E2E=$(ls -l ${E2E_DIR} | grep "^d" | awk -F" " '{print $9}') ${RULES_NODEJS_DIR}/scripts/clean_e2e.sh ${E2E[@]} diff --git a/scripts/clean_examples.sh b/scripts/clean_examples.sh index 207bb44c32..305f280408 100755 --- a/scripts/clean_examples.sh +++ b/scripts/clean_examples.sh @@ -17,6 +17,7 @@ for example in ${EXAMPLES[@]} ; do # Clean example cd "${EXAMPLES_DIR}/${example}" printf "\n\nCleaning example ${example}\n" + ${RULES_NODEJS_DIR}/scripts/unlink_deps.sh echo_and_run bazel clean --expunge echo_and_run rm -rf node_modules ) diff --git a/scripts/clean_examples_all.sh b/scripts/clean_examples_all.sh index 227e54c7ae..95017db855 100755 --- a/scripts/clean_examples_all.sh +++ b/scripts/clean_examples_all.sh @@ -8,6 +8,6 @@ set -eu -o pipefail readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd) readonly EXAMPLES_DIR="${RULES_NODEJS_DIR}/examples" -readonly EXAMPLES=$(ls ${EXAMPLES_DIR}) +readonly EXAMPLES=$(ls -l ${EXAMPLES_DIR} | grep "^d" | awk -F" " '{print $9}') ${RULES_NODEJS_DIR}/scripts/clean_examples.sh ${EXAMPLES[@]} diff --git a/scripts/clean_packages.sh b/scripts/clean_packages.sh index 8cbc3e56e8..169f50e348 100755 --- a/scripts/clean_packages.sh +++ b/scripts/clean_packages.sh @@ -17,6 +17,7 @@ for package in ${PACKAGES[@]} ; do # Clean package cd "${PACKAGES_DIR}/${package}" printf "\n\nCleaning package ${package}\n" + ${RULES_NODEJS_DIR}/scripts/unlink_deps.sh echo_and_run bazel clean --expunge echo_and_run rm -rf node_modules ) diff --git a/scripts/clean_packages_all.sh b/scripts/clean_packages_all.sh index b20cc8bf33..9481bd86dc 100755 --- a/scripts/clean_packages_all.sh +++ b/scripts/clean_packages_all.sh @@ -6,8 +6,7 @@ set -eu -o pipefail # -o pipefail: causes a pipeline to produce a failure return code if any command errors readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd) -readonly PACKAGES_DIR="${RULES_NODEJS_DIR}/package" +source "${RULES_NODEJS_DIR}/scripts/packages.sh" -readonly PACKAGES=$(ls ${PACKAGES_DIR}) +${RULES_NODEJS_DIR}/scripts/clean_packages.sh ${PACKAGES[@]} -${RULES_NODEJS_DIR}/scripts/clean_package.sh ${PACKAGES[@]} diff --git a/scripts/clean_yarn_cache_selectively.sh b/scripts/clean_yarn_cache_selectively.sh new file mode 100755 index 0000000000..2b98425454 --- /dev/null +++ b/scripts/clean_yarn_cache_selectively.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -eu -o pipefail +# -e: exits if a command fails +# -u: errors if an variable is referenced before being set +# -o pipefail: causes a pipeline to produce a failure return code if any command errors + +readonly YARN_CACHE_ROOT=$(dirname $(yarn cache dir)) +readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd) +source "${RULES_NODEJS_DIR}/scripts/packages.sh" + +echo_and_run() { echo "+ $@" ; "$@" ; } + +echo "yarn cache root: ${YARN_CACHE_ROOT}" + +for package in ${PACKAGES[@]} ; do + echo_and_run yarn cache clean @bazel/${package} +done + +# Also clean cache for different versions of yarn since the locally installed +# yarn may have a different cache version from yarn version used by Bazel +for package in ${PACKAGES[@]} ; do + echo_and_run rm -rf ${YARN_CACHE_ROOT}/*/npm-@bazel-${package}-* +done diff --git a/scripts/link_deps.sh b/scripts/link_deps.sh index c463cb44a4..2145350f38 100755 --- a/scripts/link_deps.sh +++ b/scripts/link_deps.sh @@ -7,6 +7,8 @@ set -eu -o pipefail readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd) +echo_and_run() { echo "+ $@" ; "$@" ; } + # sedi makes `sed -i` work on both OSX & Linux # See https://stackoverflow.com/questions/2320564/i-need-my-sed-i-command-for-in-place-editing-to-work-with-both-gnu-sed-and-bsd sedi () { @@ -18,26 +20,39 @@ sedi () { sed "${sedi[@]}" "$@" } -# Replaces "bazel://@npm_bazel_foobar//:npm_package" with absolute -# path to generated npm package under /dist/npm_bazel_foobar -sedi "s#\"bazel://@\([a-z_]*\)//:npm_package\"#\"file://${RULES_NODEJS_DIR}/dist/\1\"#" package.json +${RULES_NODEJS_DIR}/scripts/unlink_deps.sh DEPS=() +PACKAGES=() -# Check for WORKSPACE dependency on release +# Check for WORKSPACE dependency on release dist LINES=$(grep "/dist/build_bazel_rules_nodejs/release\"" WORKSPACE || echo "") if [[ "${LINES}" ]] ; then DEPS+=( release ) fi -# Check for dependencies in package.json -LINES=$(egrep -oh "/dist/npm_bazel_([a-z_]+)\"" package.json || echo "") +# Check for bazel://@npm_bazel_foobar dependencies in package.json +LINES=$(egrep -oh "bazel://@npm_bazel_([a-z_]+)" package.json || echo "") for line in ${LINES[@]} ; do - # Trim the match from `/dist/npm_bazel_foobar"` to `foobar` - DEP=$(echo $line | cut -c 17- | rev | cut -c 2- | rev) + # Trim the match from `bazel://@npm_bazel_foobar` to `foobar` + DEP=$(echo $line | cut -c 20-) DEPS+=(${DEP}) + PACKAGES+=(${DEP}) done if [[ ${DEPS:-} ]] ; then ${RULES_NODEJS_DIR}/scripts/check_deps.sh ${DEPS[@]} fi + +for package in ${PACKAGES[@]:-} ; do + # Find name of dist dir (the postfix $RANDOM changes each time it is re-generated) + results=$(ls -d ${RULES_NODEJS_DIR}/dist/npm_bazel_${package}\$* 2> /dev/null || :) + if [[ -z "${results}" ]] ; then + echo "ERROR: You must first run 'yarn build_packages ${package}' or 'yarn build_packages_all'"; + exit 1 + fi + + # Replaces "bazel://@npm_bazel_foobar//:npm_package" with absolute + # path to generated npm package under /dist/npm_bazel_foobar + echo_and_run sedi "s#\"bazel://@npm_bazel_${package}//:npm_package\"#\"file://${results}\"#" package.json +done diff --git a/scripts/unlink_deps.sh b/scripts/unlink_deps.sh new file mode 100755 index 0000000000..1a6ac90554 --- /dev/null +++ b/scripts/unlink_deps.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -eu -o pipefail +# -e: exits if a command fails +# -u: errors if an variable is referenced before being set +# -o pipefail: causes a pipeline to produce a failure return code if any command errors + +readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd) + +echo_and_run() { echo "+ $@" ; "$@" ; } + +# sedi makes `sed -i` work on both OSX & Linux +# See https://stackoverflow.com/questions/2320564/i-need-my-sed-i-command-for-in-place-editing-to-work-with-both-gnu-sed-and-bsd +sedi () { + case $(uname) in + Darwin*) sedi=('-i' '') ;; + *) sedi='-i' ;; + esac + + sed "${sedi[@]}" "$@" +} + +# Replaces "file://..." with absolute path to generated npm package under /dist/npm_bazel_foobar$RANDOM +# back to "bazel://@npm_bazel_foobar//:npm_package" +echo_and_run sedi "s#\"file://${RULES_NODEJS_DIR}/dist/npm_bazel_\([a-z_]*\)\$*[0-9]*\"#\"bazel://@npm_bazel_\1//:npm_package\"#" package.json