Clean packages from yarn cache in process_package_json step of yarn_i…

…nstall that have file:// URIs (#639)

Also includes local development script updates

.circleci/config.yml

@@ -204,7 +204,7 @@ jobs:
       - persist_to_workspace:
           root: ~/
           paths:
-            - ./rules_nodejs/dist/npm_bazel_typescript
+            - ./rules_nodejs/dist/npm_bazel_typescript$*
 
   build_karma_package:
     <<: *job_defaults
@@ -217,7 +217,7 @@ jobs:
       - persist_to_workspace:
           root: ~/
           paths:
-            - ./rules_nodejs/dist/npm_bazel_karma
+            - ./rules_nodejs/dist/npm_bazel_karma$*
 
   build_jasmine_package:
     <<: *job_defaults
@@ -230,7 +230,7 @@ jobs:
       - persist_to_workspace:
           root: ~/
           paths:
-            - ./rules_nodejs/dist/npm_bazel_jasmine
+            - ./rules_nodejs/dist/npm_bazel_jasmine$*
 
   build_labs_package:
     <<: *job_defaults
@@ -243,7 +243,7 @@ jobs:
       - persist_to_workspace:
           root: ~/
           paths:
-            - ./rules_nodejs/dist/npm_bazel_labs
+            - ./rules_nodejs/dist/npm_bazel_labs$*
 
   test_packages:
     <<: *job_defaults

e2e/define_var/WORKSPACE

@@ -1,8 +1,19 @@
-workspace(name = "examples_define_var")
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+workspace(name = "e2e_define_var")
 
-# In your code, you'd fetch this repository with an `http_archive` call.
-# We do this local repository only because this example lives in the same
-# repository with the rules_nodejs code and we want to test them together.
 local_repository(
     name = "build_bazel_rules_nodejs",
     path = "../../dist/build_bazel_rules_nodejs/release",

e2e/tsconfig_extends/WORKSPACE

@@ -1,19 +1,26 @@
-workspace(name = "examples_tsconfig_extends")
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+workspace(name = "e2e_tsconfig_extends")
 
-# In your code, you'd fetch this repository with an `http_archive` call.
-# We do this local repository only because this example lives in the same
-# repository with the rules_nodejs code and we want to test them together.
 local_repository(
     name = "build_bazel_rules_nodejs",
     path = "../../dist/build_bazel_rules_nodejs/release",
 )
 
 load("@build_bazel_rules_nodejs//:defs.bzl", "yarn_install")
 
-# This runs yarn install, then our generate_build_file.js to create BUILD files
-# inside the resulting node_modules directory.
-# The name "npm" here means the resulting modules are referenced like
-# @npm//jasmine
 yarn_install(
     name = "npm",
     package_json = "//:package.json",

examples/webapp/package.json

@@ -4,4 +4,4 @@
     "scripts": {
         "test": "bazel build :all"
     }
-}
+}

internal/npm_install/npm_install.bzl

@@ -178,7 +178,10 @@ cd "{root}" && "{npm}" {npm_args}
     _add_data_dependencies(repository_ctx)
     _add_scripts(repository_ctx)
 
-    result = repository_ctx.execute([node, "process_package_json.js", ",".join(repository_ctx.attr.exclude_packages)])
+    result = repository_ctx.execute(
+        [node, "process_package_json.js", "npm", ",".join(repository_ctx.attr.exclude_packages)],
+        quiet = repository_ctx.attr.quiet,
+    )
     if result.return_code:
         fail("node failed: \nSTDOUT:\n%s\nSTDERR:\n%s" % (result.stdout, result.stderr))
 
@@ -249,7 +252,10 @@ def _yarn_install_impl(repository_ctx):
     _add_data_dependencies(repository_ctx)
     _add_scripts(repository_ctx)
 
-    result = repository_ctx.execute([node, "process_package_json.js", ",".join(repository_ctx.attr.exclude_packages)])
+    result = repository_ctx.execute(
+        [node, "process_package_json.js", "yarn", ",".join(repository_ctx.attr.exclude_packages)],
+        quiet = repository_ctx.attr.quiet,
+    )
     if result.return_code:
         fail("node failed: \nSTDOUT:\n%s\nSTDERR:\n%s" % (result.stdout, result.stderr))
 

internal/npm_install/process_package_json.js

@@ -23,10 +23,13 @@
 'use strict';
 
 const fs = require('fs');
-const path = require('path');
+const child_process = require('child_process');
+
+const DEBUG = false;
 
 const args = process.argv.slice(2);
-const removePackages = args[0] ? args[0].split(',') : [];
+const packageManager = args[0];
+const excludePackages = args[1] ? args[1].split(',') : [];
 
 if (require.main === module) {
   main();
@@ -36,9 +39,26 @@ if (require.main === module) {
  * Main entrypoint.
  */
 function main() {
+  const isYarn = (packageManager === 'yarn');
+
   const pkg = JSON.parse(fs.readFileSync('_package.json', {encoding: 'utf8'}));
 
-  removePackages.forEach(p => {
+  if (DEBUG) console.error(`Pre-processing package.json`);
+
+  removeExcludedPackages(pkg);
+
+  if (isYarn) {
+    // Work-around for https://github.com/yarnpkg/yarn/issues/2165
+    // Note: there is no equivalent npm functionality to clean out individual packages
+    // from the npm cache.
+    clearYarnFilePathCaches(pkg);
+  }
+
+  fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2));
+}
+
+function removeExcludedPackages(pkg) {
+  excludePackages.forEach(p => {
     if (pkg.dependencies) {
       delete pkg.dependencies[p];
     }
@@ -52,8 +72,45 @@ function main() {
       delete pkg.optionalDependencies[p];
     }
   });
+}
 
-  fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2));
+/**
+ * Runs `yarn cache clean` for all packages that have `file://` URIs.
+ * Work-around for https://github.com/yarnpkg/yarn/issues/2165.
+ */
+function clearYarnFilePathCaches(pkg) {
+  const fileRegex = /^file\:\/\//i;
+  const clearPackages = [];
+
+  if (pkg.dependencies) {
+    Object.keys(pkg.dependencies).forEach(p => {
+      if (pkg.dependencies[p].match(fileRegex)) {
+        clearPackages.push(p);
+      }
+    });
+  }
+  if (pkg.devDependencies) {
+    Object.keys(pkg.devDependencies).forEach(p => {
+      if (pkg.devDependencies[p].match(fileRegex)) {
+        clearPackages.push(p);
+      }
+    });
+  }
+  if (pkg.optionalDependencies) {
+    Object.keys(pkg.optionalDependencies).forEach(p => {
+      if (pkg.optionalDependencies[p].match(fileRegex)) {
+        clearPackages.push(p);
+      }
+    });
+  }
+
+  if (clearPackages.length) {
+    if (DEBUG) console.error(`Cleaning packages from yarn cache: ${clearPackages.join(' ')}`);
+
+    child_process.execFileSync(
+        'yarn', ['cache', 'clean'].concat(clearPackages),
+        {stdio: [process.stdin, process.stdout, process.stderr]});
+  }
 }
 
 module.exports = {main};

scripts/build_all.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+# -e: exits if a command fails
+# -u: errors if an variable is referenced before being set
+# -o pipefail: causes a pipeline to produce a failure return code if any command errors
+
+readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd)
+
+${RULES_NODEJS_DIR}/scripts/build_release.sh
+${RULES_NODEJS_DIR}/scripts/build_packages_all.sh

scripts/build_packages.sh

@@ -15,7 +15,8 @@ echo_and_run() { echo "+ $@" ; "$@" ; }
 
 for package in ${PACKAGES[@]} ; do
   (
-    readonly DEST_DIR="${DIST_DIR}/npm_bazel_${package}"
+    readonly DEST_DIR_BASE="${DIST_DIR}/npm_bazel_${package}"
+    readonly DEST_DIR="${DEST_DIR_BASE}\$${RANDOM}"
 
     # Build npm package
     cd "${PACKAGES_DIR}/${package}"
@@ -25,7 +26,7 @@ for package in ${PACKAGES[@]} ; do
 
     # Copy the npm_package to /dist
     echo "Copying npm package to ${DEST_DIR}"
-    rm -rf ${DEST_DIR}
+    rm -rf ${DEST_DIR_BASE}\$*
     mkdir -p ${DIST_DIR}
     readonly BAZEL_BIN=$(bazel info bazel-bin)
     echo_and_run cp -R "${BAZEL_BIN}/npm_package" ${DEST_DIR}

scripts/build_packages_all.sh

@@ -8,7 +8,5 @@ set -eu -o pipefail
 readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd)
 source "${RULES_NODEJS_DIR}/scripts/packages.sh"
 
-echo_and_run() { echo "+ $@" ; "$@" ; }
-
 ${RULES_NODEJS_DIR}/scripts/build_packages.sh ${PACKAGES[@]}
 

scripts/check_deps.sh

@@ -23,7 +23,8 @@ for dep in ${DEPS} ; do
         ALL_GOOD=0
     fi
   else
-    if [[ ! -d "${RULES_NODEJS_DIR}/dist/npm_bazel_${dep}" ]] ; then
+    results=$(ls -d ${RULES_NODEJS_DIR}/dist/npm_bazel_${dep}\$* 2> /dev/null || :)
+    if [[ -z "${results}" ]] ; then
       echo "ERROR: You must first run 'yarn build_packages ${dep}' or 'yarn build_packages_all'";
       ALL_GOOD=0
     fi

scripts/clean_all.sh

@@ -17,19 +17,21 @@ echo_and_run rm -rf ./internal/npm_install/test/package/node_modules
 
 echo_and_run bazel clean --expunge
 
-for rootDir in examples e2e internal/e2e packages ; do
-  (
-    cd ${rootDir}
-    for subDir in $(ls) ; do
-      [[ -d "${subDir}" ]] || continue
-      (
-        cd ${subDir}
-        if [[ -e 'WORKSPACE' ]] ; then
-          printf "\n\nCleaning /${rootDir}/${subDir}\n"
-          echo_and_run bazel clean --expunge
-          echo_and_run rm -rf node_modules
-        fi
-      )
-    done
-  )
-done
+${RULES_NODEJS_DIR}/scripts/clean_e2e_all.sh
+${RULES_NODEJS_DIR}/scripts/clean_examples_all.sh
+${RULES_NODEJS_DIR}/scripts/clean_packages_all.sh
+
+(
+  cd internal/e2e
+  for subDir in $(ls) ; do
+    [[ -d "${subDir}" ]] || continue
+    (
+      cd ${subDir}
+      if [[ -e 'WORKSPACE' ]] ; then
+        printf "\n\nCleaning /internal/e2e/${subDir}\n"
+        echo_and_run bazel clean --expunge
+        echo_and_run rm -rf node_modules
+      fi
+    )
+  done
+)

scripts/clean_e2e.sh

@@ -17,6 +17,7 @@ for e2eTest in ${E2E_TESTS[@]} ; do
     # Clean e2e test
     cd "${E2E_DIR}/${e2eTest}"
     printf "\n\nCleaning e2e test ${e2eTest}\n"
+    ${RULES_NODEJS_DIR}/scripts/unlink_deps.sh
     echo_and_run bazel clean --expunge
     echo_and_run rm -rf node_modules
   )

scripts/clean_e2e_all.sh

@@ -8,6 +8,6 @@ set -eu -o pipefail
 readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd)
 readonly E2E_DIR="${RULES_NODEJS_DIR}/e2e"
 
-readonly E2E=$(ls ${E2E_DIR})
+readonly E2E=$(ls -l ${E2E_DIR} | grep "^d" | awk -F" " '{print $9}')
 
 ${RULES_NODEJS_DIR}/scripts/clean_e2e.sh ${E2E[@]}

scripts/clean_examples.sh

@@ -17,6 +17,7 @@ for example in ${EXAMPLES[@]} ; do
     # Clean example
     cd "${EXAMPLES_DIR}/${example}"
     printf "\n\nCleaning example ${example}\n"
+    ${RULES_NODEJS_DIR}/scripts/unlink_deps.sh
     echo_and_run bazel clean --expunge
     echo_and_run rm -rf node_modules
   )

scripts/clean_examples_all.sh

@@ -8,6 +8,6 @@ set -eu -o pipefail
 readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd)
 readonly EXAMPLES_DIR="${RULES_NODEJS_DIR}/examples"
 
-readonly EXAMPLES=$(ls ${EXAMPLES_DIR})
+readonly EXAMPLES=$(ls -l ${EXAMPLES_DIR} | grep "^d" | awk -F" " '{print $9}')
 
 ${RULES_NODEJS_DIR}/scripts/clean_examples.sh ${EXAMPLES[@]}

scripts/clean_packages.sh

@@ -17,6 +17,7 @@ for package in ${PACKAGES[@]} ; do
     # Clean package
     cd "${PACKAGES_DIR}/${package}"
     printf "\n\nCleaning package ${package}\n"
+    ${RULES_NODEJS_DIR}/scripts/unlink_deps.sh
     echo_and_run bazel clean --expunge
     echo_and_run rm -rf node_modules
   )

scripts/clean_packages_all.sh

@@ -6,8 +6,7 @@ set -eu -o pipefail
 # -o pipefail: causes a pipeline to produce a failure return code if any command errors
 
 readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd)
-readonly PACKAGES_DIR="${RULES_NODEJS_DIR}/package"
+source "${RULES_NODEJS_DIR}/scripts/packages.sh"
 
-readonly PACKAGES=$(ls ${PACKAGES_DIR})
+${RULES_NODEJS_DIR}/scripts/clean_packages.sh ${PACKAGES[@]}
 
-${RULES_NODEJS_DIR}/scripts/clean_package.sh ${PACKAGES[@]}

scripts/clean_yarn_cache_selectively.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+# -e: exits if a command fails
+# -u: errors if an variable is referenced before being set
+# -o pipefail: causes a pipeline to produce a failure return code if any command errors
+
+readonly YARN_CACHE_ROOT=$(dirname $(yarn cache dir))
+readonly RULES_NODEJS_DIR=$(cd $(dirname "$0")/..; pwd)
+source "${RULES_NODEJS_DIR}/scripts/packages.sh"
+
+echo_and_run() { echo "+ $@" ; "$@" ; }
+
+echo "yarn cache root: ${YARN_CACHE_ROOT}"
+
+for package in ${PACKAGES[@]} ; do
+  echo_and_run yarn cache clean @bazel/${package}
+done
+
+# Also clean cache for different versions of yarn since the locally installed
+# yarn may have a different cache version from yarn version used by Bazel
+for package in ${PACKAGES[@]} ; do
+  echo_and_run rm -rf ${YARN_CACHE_ROOT}/*/npm-@bazel-${package}-*
+done