ci(NODE-5090): download node to local directory (#3591)

mongodb · Mar 16, 2023 · 79d774e · 79d774e
1 parent 2d3576b
commit 79d774e
Show file tree

Hide file tree

Showing 8 changed files with 116 additions and 124 deletions.
diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml
@@ -138,7 +138,7 @@ functions:
           MONGODB_URI="${MONGODB_URI}" \
           AUTH=${AUTH} SSL=${SSL} TEST_CSFLE=true \
           MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
-          NODE_VERSION=${NODE_VERSION} SKIP_DEPS=${SKIP_DEPS|1} \
+          SKIP_DEPS=${SKIP_DEPS|1} \
             bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
 
   "run serverless tests":
@@ -238,7 +238,6 @@ functions:
           AUTH=${AUTH} \
           SSL=${SSL} \
           MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
-          NODE_VERSION=${NODE_VERSION} \
           SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \
           MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \
           TOPOLOGY="${TOPOLOGY}" \
@@ -259,7 +258,6 @@ functions:
           AUTH=${AUTH} \
           SSL=${SSL} \
           MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
-          NODE_VERSION=${NODE_VERSION} \
           TOPOLOGY="${TOPOLOGY}" \
           COMPRESSOR="${COMPRESSOR}" \
           SKIP_DEPS=${SKIP_DEPS|1} \
@@ -404,9 +402,6 @@ functions:
           ${PREPARE_SHELL}
           NODE_LTS_NAME=${NODE_LTS_NAME} NPM_OPTIONS=${NPM_OPTIONS}\
             bash ${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh
-    - command: expansions.update
-      params:
-        file: src/deps-expansion.yml
 
   "run atlas tests":
     - command: shell.exec

diff --git a/.evergreen/config.yml b/.evergreen/config.yml
@@ -112,7 +112,7 @@ functions:
           MONGODB_URI="${MONGODB_URI}" \
           AUTH=${AUTH} SSL=${SSL} TEST_CSFLE=true \
           MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
-          NODE_VERSION=${NODE_VERSION} SKIP_DEPS=${SKIP_DEPS|1} \
+          SKIP_DEPS=${SKIP_DEPS|1} \
             bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
   run serverless tests:
     - command: timeout.update
@@ -208,7 +208,6 @@ functions:
           AUTH=${AUTH} \
           SSL=${SSL} \
           MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
-          NODE_VERSION=${NODE_VERSION} \
           SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \
           MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \
           TOPOLOGY="${TOPOLOGY}" \
@@ -228,7 +227,6 @@ functions:
           AUTH=${AUTH} \
           SSL=${SSL} \
           MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
-          NODE_VERSION=${NODE_VERSION} \
           TOPOLOGY="${TOPOLOGY}" \
           COMPRESSOR="${COMPRESSOR}" \
           SKIP_DEPS=${SKIP_DEPS|1} \
@@ -358,9 +356,6 @@ functions:
           ${PREPARE_SHELL}
           NODE_LTS_NAME=${NODE_LTS_NAME} NPM_OPTIONS=${NPM_OPTIONS}\
             bash ${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh
-    - command: expansions.update
-      params:
-        file: src/deps-expansion.yml
   run atlas tests:
     - command: shell.exec
       type: test

diff --git a/.evergreen/init-nvm.sh b/.evergreen/init-nvm.sh
@@ -1,21 +1,13 @@
 #! /usr/bin/env bash
 
 export PATH="/opt/mongodbtoolchain/v2/bin:$PATH"
-NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts"
-export NVM_DIR="${NODE_ARTIFACTS_PATH}/nvm"
 
+NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts"
 if [[ "$OS" == "Windows_NT" ]]; then
-    NVM_HOME=$(cygpath -w "$NVM_DIR")
-    export NVM_HOME
-    NVM_SYMLINK=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
-    export NVM_SYMLINK
-    NVM_ARTIFACTS_PATH=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
-    export NVM_ARTIFACTS_PATH
-    PATH=$(cygpath $NVM_SYMLINK):$(cygpath $NVM_HOME):$PATH
-    export PATH
-    echo "updated path on windows PATH=$PATH"
-else
-    [ -s "$NVM_DIR/nvm.sh" ] && source "$NVM_DIR/nvm.sh"
+  NODE_ARTIFACTS_PATH=$(cygpath --unix "$NODE_ARTIFACTS_PATH")
 fi
 
+export PATH="$NODE_ARTIFACTS_PATH/npm_global/bin:$NODE_ARTIFACTS_PATH/nodejs/bin:$PATH"
+hash -r
+
 export NODE_OPTIONS="--trace-deprecation --trace-warnings"
diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh
@@ -1,108 +1,113 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -o errexit  # Exit the script with error if any of the commands fail
 
-NVM_WINDOWS_URL="https://github.com/coreybutler/nvm-windows/releases/download/1.1.9/nvm-noinstall.zip"
-NVM_URL="https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh"
-
-NODE_LTS_NAME=${NODE_LTS_NAME:-fermium}
-NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts"
-
-# this needs to be explicitly exported for the nvm install below
-export NVM_DIR="${NODE_ARTIFACTS_PATH}/nvm"
-export XDG_CONFIG_HOME=${NODE_ARTIFACTS_PATH}
-
-# create node artifacts path if needed
-mkdir -p "${NODE_ARTIFACTS_PATH}"
-
-function node_lts_to_version() {
-  case $1 in
-    "erbium")
-      echo 12
-      ;;
-    "fermium")
-      echo 14
-      ;;
-    "gallium")
-      echo 16
-      ;;
-    "hydrogen")
-      echo 18
-      ;;
-    "iron")
-      echo 20
-      ;;
-    "latest")
-      echo 'latest'
-      ;;
-    *)
-      echo "Unsupported Node LTS version $1"
-      ;;
-  esac
-}
-
-function latest_version_for_node_major() {
-  local __NODE_MAJOR_VERSION=$1
-  local NODE_DOWNLOAD_URI="https://nodejs.org/download/release/latest-v${__NODE_MAJOR_VERSION}.x/SHASUMS256.txt"
-
-  if [ $__NODE_MAJOR_VERSION == 'latest' ]
-  then
-    NODE_DOWNLOAD_URI="https://nodejs.org/download/release/latest/SHASUMS256.txt"
-  fi
-
-  # check that the requested version does exist
-  curl --silent --fail $NODE_DOWNLOAD_URI &> /dev/null
-
-  echo $(curl --retry 8 --retry-delay 5  --max-time 50 --silent -o- $NODE_DOWNLOAD_URI | head -n 1 | awk '{print $2};' | cut -d- -f2)
-}
-
-NODE_MAJOR_VERSION=$(node_lts_to_version $NODE_LTS_NAME)
-NODE_VERSION=$(latest_version_for_node_major $NODE_MAJOR_VERSION)
-NODE_VERSION=${NODE_VERSION:1} # :1 gets rid of the leading 'v'
-
-echo "set version to $NODE_VERSION"
-
-# output node version to expansions file for use in subsequent scripts
-cat <<EOT > deps-expansion.yml
-  NODE_VERSION: "$NODE_VERSION"
-EOT
+NODE_LTS_NAME=${NODE_LTS_NAME:-erbium}
+NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY:-$(pwd)}/node-artifacts"
+if [[ "$OS" = "Windows_NT" ]]; then NODE_ARTIFACTS_PATH=$(cygpath --unix "$NODE_ARTIFACTS_PATH"); fi
+
+CURL_FLAGS=(
+  --fail          # Exit code 1 if request fails
+  --compressed    # Request a compressed response should keep fetching fast
+  --location      # Follow a redirect
+  --retry 8       # Retry HTTP 408, 429, 500, 502, 503 or 504, 8 times
+  --silent        # Do not print a progress bar
+  --show-error    # Despite the silent flag still print out errors
+  --max-time 900  # 900 seconds is 15 minutes, evergreen times out at 20
+  --continue-at - # If a download is interrupted it can figure out where to resume
+)
+
+mkdir -p "$NODE_ARTIFACTS_PATH/npm_global"
+
+# Comparisons are all case insensitive
+shopt -s nocasematch
+
+# index.tab is a sorted tab separated values file with the following headers
+# 0       1    2     3   4  5  6    7       8       9   10
+# version date files npm v8 uv zlib openssl modules lts security
+curl "${CURL_FLAGS[@]}" "https://nodejs.org/dist/index.tab" --output node_index.tab
+
+while IFS=$'\t' read -r -a row; do
+  node_index_version="${row[0]}"
+  node_index_date="${row[1]}"
+  node_index_lts="${row[9]}"
+  [[ "$node_index_version" = "version" ]] && continue # skip tsv header
+  [[ "$NODE_LTS_NAME" = "latest" ]] && break # first line is latest
+  [[ "$NODE_LTS_NAME" = "$node_index_lts" ]] && break # case insensitive compare
+done < node_index.tab
+
+if [[ "$OS" = "Windows_NT" ]]; then
+  operating_system="win"
+elif [[ $(uname) = "darwin" ]]; then
+  operating_system="darwin"
+elif [[ $(uname) = "linux" ]]; then
+  operating_system="linux"
+else
+  echo "Unable to determine operating system: $operating_system"
+  exit 1
+fi
 
-# install Node.js on Windows
-if [[ "$OS" == "Windows_NT" ]]; then
-  # Delete pre-existing node to avoid version conflicts
-  rm -rf "/cygdrive/c/Program Files/nodejs"
+architecture=$(uname -m)
+if [[ $architecture = "x86_64" ]]; then
+  architecture="x64"
+elif [[ $architecture = "arm64" ]]; then
+  architecture="arm64"
+elif [[ $architecture = "aarch64" ]]; then
+  architecture="arm64"
+elif [[ $architecture == s390* ]]; then
+  architecture="s390x"
+elif [[ $architecture == ppc* ]]; then
+  architecture="ppc64le"
+else
+  echo "Unable to determine operating system: $architecture"
+  exit 1
+fi
 
+file_extension="tar.gz"
+if [[ "$OS" = "Windows_NT" ]]; then file_extension="zip"; fi
 
-  NVM_HOME=$(cygpath -w "$NVM_DIR")
-  export NVM_HOME
-  NVM_SYMLINK=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
-  export NVM_SYMLINK
-  NVM_ARTIFACTS_PATH=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
-  export NVM_ARTIFACTS_PATH
-  PATH=$(cygpath $NVM_SYMLINK):$(cygpath $NVM_HOME):$PATH
-  export PATH
+node_directory="node-${node_index_version}-${operating_system}-${architecture}"
+node_archive="${node_directory}.${file_extension}"
+node_archive_path="$NODE_ARTIFACTS_PATH/${node_archive}"
+node_download_url="https://nodejs.org/dist/${node_index_version}/${node_archive}"
 
-  curl -L $NVM_WINDOWS_URL -o nvm.zip
-  unzip -d "$NVM_DIR" nvm.zip
-  rm nvm.zip
+echo "Node.js ${node_index_version} for ${operating_system}-${architecture} released on ${node_index_date}"
 
-  chmod 777 "$NVM_DIR"
-  chmod -R a+rx "$NVM_DIR"
+set -o xtrace
 
-  cat <<EOT > "$NVM_DIR/settings.txt"
-root: $NVM_HOME
-path: $NVM_SYMLINK
-EOT
-  nvm install "$NODE_VERSION"
-  nvm use "$NODE_VERSION"
-  which node || echo "node not found, PATH=$PATH"
-  which npm || echo "npm not found, PATH=$PATH"
-  npm cache clear --force # Fixes: Cannot read properties of null (reading 'pickAlgorithm') error on windows
+curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path"
 
-# install Node.js on Linux/MacOS
+if [[ "$file_extension" = "zip" ]]; then
+  unzip -q "$node_archive_path" -d "${NODE_ARTIFACTS_PATH}"
+  mkdir -p "${NODE_ARTIFACTS_PATH}/nodejs"
+  # Windows "bins" are at the top level
+  mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs/bin"
+  # Need to add executable flag ourselves
+  chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/node.exe"
+  chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/npm"
 else
-  curl -o- $NVM_URL | bash
-  [ -s "${NVM_DIR}/nvm.sh" ] && source "${NVM_DIR}/nvm.sh"
-  nvm install --no-progress "$NODE_VERSION"
+  tar -xf "$node_archive_path" -C "${NODE_ARTIFACTS_PATH}"
+  mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs"
 fi
 
-npm install ${NPM_OPTIONS}
+export PATH="$NODE_ARTIFACTS_PATH/npm_global/bin:$NODE_ARTIFACTS_PATH/nodejs/bin:$PATH"
+hash -r
+
+# Set npm -g prefix to our local artifacts directory
+cat <<EOT > .npmrc
+prefix=$NODE_ARTIFACTS_PATH/npm_global
+EOT
+
+# Cannot upgrade npm version for node 12
+if [[ $operating_system != "win" ]] && [[ $NODE_LTS_NAME != "erbium" ]]; then
+  # Update npm to latest when we can
+  npm install --global npm@latest
+  hash -r
+elif [[ $NODE_LTS_NAME == "erbium" ]]; then
+  # Node.js 12 can run up to npm v8
+  npm install --global npm@8
+  hash -r
+fi
+
+echo "npm version: $(npm -v)"
+
+npm install "${NPM_OPTIONS}"
diff --git a/.evergreen/run-benchmarks.sh b/.evergreen/run-benchmarks.sh
@@ -1,6 +1,6 @@
 #! /bin/bash
 
-[ -s "$PROJECT_DIRECTORY/node-artifacts/nvm/nvm.sh" ] && source "$PROJECT_DIRECTORY"/node-artifacts/nvm/nvm.sh
+source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh"
 
 export MONGODB_URI=$MONGODB_URI
 

diff --git a/.evergreen/run-bson-ext-test.sh b/.evergreen/run-bson-ext-test.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-[ -s "$PROJECT_DIRECTORY/node-artifacts/nvm/nvm.sh" ] && source "$PROJECT_DIRECTORY"/node-artifacts/nvm/nvm.sh
+source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh"
 
 set -o xtrace   # Write all commands first to stderr
 set -o errexit  # Exit the script with error if any of the commands fail

diff --git a/.evergreen/run-custom-csfle-tests.sh b/.evergreen/run-custom-csfle-tests.sh
@@ -12,7 +12,7 @@ export CSFLE_KMS_PROVIDERS=${CSFLE_KMS_PROVIDERS}
 export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH}
 echo "csfle CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH"
 
-[ -s "$PROJECT_DIRECTORY/node-artifacts/nvm/nvm.sh" ] && source "$PROJECT_DIRECTORY"/node-artifacts/nvm/nvm.sh
+source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh"
 
 set -o xtrace   # Write all commands first to stderr
 set -o errexit  # Exit the script with error if any of the commands fail

diff --git a/.gitignore b/.gitignore
@@ -87,3 +87,8 @@ etc/docs/build
 !docs/**/*.png
 !docs/**/*.css
 !docs/**/*.js
+.nvmrc
+
+node_index.tab
+node-artifacts
+.npmrc