From 79d774e5d396faf52a919053674afa0cb5c50bfa Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 16 Mar 2023 14:05:20 -0400 Subject: [PATCH] ci(NODE-5090): download node to local directory (#3591) --- .evergreen/config.in.yml | 7 +- .evergreen/config.yml | 7 +- .evergreen/init-nvm.sh | 18 +-- .evergreen/install-dependencies.sh | 197 ++++++++++++++------------- .evergreen/run-benchmarks.sh | 2 +- .evergreen/run-bson-ext-test.sh | 2 +- .evergreen/run-custom-csfle-tests.sh | 2 +- .gitignore | 5 + 8 files changed, 116 insertions(+), 124 deletions(-) diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index 3a54cd92b2..8f7f190f9f 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -138,7 +138,7 @@ functions: MONGODB_URI="${MONGODB_URI}" \ AUTH=${AUTH} SSL=${SSL} TEST_CSFLE=true \ MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ - NODE_VERSION=${NODE_VERSION} SKIP_DEPS=${SKIP_DEPS|1} \ + SKIP_DEPS=${SKIP_DEPS|1} \ bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh "run serverless tests": @@ -238,7 +238,6 @@ functions: AUTH=${AUTH} \ SSL=${SSL} \ MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ - NODE_VERSION=${NODE_VERSION} \ SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \ MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \ TOPOLOGY="${TOPOLOGY}" \ @@ -259,7 +258,6 @@ functions: AUTH=${AUTH} \ SSL=${SSL} \ MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ - NODE_VERSION=${NODE_VERSION} \ TOPOLOGY="${TOPOLOGY}" \ COMPRESSOR="${COMPRESSOR}" \ SKIP_DEPS=${SKIP_DEPS|1} \ @@ -404,9 +402,6 @@ functions: ${PREPARE_SHELL} NODE_LTS_NAME=${NODE_LTS_NAME} NPM_OPTIONS=${NPM_OPTIONS}\ bash ${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh - - command: expansions.update - params: - file: src/deps-expansion.yml "run atlas tests": - command: shell.exec diff --git a/.evergreen/config.yml b/.evergreen/config.yml index b5f4f6d6d1..8ba0ed4bc3 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -112,7 +112,7 @@ functions: MONGODB_URI="${MONGODB_URI}" \ AUTH=${AUTH} SSL=${SSL} TEST_CSFLE=true \ MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ - NODE_VERSION=${NODE_VERSION} SKIP_DEPS=${SKIP_DEPS|1} \ + SKIP_DEPS=${SKIP_DEPS|1} \ bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh run serverless tests: - command: timeout.update @@ -208,7 +208,6 @@ functions: AUTH=${AUTH} \ SSL=${SSL} \ MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ - NODE_VERSION=${NODE_VERSION} \ SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \ MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \ TOPOLOGY="${TOPOLOGY}" \ @@ -228,7 +227,6 @@ functions: AUTH=${AUTH} \ SSL=${SSL} \ MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ - NODE_VERSION=${NODE_VERSION} \ TOPOLOGY="${TOPOLOGY}" \ COMPRESSOR="${COMPRESSOR}" \ SKIP_DEPS=${SKIP_DEPS|1} \ @@ -358,9 +356,6 @@ functions: ${PREPARE_SHELL} NODE_LTS_NAME=${NODE_LTS_NAME} NPM_OPTIONS=${NPM_OPTIONS}\ bash ${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh - - command: expansions.update - params: - file: src/deps-expansion.yml run atlas tests: - command: shell.exec type: test diff --git a/.evergreen/init-nvm.sh b/.evergreen/init-nvm.sh index 46a2f24752..28d14f16d8 100644 --- a/.evergreen/init-nvm.sh +++ b/.evergreen/init-nvm.sh @@ -1,21 +1,13 @@ #! /usr/bin/env bash export PATH="/opt/mongodbtoolchain/v2/bin:$PATH" -NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts" -export NVM_DIR="${NODE_ARTIFACTS_PATH}/nvm" +NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts" if [[ "$OS" == "Windows_NT" ]]; then - NVM_HOME=$(cygpath -w "$NVM_DIR") - export NVM_HOME - NVM_SYMLINK=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin") - export NVM_SYMLINK - NVM_ARTIFACTS_PATH=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin") - export NVM_ARTIFACTS_PATH - PATH=$(cygpath $NVM_SYMLINK):$(cygpath $NVM_HOME):$PATH - export PATH - echo "updated path on windows PATH=$PATH" -else - [ -s "$NVM_DIR/nvm.sh" ] && source "$NVM_DIR/nvm.sh" + NODE_ARTIFACTS_PATH=$(cygpath --unix "$NODE_ARTIFACTS_PATH") fi +export PATH="$NODE_ARTIFACTS_PATH/npm_global/bin:$NODE_ARTIFACTS_PATH/nodejs/bin:$PATH" +hash -r + export NODE_OPTIONS="--trace-deprecation --trace-warnings" diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh index 3dd3b8b708..8d836acacc 100644 --- a/.evergreen/install-dependencies.sh +++ b/.evergreen/install-dependencies.sh @@ -1,108 +1,113 @@ -#!/bin/bash +#!/usr/bin/env bash set -o errexit # Exit the script with error if any of the commands fail -NVM_WINDOWS_URL="https://github.com/coreybutler/nvm-windows/releases/download/1.1.9/nvm-noinstall.zip" -NVM_URL="https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh" - -NODE_LTS_NAME=${NODE_LTS_NAME:-fermium} -NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts" - -# this needs to be explicitly exported for the nvm install below -export NVM_DIR="${NODE_ARTIFACTS_PATH}/nvm" -export XDG_CONFIG_HOME=${NODE_ARTIFACTS_PATH} - -# create node artifacts path if needed -mkdir -p "${NODE_ARTIFACTS_PATH}" - -function node_lts_to_version() { - case $1 in - "erbium") - echo 12 - ;; - "fermium") - echo 14 - ;; - "gallium") - echo 16 - ;; - "hydrogen") - echo 18 - ;; - "iron") - echo 20 - ;; - "latest") - echo 'latest' - ;; - *) - echo "Unsupported Node LTS version $1" - ;; - esac -} - -function latest_version_for_node_major() { - local __NODE_MAJOR_VERSION=$1 - local NODE_DOWNLOAD_URI="https://nodejs.org/download/release/latest-v${__NODE_MAJOR_VERSION}.x/SHASUMS256.txt" - - if [ $__NODE_MAJOR_VERSION == 'latest' ] - then - NODE_DOWNLOAD_URI="https://nodejs.org/download/release/latest/SHASUMS256.txt" - fi - - # check that the requested version does exist - curl --silent --fail $NODE_DOWNLOAD_URI &> /dev/null - - echo $(curl --retry 8 --retry-delay 5 --max-time 50 --silent -o- $NODE_DOWNLOAD_URI | head -n 1 | awk '{print $2};' | cut -d- -f2) -} - -NODE_MAJOR_VERSION=$(node_lts_to_version $NODE_LTS_NAME) -NODE_VERSION=$(latest_version_for_node_major $NODE_MAJOR_VERSION) -NODE_VERSION=${NODE_VERSION:1} # :1 gets rid of the leading 'v' - -echo "set version to $NODE_VERSION" - -# output node version to expansions file for use in subsequent scripts -cat < deps-expansion.yml - NODE_VERSION: "$NODE_VERSION" -EOT +NODE_LTS_NAME=${NODE_LTS_NAME:-erbium} +NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY:-$(pwd)}/node-artifacts" +if [[ "$OS" = "Windows_NT" ]]; then NODE_ARTIFACTS_PATH=$(cygpath --unix "$NODE_ARTIFACTS_PATH"); fi + +CURL_FLAGS=( + --fail # Exit code 1 if request fails + --compressed # Request a compressed response should keep fetching fast + --location # Follow a redirect + --retry 8 # Retry HTTP 408, 429, 500, 502, 503 or 504, 8 times + --silent # Do not print a progress bar + --show-error # Despite the silent flag still print out errors + --max-time 900 # 900 seconds is 15 minutes, evergreen times out at 20 + --continue-at - # If a download is interrupted it can figure out where to resume +) + +mkdir -p "$NODE_ARTIFACTS_PATH/npm_global" + +# Comparisons are all case insensitive +shopt -s nocasematch + +# index.tab is a sorted tab separated values file with the following headers +# 0 1 2 3 4 5 6 7 8 9 10 +# version date files npm v8 uv zlib openssl modules lts security +curl "${CURL_FLAGS[@]}" "https://nodejs.org/dist/index.tab" --output node_index.tab + +while IFS=$'\t' read -r -a row; do + node_index_version="${row[0]}" + node_index_date="${row[1]}" + node_index_lts="${row[9]}" + [[ "$node_index_version" = "version" ]] && continue # skip tsv header + [[ "$NODE_LTS_NAME" = "latest" ]] && break # first line is latest + [[ "$NODE_LTS_NAME" = "$node_index_lts" ]] && break # case insensitive compare +done < node_index.tab + +if [[ "$OS" = "Windows_NT" ]]; then + operating_system="win" +elif [[ $(uname) = "darwin" ]]; then + operating_system="darwin" +elif [[ $(uname) = "linux" ]]; then + operating_system="linux" +else + echo "Unable to determine operating system: $operating_system" + exit 1 +fi -# install Node.js on Windows -if [[ "$OS" == "Windows_NT" ]]; then - # Delete pre-existing node to avoid version conflicts - rm -rf "/cygdrive/c/Program Files/nodejs" +architecture=$(uname -m) +if [[ $architecture = "x86_64" ]]; then + architecture="x64" +elif [[ $architecture = "arm64" ]]; then + architecture="arm64" +elif [[ $architecture = "aarch64" ]]; then + architecture="arm64" +elif [[ $architecture == s390* ]]; then + architecture="s390x" +elif [[ $architecture == ppc* ]]; then + architecture="ppc64le" +else + echo "Unable to determine operating system: $architecture" + exit 1 +fi +file_extension="tar.gz" +if [[ "$OS" = "Windows_NT" ]]; then file_extension="zip"; fi - NVM_HOME=$(cygpath -w "$NVM_DIR") - export NVM_HOME - NVM_SYMLINK=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin") - export NVM_SYMLINK - NVM_ARTIFACTS_PATH=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin") - export NVM_ARTIFACTS_PATH - PATH=$(cygpath $NVM_SYMLINK):$(cygpath $NVM_HOME):$PATH - export PATH +node_directory="node-${node_index_version}-${operating_system}-${architecture}" +node_archive="${node_directory}.${file_extension}" +node_archive_path="$NODE_ARTIFACTS_PATH/${node_archive}" +node_download_url="https://nodejs.org/dist/${node_index_version}/${node_archive}" - curl -L $NVM_WINDOWS_URL -o nvm.zip - unzip -d "$NVM_DIR" nvm.zip - rm nvm.zip +echo "Node.js ${node_index_version} for ${operating_system}-${architecture} released on ${node_index_date}" - chmod 777 "$NVM_DIR" - chmod -R a+rx "$NVM_DIR" +set -o xtrace - cat < "$NVM_DIR/settings.txt" -root: $NVM_HOME -path: $NVM_SYMLINK -EOT - nvm install "$NODE_VERSION" - nvm use "$NODE_VERSION" - which node || echo "node not found, PATH=$PATH" - which npm || echo "npm not found, PATH=$PATH" - npm cache clear --force # Fixes: Cannot read properties of null (reading 'pickAlgorithm') error on windows +curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path" -# install Node.js on Linux/MacOS +if [[ "$file_extension" = "zip" ]]; then + unzip -q "$node_archive_path" -d "${NODE_ARTIFACTS_PATH}" + mkdir -p "${NODE_ARTIFACTS_PATH}/nodejs" + # Windows "bins" are at the top level + mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs/bin" + # Need to add executable flag ourselves + chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/node.exe" + chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/npm" else - curl -o- $NVM_URL | bash - [ -s "${NVM_DIR}/nvm.sh" ] && source "${NVM_DIR}/nvm.sh" - nvm install --no-progress "$NODE_VERSION" + tar -xf "$node_archive_path" -C "${NODE_ARTIFACTS_PATH}" + mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs" fi -npm install ${NPM_OPTIONS} +export PATH="$NODE_ARTIFACTS_PATH/npm_global/bin:$NODE_ARTIFACTS_PATH/nodejs/bin:$PATH" +hash -r + +# Set npm -g prefix to our local artifacts directory +cat < .npmrc +prefix=$NODE_ARTIFACTS_PATH/npm_global +EOT + +# Cannot upgrade npm version for node 12 +if [[ $operating_system != "win" ]] && [[ $NODE_LTS_NAME != "erbium" ]]; then + # Update npm to latest when we can + npm install --global npm@latest + hash -r +elif [[ $NODE_LTS_NAME == "erbium" ]]; then + # Node.js 12 can run up to npm v8 + npm install --global npm@8 + hash -r +fi + +echo "npm version: $(npm -v)" + +npm install "${NPM_OPTIONS}" diff --git a/.evergreen/run-benchmarks.sh b/.evergreen/run-benchmarks.sh index c4fc7fd4aa..d8347077a8 100644 --- a/.evergreen/run-benchmarks.sh +++ b/.evergreen/run-benchmarks.sh @@ -1,6 +1,6 @@ #! /bin/bash -[ -s "$PROJECT_DIRECTORY/node-artifacts/nvm/nvm.sh" ] && source "$PROJECT_DIRECTORY"/node-artifacts/nvm/nvm.sh +source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh" export MONGODB_URI=$MONGODB_URI diff --git a/.evergreen/run-bson-ext-test.sh b/.evergreen/run-bson-ext-test.sh index 112ba611aa..7b49184783 100755 --- a/.evergreen/run-bson-ext-test.sh +++ b/.evergreen/run-bson-ext-test.sh @@ -1,6 +1,6 @@ #!/bin/bash -[ -s "$PROJECT_DIRECTORY/node-artifacts/nvm/nvm.sh" ] && source "$PROJECT_DIRECTORY"/node-artifacts/nvm/nvm.sh +source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh" set -o xtrace # Write all commands first to stderr set -o errexit # Exit the script with error if any of the commands fail diff --git a/.evergreen/run-custom-csfle-tests.sh b/.evergreen/run-custom-csfle-tests.sh index 5511f37140..957f5386fb 100644 --- a/.evergreen/run-custom-csfle-tests.sh +++ b/.evergreen/run-custom-csfle-tests.sh @@ -12,7 +12,7 @@ export CSFLE_KMS_PROVIDERS=${CSFLE_KMS_PROVIDERS} export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} echo "csfle CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" -[ -s "$PROJECT_DIRECTORY/node-artifacts/nvm/nvm.sh" ] && source "$PROJECT_DIRECTORY"/node-artifacts/nvm/nvm.sh +source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh" set -o xtrace # Write all commands first to stderr set -o errexit # Exit the script with error if any of the commands fail diff --git a/.gitignore b/.gitignore index 43e6877bad..1f0eb75d32 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,8 @@ etc/docs/build !docs/**/*.png !docs/**/*.css !docs/**/*.js +.nvmrc + +node_index.tab +node-artifacts +.npmrc