Adding OSD in the distribution build pipeline with build script using…

… volta to install nodejs dependencies (#2841) * Adding OSD in the distribution build pipeline with build script using volta to install nodejs dependencies Signed-off-by: Peter Zhu <[email protected]>
opensearch-project · Nov 3, 2022 · 14693cc · 14693cc
1 parent 3eea73a
commit 14693cc
Show file tree

Hide file tree

Showing 11 changed files with 105 additions and 28 deletions.
diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile
@@ -18,12 +18,12 @@ pipeline {
             H 1 * * * %INPUT_MANIFEST=1.3.7/opensearch-dashboards-1.3.7.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux
             H 1 * * * %INPUT_MANIFEST=1.4.0/opensearch-1.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos
             H 1 * * * %INPUT_MANIFEST=1.4.0/opensearch-dashboards-1.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux
-            H 1 * * * %INPUT_MANIFEST=2.3.1/opensearch-2.3.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos
-            H 1 * * * %INPUT_MANIFEST=2.3.1/opensearch-dashboards-2.3.1.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux
+            H 1 * * * %INPUT_MANIFEST=2.3.1/opensearch-2.3.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows
+            H 1 * * * %INPUT_MANIFEST=2.3.1/opensearch-dashboards-2.3.1.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows
             H/60 * * * * %INPUT_MANIFEST=2.4.0/opensearch-2.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows
-            H 1 * * * %INPUT_MANIFEST=2.4.0/opensearch-dashboards-2.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux
-            H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux
-            H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux
+            H/60 * * * * %INPUT_MANIFEST=2.4.0/opensearch-dashboards-2.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows
+            H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows
+            H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows
         '''
     }
     parameters {

diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile
@@ -43,8 +43,8 @@ pipeline {
         )
         string( // Note: need to update 'verify-parameters' entries if you add new platform(s)
             name: 'BUILD_PLATFORM',
-            description: "Build selected platform related artifacts, choices include 'linux'. Can combine multiple platforms with space in between",
-            defaultValue: 'linux',
+            description: "Build selected platform related artifacts, choices include 'linux windows'. Can combine multiple platforms with space in between",
+            defaultValue: 'linux windows',
             trim: true
         )
         choice(
@@ -81,7 +81,7 @@ pipeline {
                     echo("Verify Build Platforms")
                     def build_platform_array = params.BUILD_PLATFORM.tokenize(' ')
                     echo("User Entry Platforms: '${params.BUILD_PLATFORM}', ${build_platform_array}")
-                    def all_platforms = "linux"
+                    def all_platforms = "linux windows"
                     echo("All Supported Platforms: '${all_platforms}'")
 
                     if (params.BUILD_PLATFORM == null || params.BUILD_PLATFORM == '') {
@@ -557,6 +557,50 @@ pipeline {
                         }
                     }
                 }
+                stage('build-and-test-windows-x64-zip') {
+                    when {
+                        beforeAgent true
+                        expression{
+                            params.BUILD_PLATFORM.contains('windows')
+                        }
+                    }
+                    agent {
+                        node {
+                            label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host'
+                        }
+                    }
+                    steps {
+                        script {
+                            def buildManifestObj = buildAssembleUpload(
+                                componentName: "${COMPONENT_NAME}",
+                                inputManifest: "manifests/${INPUT_MANIFEST}",
+                                platform: 'windows',
+                                architecture: 'x64',
+                                distribution: "zip"
+                            )
+                            String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
+                            String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
+                            env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl
+                            env.INDEX_FILE_PATH = buildManifestObj.getIndexFileRoot("${JOB_NAME}")
+
+                            echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}"
+                            echo "artifactUrl (windows, x64, zip): ${artifactUrl}"
+
+                        }
+                    }
+                    post {
+                        always {
+                            script {
+                                lib.jenkins.Messages.new(this).add(
+                                    "${STAGE_NAME}",
+                                    lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"])
+                                )
+
+                                postCleanup()
+                            }
+                        }
+                    }
+                }
             }
         }
         stage('update index file') {
@@ -636,6 +680,7 @@ pipeline {
                             'assemble-archive-and-test-linux-x64-rpm',
                             'assemble-archive-and-test-linux-arm64-tar',
                             'assemble-archive-and-test-linux-arm64-rpm',
+                            'build-and-test-windows-x64-zip'
                         ])
 
                         publishNotification(

diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile
@@ -851,7 +851,8 @@ pipeline {
                             'build-and-test-linux-x64-tar',
                             'assemble-archive-and-test-linux-x64-rpm',
                             'build-and-test-linux-arm64-tar',
-                            'assemble-archive-and-test-linux-arm64-rpm'
+                            'assemble-archive-and-test-linux-arm64-rpm',
+                            'build-and-test-windows-x64-zip'
                         ])
 
                         publishNotification(
@@ -882,7 +883,7 @@ pipeline {
                     if (params.CREATE_GITHUB_ISSUE) {
                         createGithubIssue(
                             message: buildFailureMessage()
-                            )
+                        )
                     }
 
                     postCleanup()

diff --git a/scripts/components/OpenSearch-Dashboards/build.sh b/scripts/components/OpenSearch-Dashboards/build.sh
@@ -118,8 +118,9 @@ esac
 
 NVM_CMD="source $NVM_DIR/nvm.sh && nvm use"
 if [ "$PLATFORM" = "windows" ]; then
-    NVM_CMD="volta install node@`cat .nvmrc` && volta install yarn"
+    NVM_CMD="volta install node@`cat .nvmrc` && volta install yarn@`jq -r '.engines.yarn' package.json`"
 fi
+
 eval $NVM_CMD
 
 echo "Building node modules for core with $PLATFORM-$DISTRIBUTION-$ARCHITECTURE"

diff --git a/scripts/components/customImportMapDashboards/build.sh b/scripts/components/customImportMapDashboards/build.sh
@@ -67,16 +67,21 @@ fi
 [ -z "$OUTPUT" ] && OUTPUT=artifacts
 [ ! -z "$QUALIFIER" ] && QUALIFIER_IDENTIFIER="-$QUALIFIER"
 
+NVM_CMD="source $NVM_DIR/nvm.sh && nvm use"
+if [ "$PLATFORM" = "windows" ]; then
+    NVM_CMD="volta install node@`cat ../../../../OpenSearch-Dashboards/.nvmrc` && volta install yarn@`jq -r '.engines.yarn' ../../../../OpenSearch-Dashboards/package.json`"
+fi
+
 mkdir -p $OUTPUT/plugins
 PLUGIN_FOLDER=$(basename "$PWD")
 PLUGIN_NAME=customImportMapDashboards
 # TODO: [CLEANUP] Needed OpenSearch Dashboards git repo to build the required modules for plugins
 # This makes it so there is a dependency on having Dashboards pulled already.
 cp -r ../$PLUGIN_FOLDER/ ../../../../OpenSearch-Dashboards/plugins
 echo "BUILD MODULES FOR $PLUGIN_NAME"
-(cd ../../../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && yarn osd bootstrap)
+(cd ../../../../OpenSearch-Dashboards && eval $NVM_CMD && yarn osd bootstrap)
 echo "BUILD RELEASE ZIP FOR $PLUGIN_NAME"
-(cd ../../../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && cd plugins/$PLUGIN_FOLDER && yarn plugin-helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER)
+(cd ../../../../OpenSearch-Dashboards && eval $NVM_CMD && cd plugins/$PLUGIN_FOLDER && yarn plugin-helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER)
 echo "COPY $PLUGIN_NAME.zip"
 cp -r ../../../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER/build/$PLUGIN_NAME-$VERSION$QUALIFIER_IDENTIFIER.zip $OUTPUT/plugins/
 rm -rf ../../../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER
diff --git a/scripts/components/ganttChartDashboards/build.sh b/scripts/components/ganttChartDashboards/build.sh
@@ -67,6 +67,11 @@ fi
 [ -z "$OUTPUT" ] && OUTPUT=artifacts
 [ ! -z "$QUALIFIER" ] && QUALIFIER_IDENTIFIER="-$QUALIFIER"
 
+NVM_CMD="source $NVM_DIR/nvm.sh && nvm use"
+if [ "$PLATFORM" = "windows" ]; then
+    NVM_CMD="volta install node@`cat ../../OpenSearch-Dashboards/.nvmrc` && volta install yarn@`jq -r '.engines.yarn' ../../OpenSearch-Dashboards/package.json`"
+fi
+
 mkdir -p $OUTPUT/plugins
 # For hybrid plugin it actually resides in 'ganttChartDashboards/gantt-chart'
 PLUGIN_FOLDER=$(basename "$PWD")
@@ -75,9 +80,9 @@ PLUGIN_NAME=$(basename $(dirname "$PWD"))
 # This makes it so there is a dependency on having Dashboards pulled already.
 cp -r ../$PLUGIN_FOLDER/ ../../OpenSearch-Dashboards/plugins
 echo "BUILD MODULES FOR $PLUGIN_NAME"
-(cd ../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && yarn osd bootstrap)
+(cd ../../OpenSearch-Dashboards && eval $NVM_CMD && yarn osd bootstrap)
 echo "BUILD RELEASE ZIP FOR $PLUGIN_NAME"
-(cd ../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && cd plugins/$PLUGIN_FOLDER && yarn plugin-helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER)
+(cd ../../OpenSearch-Dashboards && eval $NVM_CMD && cd plugins/$PLUGIN_FOLDER && yarn plugin-helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER)
 echo "COPY $PLUGIN_NAME.zip"
 cp -r ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER/build/$PLUGIN_NAME-$VERSION$QUALIFIER_IDENTIFIER.zip $OUTPUT/plugins/
-rm -rf ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER
+rm -rf ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER
diff --git a/scripts/components/notificationsDashboards/build.sh b/scripts/components/notificationsDashboards/build.sh
@@ -67,6 +67,11 @@ fi
 [ -z "$OUTPUT" ] && OUTPUT=artifacts
 [ ! -z "$QUALIFIER" ] && QUALIFIER_IDENTIFIER="-$QUALIFIER"
 
+NVM_CMD="source $NVM_DIR/nvm.sh && nvm use"
+if [ "$PLATFORM" = "windows" ]; then
+    NVM_CMD="volta install node@`cat ../../OpenSearch-Dashboards/.nvmrc` && volta install yarn@`jq -r '.engines.yarn' ../../OpenSearch-Dashboards/package.json`"
+fi
+
 mkdir -p $OUTPUT/plugins
 # For hybrid plugin it actually resides in 'queryWorkbenchDashboards/workbench'
 PLUGIN_FOLDER=$(basename "$PWD")
@@ -75,9 +80,9 @@ PLUGIN_NAME=$(basename $(dirname "$PWD"))
 # This makes it so there is a dependency on having Dashboards pulled already.
 cp -r ../$PLUGIN_FOLDER/ ../../OpenSearch-Dashboards/plugins
 echo "BUILD MODULES FOR $PLUGIN_NAME"
-(cd ../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && yarn osd bootstrap)
+(cd ../../OpenSearch-Dashboards && eval $NVM_CMD && yarn osd bootstrap)
 echo "BUILD RELEASE ZIP FOR $PLUGIN_NAME"
-(cd ../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && cd plugins/$PLUGIN_FOLDER && yarn plugin_helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER)
+(cd ../../OpenSearch-Dashboards && eval $NVM_CMD && cd plugins/$PLUGIN_FOLDER && yarn plugin_helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER)
 echo "COPY $PLUGIN_NAME.zip"
 cp -r ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER/build/$PLUGIN_NAME-$VERSION$QUALIFIER_IDENTIFIER.zip $OUTPUT/plugins/
-rm -rf ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER
+rm -rf ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER
diff --git a/scripts/components/observabilityDashboards/build.sh b/scripts/components/observabilityDashboards/build.sh
@@ -67,6 +67,11 @@ fi
 [ -z "$OUTPUT" ] && OUTPUT=artifacts
 [ ! -z "$QUALIFIER" ] && QUALIFIER_IDENTIFIER="-$QUALIFIER"
 
+NVM_CMD="source $NVM_DIR/nvm.sh && nvm use"
+if [ "$PLATFORM" = "windows" ]; then
+    NVM_CMD="volta install node@`cat ../../OpenSearch-Dashboards/.nvmrc` && volta install yarn@`jq -r '.engines.yarn' ../../OpenSearch-Dashboards/package.json`"
+fi
+
 mkdir -p $OUTPUT/plugins
 # For hybrid plugin it actually resides in 'observability/dashboards-observability'
 PLUGIN_FOLDER=$(basename "$PWD")
@@ -75,9 +80,9 @@ PLUGIN_NAME=$(basename $(dirname "$PWD"))
 # This makes it so there is a dependency on having Dashboards pulled already.
 cp -r ../$PLUGIN_FOLDER/ ../../OpenSearch-Dashboards/plugins
 echo "BUILD MODULES FOR $PLUGIN_NAME"
-(cd ../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && yarn osd bootstrap)
+(cd ../../OpenSearch-Dashboards && eval $NVM_CMD && yarn osd bootstrap)
 echo "BUILD RELEASE ZIP FOR $PLUGIN_NAME"
-(cd ../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && cd plugins/$PLUGIN_FOLDER && yarn plugin_helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER)
+(cd ../../OpenSearch-Dashboards && eval $NVM_CMD && cd plugins/$PLUGIN_FOLDER && yarn plugin_helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER)
 echo "COPY $PLUGIN_NAME.zip"
 cp -r ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER/build/$PLUGIN_NAME-$VERSION$QUALIFIER_IDENTIFIER.zip $OUTPUT/plugins/
 rm -rf ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER
diff --git a/scripts/components/queryWorkbenchDashboards/build.sh b/scripts/components/queryWorkbenchDashboards/build.sh
@@ -67,6 +67,11 @@ fi
 [ -z "$OUTPUT" ] && OUTPUT=artifacts
 [ ! -z "$QUALIFIER" ] && QUALIFIER_IDENTIFIER="-$QUALIFIER"
 
+NVM_CMD="source $NVM_DIR/nvm.sh && nvm use"
+if [ "$PLATFORM" = "windows" ]; then
+    NVM_CMD="volta install node@`cat ../../OpenSearch-Dashboards/.nvmrc` && volta install yarn@`jq -r '.engines.yarn' ../../OpenSearch-Dashboards/package.json`"
+fi
+
 mkdir -p $OUTPUT/plugins
 # For hybrid plugin it actually resides in 'queryWorkbenchDashboards/workbench'
 PLUGIN_FOLDER=$(basename "$PWD")
@@ -75,9 +80,9 @@ PLUGIN_NAME=$(basename $(dirname "$PWD"))
 # This makes it so there is a dependency on having Dashboards pulled already.
 cp -r ../$PLUGIN_FOLDER/ ../../OpenSearch-Dashboards/plugins
 echo "BUILD MODULES FOR $PLUGIN_NAME"
-(cd ../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && yarn osd bootstrap)
+(cd ../../OpenSearch-Dashboards && eval $NVM_CMD && yarn osd bootstrap)
 echo "BUILD RELEASE ZIP FOR $PLUGIN_NAME"
-(cd ../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && cd plugins/$PLUGIN_FOLDER && yarn plugin_helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER)
+(cd ../../OpenSearch-Dashboards && eval $NVM_CMD && cd plugins/$PLUGIN_FOLDER && yarn plugin_helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER)
 echo "COPY $PLUGIN_NAME.zip"
 cp -r ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER/build/$PLUGIN_NAME-$VERSION$QUALIFIER_IDENTIFIER.zip $OUTPUT/plugins/
-rm -rf ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER
+rm -rf ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER
diff --git a/scripts/components/reportsDashboards/build.sh b/scripts/components/reportsDashboards/build.sh
@@ -94,7 +94,7 @@ esac
 
 NVM_CMD="source $NVM_DIR/nvm.sh && nvm use"
 if [ "$PLATFORM" = "windows" ]; then
-    NVM_CMD="volta install node@`cat ../../OpenSearch-Dashboards/.nvmrc` && volta install yarn"
+    NVM_CMD="volta install node@`cat ../../OpenSearch-Dashboards/.nvmrc` && volta install yarn@`jq -r '.engines.yarn' ../../OpenSearch-Dashboards/package.json`"
 fi
 
 CHROMIUM_URL="https://github.com/opensearch-project/dashboards-reports/releases/download/chromium-1.12.0.0/$CHROMIUM_TARGET"

diff --git a/scripts/default/opensearch-dashboards/build.sh b/scripts/default/opensearch-dashboards/build.sh
@@ -67,15 +67,20 @@ fi
 [ -z "$OUTPUT" ] && OUTPUT=artifacts
 [ ! -z "$QUALIFIER" ] && QUALIFIER_IDENTIFIER="-$QUALIFIER"
 
+NVM_CMD="source $NVM_DIR/nvm.sh && nvm use"
+if [ "$PLATFORM" = "windows" ]; then
+    NVM_CMD="volta install node@`cat ../OpenSearch-Dashboards/.nvmrc` && volta install yarn@`jq -r '.engines.yarn' ../OpenSearch-Dashboards/package.json`"
+fi
+
 mkdir -p $OUTPUT/plugins
 PLUGIN_NAME=$(basename "$PWD")
 # TODO: [CLEANUP] Needed OpenSearch Dashboards git repo to build the required modules for plugins
 # This makes it so there is a dependency on having Dashboards pulled already.
 cp -r ../$PLUGIN_NAME/ ../OpenSearch-Dashboards/plugins
 echo "BUILD MODULES FOR $PLUGIN_NAME"
-(cd ../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && yarn osd bootstrap)
+(cd ../OpenSearch-Dashboards && eval $NVM_CMD && yarn osd bootstrap)
 echo "BUILD RELEASE ZIP FOR $PLUGIN_NAME"
-(cd ../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && cd plugins/$PLUGIN_NAME && yarn plugin-helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER)
+(cd ../OpenSearch-Dashboards && eval $NVM_CMD && cd plugins/$PLUGIN_NAME && yarn plugin-helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER)
 echo "COPY $PLUGIN_NAME.zip"
 cp -r ../OpenSearch-Dashboards/plugins/$PLUGIN_NAME/build/$PLUGIN_NAME-$VERSION$QUALIFIER_IDENTIFIER.zip $OUTPUT/plugins/
-rm -rf ../OpenSearch-Dashboards/plugins/$PLUGIN_NAME
+rm -rf ../OpenSearch-Dashboards/plugins/$PLUGIN_NAME