Onboarding Jenkins prod docker images to github ci check (opensearch-…

…project#1538) (opensearch-project#1566) * Onboarding Jenkins prod docker images to github ci check Signed-off-by: Peter Zhu <[email protected]> * Add suggested changes in the comments Signed-off-by: Peter Zhu <[email protected]> --------- Signed-off-by: Peter Zhu <[email protected]> (cherry picked from commit 96c8d4d) Co-authored-by: Peter Zhu <[email protected]>
Zhangxunmt · Oct 30, 2023 · bb10a22 · bb10a22
1 parent 3ceb266
commit bb10a22
Showing 1 changed file with 82 additions and 31 deletions.
diff --git a/.github/workflows/CI-workflow.yml b/.github/workflows/CI-workflow.yml
@@ -14,15 +14,29 @@ permissions:
   contents: read
 
 jobs:
-  Build-ml:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  Build-ml-linux:
+    needs: Get-CI-Image-Tag
     strategy:
       matrix:
         java: [11, 17]
 
-    name: Build and Test MLCommons Plugin
+    name: Build and Test MLCommons Plugin on linux
     if: github.repository == 'opensearch-project/ml-commons'
     environment: ml-commons-cicd-env
+    outputs:
+      build-test-linux: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
 
     steps:
       - name: Setup Java ${{ matrix.java }}
@@ -41,43 +55,86 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Build and Run Tests
+        id: step-build-test-linux
         run: |
-          export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
-          export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
-          echo "::add-mask::$OPENAI_KEY"
-          echo "::add-mask::$COHERE_KEY"
-          ./gradlew build
-      - name: Publish to Maven Local
-        run: |
-          ./gradlew publishToMavenLocal
-      - name: Multi Nodes Integration Testing
-        run: |
-          export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
-          export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
-          echo "::add-mask::$OPENAI_KEY"
-          echo "::add-mask::$COHERE_KEY"
-          ./gradlew integTest -PnumNodes=3
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c 'whoami && java -version && 
+                               export OPENAI_KEY=`aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text` &&
+                               export COHERE_KEY=`aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text` &&
+                               echo "::add-mask::$OPENAI_KEY" &&
+                               echo "::add-mask::$COHERE_KEY" &&
+                               echo "build and run tests" && ./gradlew build &&
+                               echo "Publish to Maven Local" && ./gradlew publishToMavenLocal &&
+                               echo "Multi Nodes Integration Testing" && ./gradlew integTest -PnumNodes=3'
+          plugin=`basename $(ls plugin/build/distributions/*.zip)`
+          echo $plugin
+          mv -v plugin/build/distributions/$plugin ./
+          echo "build-test-linux=$plugin" >> $GITHUB_OUTPUT
+
+      - name: Upload Coverage Report
+        uses: codecov/codecov-action@v1
+        with:
+          flags: ml-commons
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ml-plugin-linux-${{ matrix.java }}
+          path: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
+          if-no-files-found: error
+
+
+  Test-ml-linux-docker:
+    needs: Build-ml-linux
+    strategy:
+      matrix:
+        java: [11, 17, 20]
+
+    name: Test MLCommons Plugin on linux docker
+    if: github.repository == 'opensearch-project/ml-commons'
+    environment: ml-commons-cicd-env
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.ML_ROLE }}
+          aws-region: us-west-2
+
+      - name: Checkout MLCommons
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: ml-plugin-linux-${{ matrix.java }}
 
       - name: Pull and Run Docker
         run: |
-          plugin=`basename $(ls plugin/build/distributions/*.zip)`
+          plugin=${{ needs.Build-ml-linux.outputs.build-test-linux }}
           version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3`
           plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4`
           qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1`
-          candidate_version=`echo $plugin|awk -F- '{print $5}'| cut -d. -f 1-1`
-          docker_version=$version-$qualifier
 
-          [[ -z $candidate_version ]] && candidate_version=$qualifier && qualifier=""
+          if [ -n "$qualifier" ] && [ "$qualifier" != "SNAPSHOT" ]; then
+              qualifier=-${qualifier}
+          else
+              qualifier=""
+          fi
 
-          echo plugin version plugin_version qualifier candidate_version docker_version
-          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($candidate_version) ($docker_version)"
+          docker_version=$version$qualifier
+
+          echo plugin version plugin_version qualifier docker_version
+          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"
+
+          pwd && ls -l ./$plugin
 
-          cd ..
           if docker pull opensearchstaging/opensearch:$docker_version
           then
             echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
             echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-ml ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-ml; fi" >> Dockerfile
-            echo "ADD ml-commons/plugin/build/distributions/$plugin /tmp/" >> Dockerfile
+            echo "COPY $plugin /tmp/" >> Dockerfile
             echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
             docker build -t opensearch-ml:test .
             echo "imagePresent=true" >> $GITHUB_ENV
@@ -107,12 +164,6 @@ jobs:
             ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster"
           fi
 
-      - name: Upload Coverage Report
-        uses: codecov/codecov-action@v1
-        with:
-          flags: ml-commons
-          token: ${{ secrets.CODECOV_TOKEN }}
-
   Build-ml-windows:
     strategy:
       matrix: