Backport 2.x #1092 #1093 (#1094)

* Merged from main Signed-off-by: Peter Zhu <[email protected]> * Fix the bwc test version retrieval (#1093) * Onboarding jenkins prod docker images to github actions Signed-off-by: Peter Zhu <[email protected]> * Add temurin Signed-off-by: Peter Zhu <[email protected]> * Add temurin Signed-off-by: Peter Zhu <[email protected]> * Add more Signed-off-by: Peter Zhu <[email protected]> * Add more Signed-off-by: Peter Zhu <[email protected]> * Add more Signed-off-by: Peter Zhu <[email protected]> --------- Signed-off-by: Peter Zhu <[email protected]> --------- Signed-off-by: Peter Zhu <[email protected]>
opensearch-project · Oct 30, 2023 · 7267369 · 7267369
1 parent cfb5fa0
commit 7267369
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 12 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -8,26 +8,40 @@ on:
       - "*"
 
 jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
   Build-ad:
+    needs: Get-CI-Image-Tag
     strategy:
       matrix:
         java: [17]
       fail-fast: false
 
     name: Run Anomaly detection model performance benchmark
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
 
     steps:
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v3
         with:
+          distribution: 'temurin'
           java-version: ${{ matrix.java }}
 
       # anomaly-detection
       - name: Checkout AD
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
       - name: Build and Run Tests
         run: |
-          ./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true
-          ./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew ':test' --tests 'org.opensearch.ad.ml.HCADModelPerfTests' -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true"
+          su `id -un 1000` -c "./gradlew integTest --tests 'org.opensearch.ad.e2e.SingleStreamModelPerfIT' -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true"
diff --git a/.github/workflows/test_build_multi_platform.yml b/.github/workflows/test_build_multi_platform.yml
@@ -8,6 +8,11 @@ on:
       - "*"
 
 jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
   Build-ad-windows:
     strategy:
       matrix:
@@ -20,6 +25,7 @@ jobs:
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v3
         with:
+          distribution: 'temurin'
           java-version: ${{ matrix.java }}
       - name: Checkout Anomaly Detection
         uses: actions/checkout@v4
@@ -38,22 +44,64 @@ jobs:
         with:
           file: ./build/reports/jacoco/test/jacocoTestReport.xml
           flags: plugin
-  Build-ad:
+
+  Build-ad-linux:
+    needs: Get-CI-Image-Tag
     strategy:
       matrix:
         java: [11,17]
-        os: [ubuntu-latest, macos-latest]
+      fail-fast: false
+    name: Build and Test Anomaly detection Plugin
+    runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    env:
+      JENKINS_URL: build.ci.opensearch.org
+
+    steps:
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+
+      - name: Checkout AD
+        uses: actions/checkout@v3
+
+      - name: Assemble / build / mavenlocal / integTest
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew assemble &&
+                               ./gradlew build &&
+                               ./gradlew publishToMavenLocal &&
+                               ./gradlew integTest -PnumNodes=3"
+      - name: Upload Coverage Report
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./build/reports/jacoco/test/jacocoTestReport.xml
+          flags: plugin
+
+  Build-ad-macos:
+    strategy:
+      matrix:
+        java: [11,17,20]
       fail-fast: false
 
     name: Build and Test Anomaly detection Plugin
-    runs-on: ${{ matrix.os }}
+    runs-on: macos-latest
     env:
       JENKINS_URL: build.ci.opensearch.org
 
     steps:
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v3
         with:
+          distribution: 'temurin'
           java-version: ${{ matrix.java }}
 
       - name: Checkout AD

diff --git a/.github/workflows/test_bwc.yml b/.github/workflows/test_bwc.yml
@@ -8,38 +8,53 @@ on:
       - "*"
 
 jobs:
-  Build-ad:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  Build-ad-linux:
+    needs: Get-CI-Image-Tag
     strategy:
       matrix:
         java: [11,17]
       fail-fast: false
 
     name: Test Anomaly detection BWC
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
 
     steps:
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v3
         with:
+          distribution: 'temurin'
           java-version: ${{ matrix.java }}
 
       # anomaly-detection
       - name: Checkout AD
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
       - name: Assemble anomaly-detection
         run: |
           plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'`
+          chown -R 1000:1000 `pwd`
           echo plugin_version $plugin_version
-          ./gradlew assemble
+          su `id -un 1000` -c "./gradlew assemble"
           echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
-          mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
+          su `id -un 1000` -c "mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version"
           echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
           ls ./build/distributions/
-          cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
+          su `id -un 1000` -c "cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version"
           echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
           ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
       - name: Run AD Backwards Compatibility Tests
         run: |
           echo "Running backwards compatibility tests ..."
-          ./gradlew bwcTestSuite -Dtests.security.manager=false
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew bwcTestSuite -Dtests.security.manager=false"
diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml
@@ -23,6 +23,7 @@ jobs:
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v3
         with:
+          distribution: 'temurin'
           java-version: ${{ matrix.java }}
 
       # anomaly-detection