Merge pull request #2404 from peterzhuamazon/issues-2248-1

Onboard jenkins prod docker images in github actions
opensearch-project · Nov 1, 2023 · 6cd2db0 · 6cd2db0
2 parents 940980e + 6b2f865
commit 6cd2db0
Show file tree

Hide file tree

Showing 5 changed files with 182 additions and 54 deletions.
diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml
diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml
@@ -10,11 +10,56 @@ on:
       - '.github/workflows/integ-tests-with-security.yml'
 
 jobs:
-  security-it:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  security-it-linux:
+    needs: Get-CI-Image-Tag
+    strategy:
+      fail-fast: false
+      matrix:
+        java: [ 11, 17 ]
+
+    runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up JDK ${{ matrix.java }}
+      uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin'
+        java-version: ${{ matrix.java }}
+
+    - name: Build with Gradle
+      run: |
+        chown -R 1000:1000 `pwd`
+        su `id -un 1000` -c "./gradlew integTestWithSecurity"
+
+    - name: Upload test reports
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v2
+      continue-on-error: true
+      with:
+        name: test-reports-${{ matrix.os }}-${{ matrix.java }}
+        path: |
+          integ-test/build/reports/**
+          integ-test/build/testclusters/*/logs/*
+          integ-test/build/testclusters/*/config/*
+
+  security-it-windows-macos:
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu-latest, windows-latest, macos-latest ]
+        os: [ windows-latest, macos-latest ]
         java: [ 11, 17 ]
 
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/sql-pitest.yml b/.github/workflows/sql-pitest.yml
@@ -11,13 +11,25 @@ run-name:
   ${{ inputs.name == '' && format('{0} @ {1}', github.ref_name, github.sha) || inputs.name }}
 
 jobs:
-  build:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  build-linux:
+    needs: Get-CI-Image-Tag
     strategy:
       matrix:
         java:
           - 11
           - 17
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
 
     steps:
     - uses: actions/checkout@v3
@@ -30,7 +42,8 @@ jobs:
 
     - name: PiTest with Gradle
       run: |
-        ./gradlew --continue :core:pitest :opensearch:pitest
+        chown -R 1000:1000 `pwd`
+        su `id -un 1000` -c "./gradlew --continue :core:pitest :opensearch:pitest"
 
     - name: Upload test reports
       if: always()

diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml
@@ -17,16 +17,95 @@ on:
       - '.github/workflows/sql-test-and-build-workflow.yml'
 
 jobs:
-  build:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  build-linux:
+    needs: Get-CI-Image-Tag
+    strategy:
+      # Run all jobs
+      fail-fast: false
+      matrix:
+        java:
+          - 11
+          - 17
+    runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up JDK ${{ matrix.java }}
+      uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin'
+        java-version: ${{ matrix.java }}
+
+    - name: Build with Gradle
+      run: |
+        chown -R 1000:1000 `pwd`
+        su `id -un 1000` -c "./gradlew --continue build"
+
+    - name: Run backward compatibility tests
+      run: |
+        chown -R 1000:1000 `pwd`
+        su `id -un 1000` -c "./scripts/bwctest.sh"
+
+    - name: Create Artifact Path
+      run: |
+        mkdir -p opensearch-sql-builds
+        cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/
+
+    # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action
+    - name: Upload SQL Coverage Report
+      if: ${{ always() }}
+      uses: codecov/codecov-action@v3
+      continue-on-error: true
+      with:
+        flags: sql-engine
+        token: ${{ secrets.CODECOV_TOKEN }}
+
+    - name: Upload Artifacts
+      uses: actions/upload-artifact@v2
+      continue-on-error: true
+      with:
+        name: opensearch-sql-ubuntu-latest-${{ matrix.java }}
+        path: opensearch-sql-builds
+
+    - name: Upload test reports
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v2
+      continue-on-error: true
+      with:
+        name: test-reports-ubuntu-latest-${{ matrix.java }}
+        path: |
+          sql/build/reports/**
+          ppl/build/reports/**
+          core/build/reports/**
+          common/build/reports/**
+          opensearch/build/reports/**
+          integ-test/build/reports/**
+          protocol/build/reports/**
+          legacy/build/reports/**
+          plugin/build/reports/**
+          doctest/build/testclusters/docTestCluster-0/logs/*
+          integ-test/build/testclusters/*/logs/*
+
+  build-windows-macos:
     strategy:
       # Run all jobs
       fail-fast: false
       matrix:
         entry:
-          - { os: ubuntu-latest, java: 11 }
           - { os: windows-latest, java: 11, os_build_args: -x doctest  -PbuildPlatform=windows }
           - { os: macos-latest, java: 11}
-          - { os: ubuntu-latest, java: 17 }
           - { os: windows-latest, java: 17, os_build_args: -x doctest -PbuildPlatform=windows }
           - { os: macos-latest, java: 17 }
     runs-on: ${{ matrix.entry.os }}
@@ -43,10 +122,6 @@ jobs:
     - name: Build with Gradle
       run: ./gradlew --continue build ${{ matrix.entry.os_build_args }}
 
-    - name: Run backward compatibility tests
-      if: ${{ matrix.entry.os == 'ubuntu-latest' }}
-      run: ./scripts/bwctest.sh
-
     - name: Create Artifact Path
       run: |
         mkdir -p opensearch-sql-builds

diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml
@@ -11,13 +11,25 @@ run-name:
   ${{ inputs.name == '' && format('{0} @ {1}', github.ref_name, github.sha) || inputs.name }}
 
 jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
   build:
+    needs: Get-CI-Image-Tag
     strategy:
       matrix:
         java:
           - 11
           - 17
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
 
     steps:
     - uses: actions/checkout@v3
@@ -31,35 +43,36 @@ jobs:
     - name: Run tests
       id: tests
       run: |
+        chown -R 1000:1000 `pwd`
         # Spotless
-        ./gradlew :opensearch:spotlessCheck              || echo "* Spotless failed for opensearch" > report.log
-        ./gradlew :sql:spotlessCheck                     || echo "* Spotless failed for sql" >> report.log
-        ./gradlew :ppl:spotlessCheck                     || echo "* Spotless failed for ppl" >> report.log
-        ./gradlew :core:spotlessCheck                    || echo "* Spotless failed for core" >> report.log
-        ./gradlew :common:spotlessCheck                  || echo "* Spotless failed for common" >> report.log
-        ./gradlew :legacy:spotlessCheck                  || echo "* Spotless failed for legacy" >> report.log
-        ./gradlew :protocol:spotlessCheck                || echo "* Spotless failed for protocol" >> report.log
-        ./gradlew :opensearch-sql-plugin:spotlessCheck   || echo "* Spotless failed for plugin" >> report.log
-        ./gradlew :integ-test:spotlessCheck              || echo "* Spotless failed for integ-test" >> report.log
+        su `id -un 1000` -c './gradlew :opensearch:spotlessCheck              || echo "* Spotless failed for opensearch" > report.log'
+        su `id -un 1000` -c './gradlew :sql:spotlessCheck                     || echo "* Spotless failed for sql" >> report.log'
+        su `id -un 1000` -c './gradlew :ppl:spotlessCheck                     || echo "* Spotless failed for ppl" >> report.log'
+        su `id -un 1000` -c './gradlew :core:spotlessCheck                    || echo "* Spotless failed for core" >> report.log'
+        su `id -un 1000` -c './gradlew :common:spotlessCheck                  || echo "* Spotless failed for common" >> report.log'
+        su `id -un 1000` -c './gradlew :legacy:spotlessCheck                  || echo "* Spotless failed for legacy" >> report.log'
+        su `id -un 1000` -c './gradlew :protocol:spotlessCheck                || echo "* Spotless failed for protocol" >> report.log'
+        su `id -un 1000` -c './gradlew :opensearch-sql-plugin:spotlessCheck   || echo "* Spotless failed for plugin" >> report.log'
+        su `id -un 1000` -c './gradlew :integ-test:spotlessCheck              || echo "* Spotless failed for integ-test" >> report.log'
         # Unit tests
-        ./gradlew :opensearch:test                        || echo "* Unit tests failed for opensearch" >> report.log
-        ./gradlew :ppl:test                               || echo "* Unit tests failed for sql" >> report.log
-        ./gradlew :sql:test                               || echo "* Unit tests failed for ppl" >> report.log
-        ./gradlew :core:test                              || echo "* Unit tests failed for core" >> report.log
-        ./gradlew :protocol:test                          || echo "* Unit tests failed for protocol" >> report.log
-        ./gradlew :opensearch-sql-plugin:test             || echo "* Unit tests failed for plugin" >> report.log
-        ./gradlew :legacy:test                            || echo "* Unit tests failed for legacy" >> report.log
+        su `id -un 1000` -c './gradlew :opensearch:test                        || echo "* Unit tests failed for opensearch" >> report.log'
+        su `id -un 1000` -c './gradlew :ppl:test                               || echo "* Unit tests failed for sql" >> report.log'
+        su `id -un 1000` -c './gradlew :sql:test                               || echo "* Unit tests failed for ppl" >> report.log'
+        su `id -un 1000` -c './gradlew :core:test                              || echo "* Unit tests failed for core" >> report.log'
+        su `id -un 1000` -c './gradlew :protocol:test                          || echo "* Unit tests failed for protocol" >> report.log'
+        su `id -un 1000` -c './gradlew :opensearch-sql-plugin:test             || echo "* Unit tests failed for plugin" >> report.log'
+        su `id -un 1000` -c './gradlew :legacy:test                            || echo "* Unit tests failed for legacy" >> report.log'
         # jacoco
-        ./gradlew :opensearch:jacocoTestCoverageVerification             || echo "* Jacoco failed for opensearch" >> report.log
-        ./gradlew :ppl:jacocoTestCoverageVerification                    || echo "* Jacoco failed for sql" >> report.log
-        ./gradlew :sql:jacocoTestCoverageVerification                    || echo "* Jacoco failed for ppl" >> report.log
-        ./gradlew :core:jacocoTestCoverageVerification                   || echo "* Jacoco failed for core" >> report.log
-        ./gradlew :protocol:jacocoTestCoverageVerification               || echo "* Jacoco failed for protocol" >> report.log
-        ./gradlew :opensearch-sql-plugin:jacocoTestCoverageVerification  || echo "* Jacoco failed for plugin" >> report.log
+        su `id -un 1000` -c './gradlew :opensearch:jacocoTestCoverageVerification             || echo "* Jacoco failed for opensearch" >> report.log'
+        su `id -un 1000` -c './gradlew :ppl:jacocoTestCoverageVerification                    || echo "* Jacoco failed for sql" >> report.log'
+        su `id -un 1000` -c './gradlew :sql:jacocoTestCoverageVerification                    || echo "* Jacoco failed for ppl" >> report.log'
+        su `id -un 1000` -c './gradlew :core:jacocoTestCoverageVerification                   || echo "* Jacoco failed for core" >> report.log'
+        su `id -un 1000` -c './gradlew :protocol:jacocoTestCoverageVerification               || echo "* Jacoco failed for protocol" >> report.log'
+        su `id -un 1000` -c './gradlew :opensearch-sql-plugin:jacocoTestCoverageVerification  || echo "* Jacoco failed for plugin" >> report.log'
         # Misc tests
-        ./gradlew :integ-test:integTest                   || echo "* Integration test failed" >> report.log
-        ./gradlew :doctest:doctest                        || echo "* Doctest failed" >> report.log
-        ./scripts/bwctest.sh                              || echo "* Backward compatibility test failed" >> report.log
+        su `id -un 1000` -c './gradlew :integ-test:integTest                   || echo "* Integration test failed" >> report.log'
+        su `id -un 1000` -c './gradlew :doctest:doctest                        || echo "* Doctest failed" >> report.log'
+        su `id -un 1000` -c './scripts/bwctest.sh                              || echo "* Backward compatibility test failed" >> report.log'
 
     - name: Verify test results
       run: |