From 88fd874dbfc4c82136456a4668973f43d28fe645 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 2 Nov 2023 14:25:52 -0400 Subject: [PATCH 1/5] Onboard prod jenkins docker images to github actions Signed-off-by: Peter Zhu --- .github/workflows/cd.yml | 68 ------------------------------------ .github/workflows/gradle.yml | 38 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 68 deletions(-) delete mode 100644 .github/workflows/cd.yml diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml deleted file mode 100644 index a4aea7f2..00000000 --- a/.github/workflows/cd.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: CD - -on: - push: - tags: - - 'v*' - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout Performance Analyzer package - uses: actions/checkout@v2 - with: - path: ./tmp/pa - - name: Set up JDK 1.12 - uses: actions/setup-java@v1 - with: - java-version: 1.12 - - name: Build Artifacts - working-directory: ./tmp/pa - run: | - ./gradlew build buildDeb buildRpm --refresh-dependencies -Dbuild.snapshot=false -Dperformance-analyzer-rca.build=true -Dperformance-analyzer-rca.branch=main -x javadoc - mkdir artifacts - artifact=`ls build/distributions/*.zip` - rpm_artifact=`ls build/distributions/*.rpm` - deb_artifact=`ls build/distributions/*.deb` - cp $artifact artifacts/ - cp $rpm_artifact artifacts/ - cp $deb_artifact artifacts/ - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Upload Artifacts to S3 - working-directory: ./tmp/pa - shell: bash - run: | - zip=`ls artifacts/*.zip` - rpm=`ls artifacts/*.rpm` - deb=`ls artifacts/*.deb` - - # Inject the build number before the suffix - zip_outfile=`basename ${zip%.zip}-build-${GITHUB_RUN_NUMBER}.zip` - rpm_outfile=`basename ${rpm%.rpm}-build-${GITHUB_RUN_NUMBER}.rpm` - deb_outfile=`basename ${deb%.deb}-build-${GITHUB_RUN_NUMBER}.deb` - - s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-plugins/performance-analyzer/" - - echo "Copying ${zip} to ${s3_prefix}${zip_outfile}" - aws s3 cp --quiet $zip ${s3_prefix}${zip_outfile} - - echo "Copying ${rpm} to ${s3_prefix}${rpm_outfile}" - aws s3 cp --quiet $rpm ${s3_prefix}${rpm_outfile} - - echo "Copying ${deb} to ${s3_prefix}${deb_outfile}" - aws s3 cp --quiet $deb ${s3_prefix}${deb_outfile} - - - name: Upload Workflow Artifacts - uses: actions/upload-artifact@v1 - with: - name: artifacts - path: ./tmp/pa/artifacts/ diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index ccc9aa2a..dc85144f 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -9,6 +9,44 @@ on: - "*" jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + assemble_pa_pkg: + needs: Get-CI-Image-Tag + strategy: + matrix: + java: + - 11 + - 17 + fail-fast: false + runs-on: [ubuntu-latest] + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + name: Building PA package + steps: + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: ${{matrix.java}} + # Performance Analyzer in ./tmp/performance-analyzer + - name: Checkout Performance Analyzer package + uses: actions/checkout@v2 + with: + path: ./tmp/performance-analyzer + - name: Assemble PA jar for BWC tests + working-directory: ./tmp/performance-analyzer + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew assemble" + build_pa_pkg: strategy: matrix: From e7b6d4341e4a284550889fa19368f0971ec44505 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 2 Nov 2023 14:30:19 -0400 Subject: [PATCH 2/5] Add more Signed-off-by: Peter Zhu --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index dc85144f..5f98f8a4 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -30,7 +30,7 @@ jobs: # need to switch to root so that github actions can install runner binary on container without permission issues. options: --user root - name: Building PA package + name: Assembling PA package steps: - name: Set up JDK uses: actions/setup-java@v1 From b3a3fdd32a859aa33d6271500882aa6963e3db70 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 2 Nov 2023 14:35:25 -0400 Subject: [PATCH 3/5] Add more Signed-off-by: Peter Zhu --- .github/workflows/gradle.yml | 124 +++++++++++++++++------------------ 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 5f98f8a4..49b258a8 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -14,7 +14,7 @@ jobs: with: product: opensearch - assemble_pa_pkg: + build_pa_pkg_on_prod_docker: needs: Get-CI-Image-Tag strategy: matrix: @@ -30,7 +30,7 @@ jobs: # need to switch to root so that github actions can install runner binary on container without permission issues. options: --user root - name: Assembling PA package + name: Building PA package on prod docker steps: - name: Set up JDK uses: actions/setup-java@v1 @@ -39,67 +39,67 @@ jobs: # Performance Analyzer in ./tmp/performance-analyzer - name: Checkout Performance Analyzer package uses: actions/checkout@v2 - with: - path: ./tmp/performance-analyzer + #with: + # path: ./tmp/performance-analyzer - name: Assemble PA jar for BWC tests - working-directory: ./tmp/performance-analyzer + #working-directory: ./tmp/performance-analyzer run: | chown -R 1000:1000 `pwd` - su `id -un 1000` -c "./gradlew assemble" + su `id -un 1000` -c "./gradlew build -x test" - build_pa_pkg: - strategy: - matrix: - java: - - 11 - - 17 - fail-fast: false - runs-on: [ubuntu-latest] - name: Building PA package - steps: - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: ${{matrix.java}} - # Performance Analyzer in ./tmp/performance-analyzer - - name: Checkout Performance Analyzer package - uses: actions/checkout@v2 - with: - path: ./tmp/performance-analyzer - # Explicitly set the docker-compose program path so that our build scripts in RCA can run the program - # This is necessary because of the Github Actions environment and the workingDir of the Gradle environment - - name: Set docker-compose path - run: DOCKER_COMPOSE_LOCATION=$(which docker-compose) - # Set the vm.max_map_count system property to the minimum required to run OpenSearch - - name: Set vm.max_map_count - run: sudo sysctl -w vm.max_map_count=262144 - - name: Build Performance Analyzer and run Unit Tests - working-directory: ./tmp/performance-analyzer - run: | - ./gradlew build -Dperformance-analyzer-rca.build=true \ - -Dperformance-analyzer-rca.repo="https://github.com/opensearch-project/performance-analyzer-rca.git" \ - -Dperformance-analyzer-rca.branch=main - - name: Assemble PA jar for BWC tests - working-directory: ./tmp/performance-analyzer - run: | - ./gradlew assemble - version=`./gradlew properties -q | grep "opensearch_version:" | awk '{print $2}'` - IFS='-' read -r -a version_array <<< "$version" - plugin_version="${version_array[0]}.0"; for entry in ${version_array[@]:1}; do plugin_version+="-$entry"; done - echo $plugin_version - mkdir -p ./src/test/resources/org/opensearch/performanceanalyzer/bwc/$plugin_version - cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/performanceanalyzer/bwc/$plugin_version - - name: Generate Jacoco coverage report - working-directory: ./tmp/performance-analyzer - run: ./gradlew jacocoTestReport - - name: Upload coverage report - working-directory: ./tmp/performance-analyzer - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - run: bash <(curl -s https://codecov.io/bash) -f ./build/reports/jacoco/test/jacocoTestReport.xml - - name: Run Integration Tests - working-directory: ./tmp/performance-analyzer - run: ./gradlew integTest -Dtests.enableIT -Dtests.useDockerCluster - - name: Run PerformanceAnalzyer Backwards Compatibility Tests - working-directory: ./tmp/performance-analyzer - run: ./gradlew bwcTestSuite -Dtests.security.manager=false +# build_pa_pkg: +# strategy: +# matrix: +# java: +# - 11 +# - 17 +# fail-fast: false +# runs-on: [ubuntu-latest] +# name: Building PA package +# steps: +# - name: Set up JDK +# uses: actions/setup-java@v1 +# with: +# java-version: ${{matrix.java}} +# # Performance Analyzer in ./tmp/performance-analyzer +# - name: Checkout Performance Analyzer package +# uses: actions/checkout@v2 +# with: +# path: ./tmp/performance-analyzer +# # Explicitly set the docker-compose program path so that our build scripts in RCA can run the program +# # This is necessary because of the Github Actions environment and the workingDir of the Gradle environment +# - name: Set docker-compose path +# run: DOCKER_COMPOSE_LOCATION=$(which docker-compose) +# # Set the vm.max_map_count system property to the minimum required to run OpenSearch +# - name: Set vm.max_map_count +# run: sudo sysctl -w vm.max_map_count=262144 +# - name: Build Performance Analyzer and run Unit Tests +# working-directory: ./tmp/performance-analyzer +# run: | +# ./gradlew build -Dperformance-analyzer-rca.build=true \ +# -Dperformance-analyzer-rca.repo="https://github.com/opensearch-project/performance-analyzer-rca.git" \ +# -Dperformance-analyzer-rca.branch=main +# - name: Assemble PA jar for BWC tests +# working-directory: ./tmp/performance-analyzer +# run: | +# ./gradlew assemble +# version=`./gradlew properties -q | grep "opensearch_version:" | awk '{print $2}'` +# IFS='-' read -r -a version_array <<< "$version" +# plugin_version="${version_array[0]}.0"; for entry in ${version_array[@]:1}; do plugin_version+="-$entry"; done +# echo $plugin_version +# mkdir -p ./src/test/resources/org/opensearch/performanceanalyzer/bwc/$plugin_version +# cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/performanceanalyzer/bwc/$plugin_version +# - name: Generate Jacoco coverage report +# working-directory: ./tmp/performance-analyzer +# run: ./gradlew jacocoTestReport +# - name: Upload coverage report +# working-directory: ./tmp/performance-analyzer +# env: +# CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} +# run: bash <(curl -s https://codecov.io/bash) -f ./build/reports/jacoco/test/jacocoTestReport.xml +# - name: Run Integration Tests +# working-directory: ./tmp/performance-analyzer +# run: ./gradlew integTest -Dtests.enableIT -Dtests.useDockerCluster +# - name: Run PerformanceAnalzyer Backwards Compatibility Tests +# working-directory: ./tmp/performance-analyzer +# run: ./gradlew bwcTestSuite -Dtests.security.manager=false From 0da707455018ee8fb56c046974b8994c43646dfa Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 2 Nov 2023 14:42:45 -0400 Subject: [PATCH 4/5] Add more Signed-off-by: Peter Zhu --- .github/workflows/gradle.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 49b258a8..e32292b6 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -41,11 +41,13 @@ jobs: uses: actions/checkout@v2 #with: # path: ./tmp/performance-analyzer - - name: Assemble PA jar for BWC tests + - name: Build PA with same command on prod docker #working-directory: ./tmp/performance-analyzer run: | chown -R 1000:1000 `pwd` - su `id -un 1000` -c "./gradlew build -x test" + su `id -un 1000` -c './gradlew build -Dperformance-analyzer-rca.build=true \ + -Dperformance-analyzer-rca.repo="https://github.com/opensearch-project/performance-analyzer-rca.git" \ + -Dperformance-analyzer-rca.branch=main' # build_pa_pkg: # strategy: From 4204c05d500f73818f2c7fc2e95eff5cbd0f8df4 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 2 Nov 2023 15:02:48 -0400 Subject: [PATCH 5/5] Add more Signed-off-by: Peter Zhu --- .github/workflows/gradle.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index e32292b6..615eb29f 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -44,10 +44,10 @@ jobs: - name: Build PA with same command on prod docker #working-directory: ./tmp/performance-analyzer run: | - chown -R 1000:1000 `pwd` - su `id -un 1000` -c './gradlew build -Dperformance-analyzer-rca.build=true \ + #chown -R 1000:1000 `pwd` + ./gradlew build -Dperformance-analyzer-rca.build=true \ -Dperformance-analyzer-rca.repo="https://github.com/opensearch-project/performance-analyzer-rca.git" \ - -Dperformance-analyzer-rca.branch=main' + -Dperformance-analyzer-rca.branch=main # build_pa_pkg: # strategy: