From 5d03acb2ddb2a7300cb1c70feb83284986449458 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Mon, 28 Aug 2023 15:10:16 -0700 Subject: [PATCH 1/7] Fixed the github actions to use the image used by OpenSearch-build repo for building the distribution Signed-off-by: Navneet Verma --- .github/workflows/CI.yml | 45 ++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 31d16f21f..8dc85091d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -16,27 +16,52 @@ jobs: strategy: matrix: java: [11, 17] - os: [ubuntu-latest, macos-latest] - name: Build and Test k-NN Plugin - runs-on: ${{ matrix.os }} + name: Build and Test k-NN Plugin on Ubnutu + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + image: public.ecr.aws/opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v4 + options: --user root steps: - name: Checkout k-NN uses: actions/checkout@v1 + with: + submodules: true - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - - name: Install dependencies on ubuntu - if: startsWith(matrix.os,'ubuntu') + - name: Run build run: | - sudo apt-get install libopenblas-dev gfortran -y + ./gradlew build + + - name: Upload Coverage Report + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + Build-k-NN-MacOS: + strategy: + matrix: + java: [ 11, 17 ] + + name: Build and Test k-NN Plugin on MacOS + runs-on: macos-latest + + steps: + - name: Checkout k-NN + uses: actions/checkout@v1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} - name: Install dependencies on macos - if: startsWith(matrix.os, 'macos') run: | brew reinstall gcc export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran @@ -45,12 +70,6 @@ jobs: run: | ./gradlew build - - name: Upload Coverage Report - if: startsWith(matrix.os,'ubuntu') - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - Build-k-NN-Windows: strategy: matrix: From 8fc7cf70c3f5970bdb4876c0da6b1dae09c918b3 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 29 Aug 2023 15:23:28 -0400 Subject: [PATCH 2/7] Retry switch user on the CI using prod docker images Signed-off-by: Peter Zhu --- .github/workflows/CI.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8dc85091d..2272c54fd 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -37,7 +37,10 @@ jobs: - name: Run build run: | - ./gradlew build + pwd + chown -R opensearch.opensearch ./* `pwd` + ls -l ./ ../ + su opensearch -c "whoami && java -version && ./gradlew build" - name: Upload Coverage Report uses: codecov/codecov-action@v1 From c2e7f90d4b3dcdb19a8f0c85c12d859ceaa1657c Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 29 Aug 2023 16:24:51 -0400 Subject: [PATCH 3/7] Cleanup debug commands Signed-off-by: Peter Zhu --- .github/workflows/CI.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2272c54fd..9eecef393 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -37,9 +37,7 @@ jobs: - name: Run build run: | - pwd chown -R opensearch.opensearch ./* `pwd` - ls -l ./ ../ su opensearch -c "whoami && java -version && ./gradlew build" - name: Upload Coverage Report From 5d69b149b5581682960d2b519b0b79f53446ca84 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Tue, 29 Aug 2023 13:34:49 -0700 Subject: [PATCH 4/7] Added some comments for future reference Signed-off-by: Navneet Verma --- .github/workflows/CI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9eecef393..8fe838e68 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -22,6 +22,7 @@ jobs: container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution image: public.ecr.aws/opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v4 + # need to add this option so that docker image can be pulled and run. options: --user root steps: @@ -36,6 +37,7 @@ jobs: java-version: ${{ matrix.java }} - name: Run build + # switching the user, as OpenSearch cluster cannot be started as root. run: | chown -R opensearch.opensearch ./* `pwd` su opensearch -c "whoami && java -version && ./gradlew build" From d2cc777c57c86b643e59ff224d65c225c88c7d5a Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 29 Aug 2023 16:40:13 -0400 Subject: [PATCH 5/7] Add more tweaks Signed-off-by: Peter Zhu --- .github/workflows/CI.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8fe838e68..6bf84dc36 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -21,8 +21,9 @@ jobs: runs-on: ubuntu-latest container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time image: public.ecr.aws/opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v4 - # need to add this option so that docker image can be pulled and run. + # need to switch to root so that github actions can install runner binary on container without permission issues. options: --user root steps: @@ -37,9 +38,9 @@ jobs: java-version: ${{ matrix.java }} - name: Run build - # switching the user, as OpenSearch cluster cannot be started as root. + # switching the user, as OpenSearch cluster cannot only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | - chown -R opensearch.opensearch ./* `pwd` + chown -R opensearch.opensearch `pwd` su opensearch -c "whoami && java -version && ./gradlew build" - name: Upload Coverage Report From 3b16541232474e0967dd7b930ad7ed64ea0620ee Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 29 Aug 2023 16:41:25 -0400 Subject: [PATCH 6/7] Add more tweaks Signed-off-by: Peter Zhu --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6bf84dc36..b08b48135 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -38,7 +38,7 @@ jobs: java-version: ${{ matrix.java }} - name: Run build - # switching the user, as OpenSearch cluster cannot only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. + # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | chown -R opensearch.opensearch `pwd` su opensearch -c "whoami && java -version && ./gradlew build" From 3fc9e4852744c53c2740bf2b6d7e480e4479b0fb Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 29 Aug 2023 16:57:07 -0400 Subject: [PATCH 7/7] More tweaks Signed-off-by: Peter Zhu --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b08b48135..d16287dd3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -17,7 +17,7 @@ jobs: matrix: java: [11, 17] - name: Build and Test k-NN Plugin on Ubnutu + name: Build and Test k-NN Plugin on Linux runs-on: ubuntu-latest container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution