diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml deleted file mode 100644 index cf30ea89dc..0000000000 --- a/.github/workflows/dco.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Developer Certificate of Origin Check - -on: [pull_request] - -jobs: - check: - runs-on: ubuntu-latest - - steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@v1.1.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@v1.1.0 - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml index 0d54b8cfef..4ff9ff6faa 100644 --- a/.github/workflows/integ-tests-with-security.yml +++ b/.github/workflows/integ-tests-with-security.yml @@ -10,11 +10,56 @@ on: - '.github/workflows/integ-tests-with-security.yml' jobs: - security-it: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + security-it-linux: + needs: Get-CI-Image-Tag + strategy: + fail-fast: false + matrix: + java: [ 11, 17 ] + + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + steps: + - uses: actions/checkout@v3 + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Build with Gradle + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew integTestWithSecurity" + + - name: Upload test reports + if: ${{ always() }} + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: test-reports-${{ matrix.os }}-${{ matrix.java }} + path: | + integ-test/build/reports/** + integ-test/build/testclusters/*/logs/* + integ-test/build/testclusters/*/config/* + + security-it-windows-macos: strategy: fail-fast: false matrix: - os: [ ubuntu-latest, windows-latest, macos-latest ] + os: [ windows-latest, macos-latest ] java: [ 11, 17 ] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/sql-pitest.yml b/.github/workflows/sql-pitest.yml index fb23ff829f..bc751daefa 100644 --- a/.github/workflows/sql-pitest.yml +++ b/.github/workflows/sql-pitest.yml @@ -11,13 +11,25 @@ run-name: ${{ inputs.name == '' && format('{0} @ {1}', github.ref_name, github.sha) || inputs.name }} jobs: - build: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + build-linux: + needs: Get-CI-Image-Tag strategy: matrix: java: - 11 - 17 runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root steps: - uses: actions/checkout@v3 @@ -30,7 +42,8 @@ jobs: - name: PiTest with Gradle run: | - ./gradlew --continue :core:pitest :opensearch:pitest + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew --continue :core:pitest :opensearch:pitest" - name: Upload test reports if: always() diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index fecfe7adc2..34c45d2108 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -17,16 +17,95 @@ on: - '.github/workflows/sql-test-and-build-workflow.yml' jobs: - build: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + build-linux: + needs: Get-CI-Image-Tag + strategy: + # Run all jobs + fail-fast: false + matrix: + java: + - 11 + - 17 + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + steps: + - uses: actions/checkout@v3 + + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Build with Gradle + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew --continue build" + + - name: Run backward compatibility tests + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./scripts/bwctest.sh" + + - name: Create Artifact Path + run: | + mkdir -p opensearch-sql-builds + cp -r ./plugin/build/distributions/*.zip opensearch-sql-builds/ + + # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action + - name: Upload SQL Coverage Report + if: ${{ always() }} + uses: codecov/codecov-action@v3 + continue-on-error: true + with: + flags: sql-engine + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: opensearch-sql-ubuntu-latest-${{ matrix.java }} + path: opensearch-sql-builds + + - name: Upload test reports + if: ${{ always() }} + uses: actions/upload-artifact@v2 + continue-on-error: true + with: + name: test-reports-ubuntu-latest-${{ matrix.java }} + path: | + sql/build/reports/** + ppl/build/reports/** + core/build/reports/** + common/build/reports/** + opensearch/build/reports/** + integ-test/build/reports/** + protocol/build/reports/** + legacy/build/reports/** + plugin/build/reports/** + doctest/build/testclusters/docTestCluster-0/logs/* + integ-test/build/testclusters/*/logs/* + + build-windows-macos: strategy: # Run all jobs fail-fast: false matrix: entry: - - { os: ubuntu-latest, java: 11 } - { os: windows-latest, java: 11, os_build_args: -x doctest -PbuildPlatform=windows } - { os: macos-latest, java: 11} - - { os: ubuntu-latest, java: 17 } - { os: windows-latest, java: 17, os_build_args: -x doctest -PbuildPlatform=windows } - { os: macos-latest, java: 17 } runs-on: ${{ matrix.entry.os }} @@ -43,10 +122,6 @@ jobs: - name: Build with Gradle run: ./gradlew --continue build ${{ matrix.entry.os_build_args }} - - name: Run backward compatibility tests - if: ${{ matrix.entry.os == 'ubuntu-latest' }} - run: ./scripts/bwctest.sh - - name: Create Artifact Path run: | mkdir -p opensearch-sql-builds diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml index 9ca27dffaf..46c1930cc8 100644 --- a/.github/workflows/sql-test-workflow.yml +++ b/.github/workflows/sql-test-workflow.yml @@ -11,13 +11,25 @@ run-name: ${{ inputs.name == '' && format('{0} @ {1}', github.ref_name, github.sha) || inputs.name }} jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + build: + needs: Get-CI-Image-Tag strategy: matrix: java: - 11 - 17 runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root steps: - uses: actions/checkout@v3 @@ -31,35 +43,36 @@ jobs: - name: Run tests id: tests run: | + chown -R 1000:1000 `pwd` # Spotless - ./gradlew :opensearch:spotlessCheck || echo "* Spotless failed for opensearch" > report.log - ./gradlew :sql:spotlessCheck || echo "* Spotless failed for sql" >> report.log - ./gradlew :ppl:spotlessCheck || echo "* Spotless failed for ppl" >> report.log - ./gradlew :core:spotlessCheck || echo "* Spotless failed for core" >> report.log - ./gradlew :common:spotlessCheck || echo "* Spotless failed for common" >> report.log - ./gradlew :legacy:spotlessCheck || echo "* Spotless failed for legacy" >> report.log - ./gradlew :protocol:spotlessCheck || echo "* Spotless failed for protocol" >> report.log - ./gradlew :opensearch-sql-plugin:spotlessCheck || echo "* Spotless failed for plugin" >> report.log - ./gradlew :integ-test:spotlessCheck || echo "* Spotless failed for integ-test" >> report.log + su `id -un 1000` -c './gradlew :opensearch:spotlessCheck || echo "* Spotless failed for opensearch" > report.log' + su `id -un 1000` -c './gradlew :sql:spotlessCheck || echo "* Spotless failed for sql" >> report.log' + su `id -un 1000` -c './gradlew :ppl:spotlessCheck || echo "* Spotless failed for ppl" >> report.log' + su `id -un 1000` -c './gradlew :core:spotlessCheck || echo "* Spotless failed for core" >> report.log' + su `id -un 1000` -c './gradlew :common:spotlessCheck || echo "* Spotless failed for common" >> report.log' + su `id -un 1000` -c './gradlew :legacy:spotlessCheck || echo "* Spotless failed for legacy" >> report.log' + su `id -un 1000` -c './gradlew :protocol:spotlessCheck || echo "* Spotless failed for protocol" >> report.log' + su `id -un 1000` -c './gradlew :opensearch-sql-plugin:spotlessCheck || echo "* Spotless failed for plugin" >> report.log' + su `id -un 1000` -c './gradlew :integ-test:spotlessCheck || echo "* Spotless failed for integ-test" >> report.log' # Unit tests - ./gradlew :opensearch:test || echo "* Unit tests failed for opensearch" >> report.log - ./gradlew :ppl:test || echo "* Unit tests failed for sql" >> report.log - ./gradlew :sql:test || echo "* Unit tests failed for ppl" >> report.log - ./gradlew :core:test || echo "* Unit tests failed for core" >> report.log - ./gradlew :protocol:test || echo "* Unit tests failed for protocol" >> report.log - ./gradlew :opensearch-sql-plugin:test || echo "* Unit tests failed for plugin" >> report.log - ./gradlew :legacy:test || echo "* Unit tests failed for legacy" >> report.log + su `id -un 1000` -c './gradlew :opensearch:test || echo "* Unit tests failed for opensearch" >> report.log' + su `id -un 1000` -c './gradlew :ppl:test || echo "* Unit tests failed for sql" >> report.log' + su `id -un 1000` -c './gradlew :sql:test || echo "* Unit tests failed for ppl" >> report.log' + su `id -un 1000` -c './gradlew :core:test || echo "* Unit tests failed for core" >> report.log' + su `id -un 1000` -c './gradlew :protocol:test || echo "* Unit tests failed for protocol" >> report.log' + su `id -un 1000` -c './gradlew :opensearch-sql-plugin:test || echo "* Unit tests failed for plugin" >> report.log' + su `id -un 1000` -c './gradlew :legacy:test || echo "* Unit tests failed for legacy" >> report.log' # jacoco - ./gradlew :opensearch:jacocoTestCoverageVerification || echo "* Jacoco failed for opensearch" >> report.log - ./gradlew :ppl:jacocoTestCoverageVerification || echo "* Jacoco failed for sql" >> report.log - ./gradlew :sql:jacocoTestCoverageVerification || echo "* Jacoco failed for ppl" >> report.log - ./gradlew :core:jacocoTestCoverageVerification || echo "* Jacoco failed for core" >> report.log - ./gradlew :protocol:jacocoTestCoverageVerification || echo "* Jacoco failed for protocol" >> report.log - ./gradlew :opensearch-sql-plugin:jacocoTestCoverageVerification || echo "* Jacoco failed for plugin" >> report.log + su `id -un 1000` -c './gradlew :opensearch:jacocoTestCoverageVerification || echo "* Jacoco failed for opensearch" >> report.log' + su `id -un 1000` -c './gradlew :ppl:jacocoTestCoverageVerification || echo "* Jacoco failed for sql" >> report.log' + su `id -un 1000` -c './gradlew :sql:jacocoTestCoverageVerification || echo "* Jacoco failed for ppl" >> report.log' + su `id -un 1000` -c './gradlew :core:jacocoTestCoverageVerification || echo "* Jacoco failed for core" >> report.log' + su `id -un 1000` -c './gradlew :protocol:jacocoTestCoverageVerification || echo "* Jacoco failed for protocol" >> report.log' + su `id -un 1000` -c './gradlew :opensearch-sql-plugin:jacocoTestCoverageVerification || echo "* Jacoco failed for plugin" >> report.log' # Misc tests - ./gradlew :integ-test:integTest || echo "* Integration test failed" >> report.log - ./gradlew :doctest:doctest || echo "* Doctest failed" >> report.log - ./scripts/bwctest.sh || echo "* Backward compatibility test failed" >> report.log + su `id -un 1000` -c './gradlew :integ-test:integTest || echo "* Integration test failed" >> report.log' + su `id -un 1000` -c './gradlew :doctest:doctest || echo "* Doctest failed" >> report.log' + su `id -un 1000` -c './scripts/bwctest.sh || echo "* Backward compatibility test failed" >> report.log' - name: Verify test results run: |