Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Onboarding jenkins prod docker images to github actions #1092

Merged
merged 4 commits into from
Oct 30, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,40 @@ on:
- "*"

jobs:
Get-CI-Image-Tag:
uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
with:
product: opensearch

Build-ad:
needs: Get-CI-Image-Tag
strategy:
matrix:
java: [17]
fail-fast: false

name: Run Anomaly detection model performance benchmark
runs-on: ubuntu-latest
container:
# using the same image which is used by opensearch-build team to build the OpenSearch Distribution
# this image tag is subject to change as more dependencies and updates will arrive over time
image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
# need to switch to root so that github actions can install runner binary on container without permission issues.
options: --user root

steps:
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}

# anomaly-detection
- name: Checkout AD
uses: actions/checkout@v4
uses: actions/checkout@v3

- name: Build and Run Tests
run: |
./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true
./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true
chown -R 1000:1000 `pwd`
su `id -un 1000` -c "./gradlew ':test' --tests 'org.opensearch.ad.ml.HCADModelPerfTests' -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true"
su `id -un 1000` -c "./gradlew integTest --tests 'org.opensearch.ad.e2e.SingleStreamModelPerfIT' -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true"
54 changes: 51 additions & 3 deletions .github/workflows/test_build_multi_platform.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ on:
- "*"

jobs:
Get-CI-Image-Tag:
uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
with:
product: opensearch

Build-ad-windows:
strategy:
matrix:
Expand All @@ -20,6 +25,7 @@ jobs:
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}
- name: Checkout Anomaly Detection
uses: actions/checkout@v4
Expand All @@ -38,22 +44,64 @@ jobs:
with:
file: ./build/reports/jacoco/test/jacocoTestReport.xml
flags: plugin
Build-ad:

Build-ad-linux:
needs: Get-CI-Image-Tag
strategy:
matrix:
java: [11, 17, 20]
fail-fast: false
name: Build and Test Anomaly detection Plugin
runs-on: ubuntu-latest
container:
# using the same image which is used by opensearch-build team to build the OpenSearch Distribution
# this image tag is subject to change as more dependencies and updates will arrive over time
image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
# need to switch to root so that github actions can install runner binary on container without permission issues.
options: --user root

env:
JENKINS_URL: build.ci.opensearch.org

steps:
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}

- name: Checkout AD
uses: actions/checkout@v3

- name: Assemble / build / mavenlocal / integTest
run: |
chown -R 1000:1000 `pwd`
su `id -un 1000` -c "./gradlew assemble &&
./gradlew build &&
./gradlew publishToMavenLocal &&
./gradlew integTest -PnumNodes=3"
- name: Upload Coverage Report
uses: codecov/codecov-action@v3
with:
file: ./build/reports/jacoco/test/jacocoTestReport.xml
flags: plugin

Build-ad-macos:
strategy:
matrix:
java: [11,17,20]
os: [ubuntu-latest, macos-latest]
fail-fast: false

name: Build and Test Anomaly detection Plugin
runs-on: ${{ matrix.os }}
runs-on: macos-latest
env:
JENKINS_URL: build.ci.opensearch.org

steps:
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}

- name: Checkout AD
Expand Down
30 changes: 23 additions & 7 deletions .github/workflows/test_bwc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,38 +8,54 @@ on:
- "*"

jobs:
Build-ad:
Get-CI-Image-Tag:
uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
with:
product: opensearch

Build-ad-linux:
needs: Get-CI-Image-Tag
strategy:
matrix:
java: [11,17,20]
fail-fast: false

name: Test Anomaly detection BWC
runs-on: ubuntu-latest
container:
# using the same image which is used by opensearch-build team to build the OpenSearch Distribution
# this image tag is subject to change as more dependencies and updates will arrive over time
image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
# need to switch to root so that github actions can install runner binary on container without permission issues.
options: --user root

steps:
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}

# anomaly-detection
- name: Checkout AD
uses: actions/checkout@v4
uses: actions/checkout@v3

- name: Assemble anomaly-detection
run: |
plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'`
chown -R 1000:1000 `pwd`
su `id -un 1000` -c "./gradlew properties -q | grep 'opensearch_build:' | awk '{print $2}'" > /tmp/plugin_version.txt
plugin_version=`cat /tmp/plugin_version.txt`
echo plugin_version $plugin_version
./gradlew assemble
su `id -un 1000` -c "./gradlew assemble"
echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
su `id -un 1000` -c "mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version"
echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
ls ./build/distributions/
cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
su `id -un 1000` -c "cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version"
echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
- name: Run AD Backwards Compatibility Tests
run: |
echo "Running backwards compatibility tests ..."
./gradlew bwcTestSuite -Dtests.security.manager=false
chown -R 1000:1000 `pwd`
su `id -un 1000` -c "./gradlew bwcTestSuite -Dtests.security.manager=false"
1 change: 1 addition & 0 deletions .github/workflows/test_security.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ jobs:
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}

# time-series-analytics
Expand Down
Loading