From f6d4780c3a4d552ffd4587f5f8a8e634955d46d3 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 30 Jul 2020 13:15:32 -0400 Subject: [PATCH] Workflow Consistency / Simplification Updates in Windows and RPM (#331) --- .github/scripts/setup_runners_service.sh | 127 ++++---- .../scripts/setup_runners_service_windows.ps1 | 140 +++++++++ ...st-build-rpm.yml => staging-build-rpm.yml} | 102 +++---- ...-windows.yml => staging-build-windows.yml} | 280 ++++++++---------- .github/workflows/test-build-deb.yml | 4 +- .github/workflows/test-build-docker.yml | 2 +- .github/workflows/test-build-tar.yml | 2 +- 7 files changed, 375 insertions(+), 282 deletions(-) create mode 100755 .github/scripts/setup_runners_service_windows.ps1 rename .github/workflows/{test-build-rpm.yml => staging-build-rpm.yml} (93%) rename .github/workflows/{test-build-windows.yml => staging-build-windows.yml} (54%) diff --git a/.github/scripts/setup_runners_service.sh b/.github/scripts/setup_runners_service.sh index e451ca375..bac77611d 100755 --- a/.github/scripts/setup_runners_service.sh +++ b/.github/scripts/setup_runners_service.sh @@ -3,7 +3,7 @@ # This script allows users to manually assign parameters if [ "$#" -ne 2 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ] then - echo "Please assign a2 parameters when running this script" + echo "Please assign 2 parameters when running this script" echo "Example: $0 \$SETUP_DISTRO \$SETUP_ACTION" echo "Example: $0 \"tar | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec\"" exit 1 @@ -25,6 +25,7 @@ ES_VERSION=`python $REPO_ROOT/bin/version-info --es` echo "############################################################" echo "Setup ES/KIBANA to start with YES/NO security configurations" +echo "User enters $SETUP_ACTION" echo "############################################################" echo "setup parameters" @@ -36,7 +37,7 @@ if [ "$SETUP_DISTRO" = "tar" ] then mkdir -p odfe-testing cd odfe-testing - aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/tarball/opendistro-elasticsearch/opendistroforelasticsearch-$OD_VERSION.tar.gz . + aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/tarball/opendistro-elasticsearch/opendistroforelasticsearch-$OD_VERSION.tar.gz . --quiet; echo $? tar -zxf opendistroforelasticsearch-$OD_VERSION.tar.gz cd opendistroforelasticsearch-$OD_VERSION fi @@ -59,129 +60,129 @@ then sudo yum install opendistroforelasticsearch-$OD_VERSION -y fi - - ##################################################################################################### echo "setup es" + +# everything needs es if [ "$SETUP_DISTRO" = "tar" ] then sed -i /install_demo_configuration/d opendistro-tar-install.sh sed -i '/http\.port/s/^# *//' config/elasticsearch.yml + nohup ./opendistro-tar-install.sh & else sudo mkdir -p /home/repo sudo chmod 777 /home/repo sudo chmod 777 /etc/elasticsearch/elasticsearch.yml sudo sed -i '/path.logs/a path.repo: ["/home/repo"]' /etc/elasticsearch/elasticsearch.yml sudo sed -i /^node.max_local_storage_nodes/d /etc/elasticsearch/elasticsearch.yml + sudo systemctl restart elasticsearch.service fi if [ "$SETUP_ACTION" = "--es" ] then - if [ "$SETUP_DISTRO" = "tar" ] - then - nohup ./opendistro-tar-install.sh & - sleep 45 - exit 0 - else - sudo systemctl restart elasticsearch.service - sleep 30 - exit 0 - fi + sleep 60 + curl -XGET https://localhost:9200 -u admin:admin --insecure + echo "es start" + cd $REPO_ROOT + exit 0 fi ##################################################################################################### -echo "remove es security" -if [ "$SETUP_DISTRO" = "tar" ] +# *nosec remove es-security +if [ "$SETUP_ACTION" = "--es-nosec" ] || [ "$SETUP_ACTION" = "--kibana-nosec" ] then - ./bin/elasticsearch-plugin remove opendistro_security -else - sudo /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security - sudo sed -i /^opendistro_security/d /etc/elasticsearch/elasticsearch.yml - sudo sed -i /CN=kirk/d /etc/elasticsearch/elasticsearch.yml - sudo sed -i /^cluster.routing.allocation.disk.threshold_enabled/d /etc/elasticsearch/elasticsearch.yml -fi + echo "remove es security" -if [ "$SETUP_ACTION" = "--es-nosec" ] -then if [ "$SETUP_DISTRO" = "tar" ] then + ./bin/elasticsearch-plugin remove opendistro_security nohup ./opendistro-tar-install.sh & - sleep 45 - exit 0 else + sudo /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security + sudo sed -i /^opendistro_security/d /etc/elasticsearch/elasticsearch.yml + sudo sed -i /CN=kirk/d /etc/elasticsearch/elasticsearch.yml + sudo sed -i /^cluster.routing.allocation.disk.threshold_enabled/d /etc/elasticsearch/elasticsearch.yml sudo systemctl restart elasticsearch.service - sleep 30 - exit 0 fi fi -##################################################################################################### - -echo "setup kibana" -if [ "$SETUP_DISTRO" = "tar" ] +if [ "$SETUP_ACTION" = "--es-nosec" ] then - cd ../../ - mkdir -p kibana-testing - cd kibana-testing - aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/tarball/opendistroforelasticsearch-kibana/opendistroforelasticsearch-kibana-$odfe_version.tar.gz . - tar -zxf opendistroforelasticsearch-kibana-$odfe_version.tar.gz - cd opendistroforelasticsearch-kibana - sed -i /install_demo_configuration/d opendistro-tar-install.sh -else - sudo apt install opendistroforelasticsearch-kibana-$OD_VERSION -y || sudo yum install opendistroforelasticsearch-kibana-$OD_VERSION -y + sleep 60 + curl -XGET http://localhost:9200 + echo "es-nosec start" + cd $REPO_ROOT + exit 0 fi -if [ "$SETUP_ACTION" = "--kibana" ] +##################################################################################################### + +# kibana* needs kibana +if [ "$SETUP_ACTION" = "--kibana" ] || [ "$SETUP_ACTION" = "--kibana-nosec" ] then + echo "setup kibana" + if [ "$SETUP_DISTRO" = "tar" ] then + cd ../../ + mkdir -p kibana-testing + cd kibana-testing + aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/tarball/opendistroforelasticsearch-kibana/opendistroforelasticsearch-kibana-$odfe_version.tar.gz . --quiet; echo $? + tar -zxf opendistroforelasticsearch-kibana-$odfe_version.tar.gz + cd opendistroforelasticsearch-kibana + sed -i /install_demo_configuration/d opendistro-tar-install.sh cd ../../odfe-testing nohup ./opendistro-tar-install.sh & - sleep 45 cd ../../kibana-testing nohup ./bin/kibana & - sleep 120 - exit 0 else + sudo apt install opendistroforelasticsearch-kibana-$OD_VERSION -y || sudo yum install opendistroforelasticsearch-kibana-$OD_VERSION -y sudo systemctl restart elasticsearch.service sudo systemctl restart kibana.service - sleep 120 - exit 0 fi fi -##################################################################################################### - -echo "remove kibana security" -if [ "$SETUP_DISTRO" = "tar" ] +if [ "$SETUP_ACTION" = "--kibana" ] then - ./bin/kibana-plugin remove opendistro_security - sed -i /^opendistro_security/d ./config/kibana.yml - sed -i 's/https/http/' ./config/kibana.yml -else - sudo /usr/share/kibana/bin/kibana-plugin remove opendistro_security --allow-root - sudo sed -i /^opendistro_security/d /etc/kibana/kibana.yml - sudo sed -i 's/https/http/' /etc/kibana/kibana.yml + sleep 120 + curl -XGET https://localhost:9200 -u admin:admin --insecure + curl -v -XGET https://localhost:5601 --insecure + echo "es & kibana start" + cd $REPO_ROOT + exit 0 fi +##################################################################################################### + +# kibana-nosec remove kibana-security if [ "$SETUP_ACTION" = "--kibana-nosec" ] then + echo "remove kibana security" + if [ "$SETUP_DISTRO" = "tar" ] then + ./bin/kibana-plugin remove opendistro_security + sed -i /^opendistro_security/d ./config/kibana.yml + sed -i 's/https/http/' ./config/kibana.yml cd ../../odfe-testing nohup ./opendistro-tar-install.sh & - sleep 45 cd ../../kibana-testing nohup ./bin/kibana & - sleep 120 - exit 0 else + sudo /usr/share/kibana/bin/kibana-plugin remove opendistro_security --allow-root + sudo sed -i /^opendistro_security/d /etc/kibana/kibana.yml + sudo sed -i 's/https/http/' /etc/kibana/kibana.yml sudo systemctl restart elasticsearch.service sudo systemctl restart kibana.service + fi + sleep 120 + curl -XGET http://localhost:9200 + curl -v -XGET http://localhost:5601 + echo "es & kibana-nosec start" + cd $REPO_ROOT exit 0 - fi fi diff --git a/.github/scripts/setup_runners_service_windows.ps1 b/.github/scripts/setup_runners_service_windows.ps1 new file mode 100755 index 000000000..814cda7e4 --- /dev/null +++ b/.github/scripts/setup_runners_service_windows.ps1 @@ -0,0 +1,140 @@ +#!/usr/bin/env pwsh + +# Keep the pwsh script running even with errors +#$ErrorActionPreference = 'SilentlyContinue' + +# setup user parameters +$SETUP_ACTION=$args[0] +if (!$SETUP_ACTION) { + echo "Please enter 1 parameter: --es | --es-nosec | --kibana | --kibana-nosec" + exit 1 +} + +echo "############################################################" +echo "Setup ES/KIBANA to start with YES/NO security configurations" +echo "User enters $SETUP_ACTION" +echo "############################################################" + +echo "setup es" +dir +python -m pip install --upgrade pip +echo pip3 -version +pip3 install awscli +$PACKAGE="opendistroforelasticsearch" +$OD_VERSION=$(python ./bin/version-info --od) +$S3_PACKAGE="odfe-"+$OD_VERSION+".zip" +dir + +############################################################### + +# everyone needs es +echo "downloading zip from S3" +aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/odfe-windows/staging/odfe-window-zip/$S3_PACKAGE . --quiet; echo $?\ +echo "unzipping $S3_PACKAGE" +unzip -qq .\$S3_PACKAGE + +if ($SETUP_ACTION -eq "--es"){ + echo "removing useless config" #deprecated since 7.8.0 and will crash --es-nosec now + findstr /V "node.max_local_storage_nodes" .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml > .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml.new + del .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml + move .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml.new .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml + type .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml + + echo "running es" + nohup .\$PACKAGE-$OD_VERSION\bin\elasticsearch.bat & + + echo "Waiting for 60 seconds" + ping -n 60 127.0.0.1 >.\out.txt + curl -XGET https://localhost:9200 -u admin:admin --insecure + + echo "es start" + exit 0 +} + +############################################################### + +# *nosec remove es-security +if ($SETUP_ACTION -eq "--es-nosec" -Or $SETUP_ACTION -eq "--kibana-nosec"){ + echo "removing es security" + cd $PACKAGE-$OD_VERSION\bin + .\elasticsearch-plugin.bat remove opendistro_security + cd ..\.. + + echo "Overriding with elasticsearch.yml having no certificates" + del .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml + aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/utils/elasticsearch.yml .\$PACKAGE-$OD_VERSION\config --quiet; echo $? +} + +if ($SETUP_ACTION -eq "--es-nosec"){ + echo "running es" + nohup .\$PACKAGE-$OD_VERSION\bin\elasticsearch.bat & + + echo "Waiting for 60 seconds" + ping -n 60 127.0.0.1 >.\out.txt + curl -XGET http://localhost:9200 + + echo "es-nosec start" + exit 0 +} + +############################################################### + +# kibana* needs kibana +if ($SETUP_ACTION -eq "--kibana" -Or $SETUP_ACTION -eq "--kibana-nosec"){ + echo "setup kibana" + mkdir kibana-it-test + cd kibana-it-test + $S3_KIBANA_PACKAGE="odfe-"+$OD_VERSION+"-kibana.zip" + aws s3 cp --quiet s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/odfe-windows/ode-windows-zip/$S3_KIBANA_PACKAGE . --quiet; echo $?\ + unzip -qq .\$S3_KIBANA_PACKAGE +} + +if ($SETUP_ACTION -eq "--kibana"){ + cd .. + + echo "running es" + nohup .\$PACKAGE-$OD_VERSION\bin\elasticsearch.bat & + + echo "running kibana" + cd kibana-it-test\opendistroforelasticsearch-kibana + nohup .\bin\kibana.bat & + cd ..\.. + + echo "Waiting for 120 seconds" + ping -n 120 127.0.0.1 >.\out.txt + curl -XGET https://localhost:9200 -u admin:admin --insecure + curl -v -XGET https://localhost:5601 --insecure + + echo "kibana start" + exit 0 +} + +############################################################### + +# kibana-nosec remove kibana-security +if ($SETUP_ACTION -eq "--kibana-nosec"){ + echo "removing kibana security" + cd opendistroforelasticsearch-kibana + .\bin\kibana-plugin.bat remove opendistro_security + del .\config\kibana.yml + aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/utils/kibana-config-without-security/kibana.yml .\config --quiet; echo $? + + cd ..\.. + + echo "running es" + nohup .\$PACKAGE-$OD_VERSION\bin\elasticsearch.bat & + + echo "running kibana" + cd kibana-it-test\opendistroforelasticsearch-kibana + nohup .\bin\kibana.bat & + cd ..\.. + + echo "Waiting for 120 seconds" + ping -n 120 127.0.0.1 >.\out.txt + curl -XGET http://localhost:9200 + curl -v -XGET http://localhost:5601 + + echo "kibana-nosec start" + exit 0 +} + diff --git a/.github/workflows/test-build-rpm.yml b/.github/workflows/staging-build-rpm.yml similarity index 93% rename from .github/workflows/test-build-rpm.yml rename to .github/workflows/staging-build-rpm.yml index a466466a5..e669bb20e 100644 --- a/.github/workflows/test-build-rpm.yml +++ b/.github/workflows/staging-build-rpm.yml @@ -4,7 +4,7 @@ on: #schedule: # - cron: '0 10 * * *' repository_dispatch: - types: [test-build-rpm] + types: [staging-build-rpm] jobs: plugin-availability: @@ -111,7 +111,7 @@ jobs: with: repository: opendistro-for-elasticsearch/opendistro-infra ref: jenkins-test - token: ${{ secrets.ODFE_RELEASE_BOT_PRIVATE_READ_WRITE_TOKEN }} + token: ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} - name: Build Centos Docker Image env: DOCKER_USER: ${{ secrets.DOCKER_USER }} @@ -178,48 +178,6 @@ jobs: RUNNERS+="odfe-rpm-ad-kibana-nosec" .github/scripts/setup_runners.sh run $RUNNERS ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} - Test-SQL: - needs: [Provision-Runners] - name: Test-SQL - runs-on: [self-hosted, Linux, X64, odfe-rpm-sql] - steps: - - uses: actions/checkout@v1 - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: 14 - - name: Starting ES service and retrieve tags - run: echo ::set-env name=p_tag_sql::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql); .github/scripts/setup_runners_service.sh rpm --es - - uses: actions/checkout@v1 - with: - repository: opendistro-for-elasticsearch/sql - ref: ${{env.p_tag_sql}} - - name: IT for SQL - run: | - curl -XGET https://127.0.0.1:9200 -u admin:admin --insecure; export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../sql - ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest -Dhttps=true -Duser=admin -Dpassword=admin - - Test-AD: - needs: [Provision-Runners] - name: Test-AD - runs-on: [self-hosted, Linux, X64, odfe-rpm-ad] - steps: - - uses: actions/checkout@v1 - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: 14 - - name: Starting ES service and retrieve tags - run: echo ::set-env name=p_tag_ad::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection); .github/scripts/setup_runners_service.sh rpm --es - - uses: actions/checkout@v1 - with: - repository: opendistro-for-elasticsearch/anomaly-detection - ref: ${{env.p_tag_ad}} - - name: IT for AD - run: | - curl -XGET https://127.0.0.1:9200 -u admin:admin --insecure; export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../anomaly-detection - ./gradlew :integTestRunner --tests "com.amazon.opendistroforelasticsearch.ad.rest.*IT" --tests "com.amazon.opendistroforelasticsearch.ad.e2e.*IT" -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="es-integrationtest" -Dhttps=true -Duser=admin -Dpassword=admin - Test-ISM-NoSec: needs: [Provision-Runners] name: Test-ISM-NoSec @@ -238,7 +196,7 @@ jobs: ref: ${{env.p_tag_ism_nosec}} - name: IT for ISM NoSec run: | - curl -XGET http://127.0.0.1:9200; export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../index-management + export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../index-management; pwd ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest Test-ALERTING-NoSec: @@ -259,7 +217,7 @@ jobs: ref: ${{env.p_tag_alerting_nosec}} - name: IT for ALERTING NoSec run: | - curl -XGET http://127.0.0.1:9200; export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../alerting + export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../alerting; pwd ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest Test-SQL-NoSec: @@ -280,7 +238,7 @@ jobs: ref: ${{env.p_tag_sql_nosec}} - name: IT for SQL NoSec run: | - curl -XGET http://127.0.0.1:9200; export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../sql + export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../sql; pwd ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest Test-KNN-NoSec: @@ -301,7 +259,7 @@ jobs: ref: ${{env.p_tag_knn_nosec}} - name: IT for KNN NoSec run: | - curl -XGET http://127.0.0.1:9200; export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../k-NN + export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../k-NN; pwd ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest Test-AD-NoSec: @@ -322,9 +280,51 @@ jobs: ref: ${{env.p_tag_ad_nosec}} - name: IT for AD NoSec run: | - curl -XGET http://127.0.0.1:9200; export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../anomaly-detection + export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../anomaly-detection; pwd ./gradlew :integTestRunner --tests "com.amazon.opendistroforelasticsearch.ad.rest.*IT" --tests "com.amazon.opendistroforelasticsearch.ad.e2e.*IT" -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest + Test-SQL: + needs: [Provision-Runners] + name: Test-SQL + runs-on: [self-hosted, Linux, X64, odfe-rpm-sql] + steps: + - uses: actions/checkout@v1 + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: 14 + - name: Starting ES service and retrieve tags + run: echo ::set-env name=p_tag_sql::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql); .github/scripts/setup_runners_service.sh rpm --es + - uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/sql + ref: ${{env.p_tag_sql}} + - name: IT for SQL + run: | + export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../sql; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest -Dhttps=true -Duser=admin -Dpassword=admin + + Test-AD: + needs: [Provision-Runners] + name: Test-AD + runs-on: [self-hosted, Linux, X64, odfe-rpm-ad] + steps: + - uses: actions/checkout@v1 + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: 14 + - name: Starting ES service and retrieve tags + run: echo ::set-env name=p_tag_ad::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection); .github/scripts/setup_runners_service.sh rpm --es + - uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/anomaly-detection + ref: ${{env.p_tag_ad}} + - name: IT for AD + run: | + export PATH=$PATH:$JAVA_HOME; cd $GITHUB_WORKSPACE/../anomaly-detection; pwd + ./gradlew :integTestRunner --tests "com.amazon.opendistroforelasticsearch.ad.rest.*IT" --tests "com.amazon.opendistroforelasticsearch.ad.e2e.*IT" -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="es-integrationtest" -Dhttps=true -Duser=admin -Dpassword=admin + Test-AD-KIBANA-NoSec: needs: [Provision-Runners] runs-on: [self-hosted, Linux, X64, odfe-rpm-ad-kibana-nosec] @@ -339,14 +339,14 @@ jobs: run: | echo ::set-env name=es_version_kibana::$(python ./bin/version-info --es) echo ::set-env name=p_tag_ad_kibana_nosec::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection-kibana-plugin) - .github/scripts/setup_runners_service.sh rpm --kibana-nosec; curl -XGET http://127.0.0.1:9200; curl -v http://127.0.0.1:5601 + .github/scripts/setup_runners_service.sh rpm --kibana-nosec - name: Checkout Kibana uses: actions/checkout@v2 with: repository: opendistro-for-elasticsearch/kibana-oss ref: ${{env.es_version_kibana}} - token: ${{ secrets.ODFE_RELEASE_BOT_PRIVATE_READ_WRITE_TOKEN }} + token: ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} path: kibana - name: Get node and yarn versions diff --git a/.github/workflows/test-build-windows.yml b/.github/workflows/staging-build-windows.yml similarity index 54% rename from .github/workflows/test-build-windows.yml rename to .github/workflows/staging-build-windows.yml index 537d82aa8..c479b619e 100644 --- a/.github/workflows/test-build-windows.yml +++ b/.github/workflows/staging-build-windows.yml @@ -4,7 +4,7 @@ on: #schedule: # - cron: '30 10 * * *' repository_dispatch: - types: [test-build-windows] + types: [staging-build-windows] jobs: build-es-artifacts: @@ -48,9 +48,10 @@ jobs: - name: Build Kibana run: ./kibana/windows/opendistro-windows-kibana-build.sh - Test-ISM-Plugin: + Test-ISM-NoSec: needs: [build-es-artifacts, build-kibana-artifacts] runs-on: windows-latest + name: Test-ISM-NoSec steps: - uses: actions/checkout@v1 - name: Retrieve plugin tags @@ -74,39 +75,17 @@ jobs: - name: RUN ES and ISM IntegTest run: | - dir - python -m pip install --upgrade pip - echo pip3 -version - pip3 install awscli - dir - $PACKAGE="opendistroforelasticsearch" - $OD_VERSION=$(python ./bin/version-info --od) - $S3_PACKAGE="odfe-"+$OD_VERSION+".zip" - dir - echo downloading zip from S3 - aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/odfe-windows/staging/odfe-window-zip/$S3_PACKAGE .\ - echo unzipping $S3_PACKAGE - unzip .\$S3_PACKAGE - dir - echo "removing security plugin" - cd $PACKAGE-$OD_VERSION\bin - .\elasticsearch-plugin.bat remove opendistro_security - cd ..\.. - echo "Overriding with elasticsearch.yml having no certificates" - del .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml - aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/utils/elasticsearch.yml .\$PACKAGE-$OD_VERSION\config - echo running es - nohup .\$PACKAGE-$OD_VERSION\bin\elasticsearch.bat & - echo Waiting for 2min - ping -n 120 127.0.0.1 >.\out.txt + .github\scripts\setup_runners_service_windows.ps1 --es-nosec + $ErrorActionPreference = 'SilentlyContinue' echo running tests cd ..\index-management dir ./gradlew.bat integTest -D tests.rest.cluster=localhost:9200 -D tests.cluster=localhost:9200 -D tests.clustername=es-integrationtest --stacktrace - Test-Alerting-plugin: + Test-ALERTING-NoSec: needs: [build-es-artifacts, build-kibana-artifacts] runs-on: windows-latest + name: Test-ALERTING-NoSec steps: - uses: actions/checkout@v1 - name: Retrieve plugin tags @@ -129,46 +108,26 @@ jobs: - name: RUN ES and Alerting IntegTest run: | - dir - python -m pip install --upgrade pip - echo pip3 -version - pip3 install awscli - $PACKAGE="opendistroforelasticsearch" - $OD_VERSION=$(python ./bin/version-info --od) - $S3_PACKAGE="odfe-"+$OD_VERSION+".zip" - dir - echo downloading zip from S3 - aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/odfe-windows/staging/odfe-window-zip/$S3_PACKAGE .\ - echo unzipping $S3_PACKAGE - unzip .\$S3_PACKAGE - echo "removing security plugin" - cd $PACKAGE-$OD_VERSION\bin - .\elasticsearch-plugin.bat remove opendistro_security - cd ..\.. - echo "Overriding with elasticsearch.yml having no certificates" - del .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml - aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/utils/elasticsearch.yml .\$PACKAGE-$OD_VERSION\config - echo running es - nohup .\$PACKAGE-$OD_VERSION\bin\elasticsearch.bat & - echo Waiting for 2min - ping -n 120 127.0.0.1 >.\out.txt + .github\scripts\setup_runners_service_windows.ps1 --es-nosec + $ErrorActionPreference = 'SilentlyContinue' echo running tests - cd ..\alerting\alerting + cd ..\alerting dir - ../gradlew.bat integTest -D tests.rest.cluster=localhost:9200 -D tests.cluster=localhost:9200 -D tests.clustername=es-integrationtest --stacktrace - - Test-AD-plugin: + ./gradlew.bat integTest -D tests.rest.cluster=localhost:9200 -D tests.cluster=localhost:9200 -D tests.clustername=es-integrationtest --stacktrace + + Test-SQL-NoSec: needs: [build-es-artifacts, build-kibana-artifacts] runs-on: windows-latest + name: Test-SQL-NoSec steps: - uses: actions/checkout@v1 - name: Retrieve plugin tags - run: echo ::set-env name=p_tag_ad::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection) + run: echo ::set-env name=p_tag_sql::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql) shell: bash - uses: actions/checkout@v1 with: - repository: opendistro-for-elasticsearch/anomaly-detection - ref: ${{env.p_tag_ad}} + repository: opendistro-for-elasticsearch/sql + ref: ${{env.p_tag_sql}} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: @@ -180,39 +139,19 @@ jobs: with: java-version: 14 - - name: RUN ES and AD IntegTest + - name: RUN ES and SQL IntegTest run: | + .github\scripts\setup_runners_service_windows.ps1 --es-nosec + $ErrorActionPreference = 'SilentlyContinue' + echo "running tests" + cd ..\sql dir - python -m pip install --upgrade pip - echo pip3 -version - pip3 install awscli - $PACKAGE="opendistroforelasticsearch" - $OD_VERSION=$(python ./bin/version-info --od) - $S3_PACKAGE="odfe-"+$OD_VERSION+".zip" - dir - echo downloading zip from S3 - aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/odfe-windows/staging/odfe-window-zip/$S3_PACKAGE .\ - echo unzipping $S3_PACKAGE - unzip .\$S3_PACKAGE - echo "removing security plugin" - cd $PACKAGE-$OD_VERSION\bin - .\elasticsearch-plugin.bat remove opendistro_security - cd ..\.. - echo "Overriding with elasticsearch.yml having no certificates" - del .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml - aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/utils/elasticsearch.yml .\$PACKAGE-$OD_VERSION\config - echo running es - nohup .\$PACKAGE-$OD_VERSION\bin\elasticsearch.bat & - echo Waiting for 2min - ping -n 120 127.0.0.1 >.\out.txt - echo running tests - cd ..\anomaly-detection - dir - .\gradlew.bat :integTestRunner --tests "com.amazon.opendistroforelasticsearch.ad.rest.*IT" --tests "com.amazon.opendistroforelasticsearch.ad.e2e.*IT" -D tests.rest.cluster=localhost:9200 -D tests.cluster=localhost:9200 -D tests.clustername=es-integrationtest --stacktrace - - Test-AD-plugin-With-Security: + .\gradlew.bat integTest -D tests.rest.cluster=localhost:9200 -D tests.cluster=localhost:9200 -D tests.clustername=es-integrationtest --stacktrace + + Test-AD-NoSec: needs: [build-es-artifacts, build-kibana-artifacts] runs-on: windows-latest + name: Test-AD-NoSec steps: - uses: actions/checkout@v1 - name: Retrieve plugin tags @@ -235,31 +174,17 @@ jobs: - name: RUN ES and AD IntegTest run: | - dir - python -m pip install --upgrade pip - echo pip3 -version - pip3 install awscli - $PACKAGE="opendistroforelasticsearch" - $OD_VERSION=$(python ./bin/version-info --od) - $S3_PACKAGE="odfe-"+$OD_VERSION+".zip" - dir - echo downloading zip from S3 - aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/odfe-windows/staging/odfe-window-zip/$S3_PACKAGE .\ - echo "unzipping $S3_PACKAGE" - unzip .\$S3_PACKAGE - echo "running es" - nohup .\$PACKAGE-$OD_VERSION\bin\elasticsearch.bat & - echo "Waiting for 2min" - ping -n 120 127.0.0.1 >.\out.txt - curl -XGET https://127.0.0.1:9200 -u admin:admin --insecure - echo "running tests" + .github\scripts\setup_runners_service_windows.ps1 --es-nosec + $ErrorActionPreference = 'SilentlyContinue' + echo running tests cd ..\anomaly-detection dir - .\gradlew.bat :integTestRunner --tests "com.amazon.opendistroforelasticsearch.ad.rest.*IT" --tests "com.amazon.opendistroforelasticsearch.ad.e2e.*IT" -D tests.rest.cluster=localhost:9200 -D tests.cluster=localhost:9200 -D tests.clustername=es-integrationtest -D https=true -D user=admin -D password=admin --stacktrace + .\gradlew.bat :integTestRunner --tests "com.amazon.opendistroforelasticsearch.ad.rest.*IT" --tests "com.amazon.opendistroforelasticsearch.ad.e2e.*IT" -D tests.rest.cluster=localhost:9200 -D tests.cluster=localhost:9200 -D tests.clustername=es-integrationtest --stacktrace - Test-SQL-plugin: + Test-SQL: needs: [build-es-artifacts, build-kibana-artifacts] runs-on: windows-latest + name: Test-SQL steps: - uses: actions/checkout@v1 - name: Retrieve plugin tags @@ -282,48 +207,26 @@ jobs: - name: RUN ES and SQL IntegTest run: | - dir - python -m pip install --upgrade pip - echo pip3 -version - pip3 install awscli - $PACKAGE="opendistroforelasticsearch" - $OD_VERSION=$(python ./bin/version-info --od) - $S3_PACKAGE="odfe-"+$OD_VERSION+".zip" - dir - echo "downloading zip from S3" - aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/odfe-windows/staging/odfe-window-zip/$S3_PACKAGE .\ - echo "unzipping $S3_PACKAGE" - unzip .\$S3_PACKAGE - echo "removing security plugin" - cd $PACKAGE-$OD_VERSION\bin - .\elasticsearch-plugin.bat remove opendistro_security - cd ..\.. - echo "Overriding with elasticsearch.yml having no certificates" - del .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml - aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/utils/elasticsearch.yml .\$PACKAGE-$OD_VERSION\config - echo "running es" - nohup .\$PACKAGE-$OD_VERSION\bin\elasticsearch.bat & - echo "Waiting for 2min" - ping -n 120 127.0.0.1 >.\out.txt - curl -XGET http://127.0.0.1:9200 + .github\scripts\setup_runners_service_windows.ps1 --es + $ErrorActionPreference = 'SilentlyContinue' echo "running tests" cd ..\sql dir - .\gradlew.bat integTest -D tests.rest.cluster=localhost:9200 -D tests.cluster=localhost:9200 -D tests.clustername=es-integrationtest --stacktrace - - - Test-SQL-plugin-With-Security: + .\gradlew.bat integTest -D tests.rest.cluster=localhost:9200 -D tests.cluster=localhost:9200 -D tests.clustername=es-integrationtest -D https=true -D user=admin -D password=admin --stacktrace + + Test-AD: needs: [build-es-artifacts, build-kibana-artifacts] runs-on: windows-latest + name: Test-AD steps: - uses: actions/checkout@v1 - name: Retrieve plugin tags - run: echo ::set-env name=p_tag_sql::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql) + run: echo ::set-env name=p_tag_ad::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection) shell: bash - uses: actions/checkout@v1 with: - repository: opendistro-for-elasticsearch/sql - ref: ${{env.p_tag_sql}} + repository: opendistro-for-elasticsearch/anomaly-detection + ref: ${{env.p_tag_ad}} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: @@ -334,32 +237,81 @@ jobs: uses: actions/setup-java@v1 with: java-version: 14 - - - name: RUN ES and SQL IntegTest + - name: RUN ES and AD IntegTest run: | - dir - python -m pip install --upgrade pip - echo pip3 -version - pip3 install awscli - $PACKAGE="opendistroforelasticsearch" - $OD_VERSION=$(python ./bin/version-info --od) - $S3_PACKAGE="odfe-"+$OD_VERSION+".zip" - dir - echo "downloading zip from S3" - aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/odfe-windows/staging/odfe-window-zip/$S3_PACKAGE .\ - echo "unzipping $S3_PACKAGE" - unzip .\$S3_PACKAGE - echo "removing useless config" - findstr /V "node.max_local_storage_nodes" .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml > .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml.new - del .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml - move .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml.new .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml - type .\$PACKAGE-$OD_VERSION\config\elasticsearch.yml - echo "running es" - nohup .\$PACKAGE-$OD_VERSION\bin\elasticsearch.bat & - echo "Waiting for 2min" - ping -n 120 127.0.0.1 >.\out.txt - curl -XGET https://127.0.0.1:9200 -u admin:admin --insecure + .github\scripts\setup_runners_service_windows.ps1 --es + $ErrorActionPreference = 'SilentlyContinue' echo "running tests" - cd ..\sql + cd ..\anomaly-detection dir - .\gradlew.bat integTest -D tests.rest.cluster=localhost:9200 -D tests.cluster=localhost:9200 -D tests.clustername=es-integrationtest -D https=true -D user=admin -D password=admin --stacktrace + .\gradlew.bat :integTestRunner --tests "com.amazon.opendistroforelasticsearch.ad.rest.*IT" --tests "com.amazon.opendistroforelasticsearch.ad.e2e.*IT" -D tests.rest.cluster=localhost:9200 -D tests.cluster=localhost:9200 -D tests.clustername=es-integrationtest -D https=true -D user=admin -D password=admin --stacktrace + + Test-AD-KIBANA-NoSec: + needs: [build-es-artifacts, build-kibana-artifacts] + runs-on: windows-latest + steps: + - uses: actions/checkout@v1 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Set Up JDK + uses: actions/setup-java@v1 + with: + java-version: 14 + + - name: Retrieve plugin tags and odfe versions + run: | + echo ::set-env name=od_version::$(python ./bin/version-info --od) + echo ::set-env name=es_version::$(python ./bin/version-info --es) + echo ::set-env name=p_tag_ad::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection-kibana-plugin) + shell: bash + + - name: Checkout Kibana + uses: actions/checkout@v2 + with: + repository: opendistro-for-elasticsearch/kibana-oss + ref: ${{env.es_version}} + token: ${{ secrets.READ_TOKEN }} + path: kibana + + - name: Get node and yarn versions + id: node_yarn_versions + run: | + echo ::set-env name=kibana_node_version::$(node -p "(require('./kibana/package.json').engines.node).match(/[.0-9]+/)[0]") + echo ::set-env name=kibana_yarn_version::$(node -p "(require('./kibana/package.json').engines.yarn).match(/[.0-9]+/)[0]") + shell: bash + + - name: Setup node + uses: actions/setup-node@v1 + with: + node-version: ${{env.kibana_node_version}} + registry-url: 'https://registry.npmjs.org' + + - name: Install correct yarn version for Kibana + run: | + npm uninstall -g yarn + echo "Installing yarn ${{ env.kibana_yarn_version }}" + npm i -g yarn@${{ env.kibana_yarn_version }} + + - name: Checking out ad kibana repo + uses: actions/checkout@v2 + with: + repository: opendistro-for-elasticsearch/anomaly-detection-kibana-plugin + ref: ${{env.p_tag_ad}} + path: kibana/plugins/anomaly-detection-kibana-plugin + + - name: Bootstrap the AD kibana plugin + run: | + df -h + cd ./kibana/plugins/anomaly-detection-kibana-plugin + yarn kbn bootstrap + - name: run IT for AD kibana plugin + run: | + .github\scripts\setup_runners_service_windows.ps1 --kibana-nosec + dir + npx cypress verify + cd ./kibana/plugins/anomaly-detection-kibana-plugin + yarn cy:run —config baseurl=http://localhost:5601 diff --git a/.github/workflows/test-build-deb.yml b/.github/workflows/test-build-deb.yml index ba8cfb878..c4fff3cd2 100644 --- a/.github/workflows/test-build-deb.yml +++ b/.github/workflows/test-build-deb.yml @@ -133,7 +133,7 @@ jobs: with: repository: opendistro-for-elasticsearch/opendistro-infra ref: jenkins-test - token: ${{ secrets.ODFE_RELEASE_BOT_PRIVATE_READ_WRITE_TOKEN }} + token: ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} - name: Build Ubuntu Docker Image env: DOCKER_USER: ${{ secrets.DOCKER_USER }} @@ -593,7 +593,7 @@ jobs: with: repository: opendistro-for-elasticsearch/kibana-oss ref: ${{env.es_version}} - token: ${{ secrets.ODFE_RELEASE_BOT_PRIVATE_READ_WRITE_TOKEN }} + token: ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} path: kibana - uses: actions/checkout@v2 diff --git a/.github/workflows/test-build-docker.yml b/.github/workflows/test-build-docker.yml index b1e0deb0e..e3385b0b7 100644 --- a/.github/workflows/test-build-docker.yml +++ b/.github/workflows/test-build-docker.yml @@ -365,7 +365,7 @@ jobs: with: repository: opendistro-for-elasticsearch/kibana-oss ref: ${{env.es_version}} - token: ${{ secrets.ODFE_RELEASE_BOT_PRIVATE_READ_WRITE_TOKEN }} + token: ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} path: kibana - uses: actions/checkout@v2 diff --git a/.github/workflows/test-build-tar.yml b/.github/workflows/test-build-tar.yml index 668dcf9db..5d0f8cc30 100644 --- a/.github/workflows/test-build-tar.yml +++ b/.github/workflows/test-build-tar.yml @@ -390,7 +390,7 @@ jobs: with: repository: opendistro-for-elasticsearch/kibana-oss ref: ${{env.es_version}} - token: ${{ secrets.ODFE_RELEASE_BOT_PRIVATE_READ_WRITE_TOKEN }} + token: ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} path: kibana - uses: actions/checkout@v2