From a5e49d5cf83e004c7d851c3884e10e5d4a17e0f2 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 4 Aug 2020 17:47:59 -0400 Subject: [PATCH] Workflow Consistency / Simplification Updates in All Workflows - Part3 (#336) * Simplification workflow for docker and integTests * tweak tar parameters --- .github/scripts/setup_runners_service.sh | 80 +++- .github/workflows/staging-build-docker.yml | 430 ++++++++++----------- 2 files changed, 273 insertions(+), 237 deletions(-) diff --git a/.github/scripts/setup_runners_service.sh b/.github/scripts/setup_runners_service.sh index e22bafa46..61b4f3dff 100755 --- a/.github/scripts/setup_runners_service.sh +++ b/.github/scripts/setup_runners_service.sh @@ -8,13 +8,13 @@ # About: Setup ES/KIBANA for integTests on *NIX based ODFE distros w/wo Security # # Usage: ./setup_runners_service.sh $SETUP_DISTRO $SETUP_ACTION -# $SETUP_DISTRO: zip | deb | rpm (required) +# $SETUP_DISTRO: zip | docker | deb | rpm (required) # $SETUP_ACTION: --es | --es-nosec | --kibana | --kibana-nosec (required) # # Requirements: This script assumes java 14 is already installed on the servers # # Starting Date: 2020-07-27 -# Modified Date: 2020-08-02 +# Modified Date: 2020-08-04 ############################################################################################### # This script allows users to manually assign parameters @@ -22,7 +22,7 @@ if [ "$#" -ne 2 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ] then echo "Please assign 2 parameters when running this script" echo "Example: $0 \$SETUP_DISTRO \$SETUP_ACTION" - echo "Example: $0 \"zip | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec\"" + echo "Example: $0 \"zip | docker | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec\"" exit 1 fi @@ -43,13 +43,18 @@ ES_ROOT="${ROOT}/odfe-testing/${ES_PACKAGE_NAME}" KIBANA_PACKAGE_NAME="opendistroforelasticsearch-kibana" KIBANA_ROOT="${ROOT}/kibana-testing/${KIBANA_PACKAGE_NAME}" +DOCKER_NAME="Test-Docker-${OD_VERSION}" +DOCKER_NAME_KIBANA="Test-Docker-Kibana-${OD_VERSION}" +DOCKER_NAME_NoSec="Test-Docker-${OD_VERSION}-NoSec" +DOCKER_NAME_KIBANA_NoSec="Test-Docker-Kibana-${OD_VERSION}-NoSec" + S3_BUCKET="artifacts.opendistroforelasticsearch.amazon.com" ##################################################################################################### echo "############################################################" echo "Setup ES/KIBANA to start with YES/NO security configurations" -echo "User enters $SETUP_ACTION" +echo "User enters ${SETUP_DISTRO}:${SETUP_ACTION}" echo "############################################################" echo "setup parameters" @@ -63,7 +68,14 @@ if [ "$SETUP_DISTRO" = "zip" ] then mkdir -p $ES_ROOT aws s3 cp s3://$S3_BUCKET/downloads/tarball/opendistro-elasticsearch/$ES_PACKAGE_NAME.tar.gz . --quiet; echo $? - tar -zxf $ES_PACKAGE_NAME.tar.gz -C $ES_ROOT --strip 1 + tar -zxf $ES_PACKAGE_NAME.tar.gz -C $ES_ROOT --strip-components 1 +fi + +if [ "$SETUP_DISTRO" = "docker" ] +then + echo "setup docker" + echo -n > Dockerfile + echo -n > Dockerfile.kibana fi if [ "$SETUP_DISTRO" = "deb" ] @@ -93,10 +105,20 @@ echo "setup es" # everything needs es if [ "$SETUP_DISTRO" = "zip" ] then - sed -i /install_demo_configuration/d $ES_ROOT/opendistro-tar-install.sh - sed -i /^node.max_local_storage_nodes/d $ES_ROOT/config/elasticsearch.yml mkdir -p snapshots echo "path.repo: [\"$PWD/snapshots\"]" >> $ES_ROOT/config/elasticsearch.yml +elif [ "$SETUP_DISTRO" = "docker" ] +then + echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$OD_VERSION" >> Dockerfile + echo "RUN echo '' >> /usr/share/elasticsearch/config/elasticsearch.yml" >> Dockerfile + echo "RUN echo \"path.repo: [\\\"/usr/share/elasticsearch\\\"]\" >> /usr/share/elasticsearch/config/elasticsearch.yml" >> Dockerfile + docker build -t odfe-http:security -f Dockerfile . + sleep 5 + docker run -d -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" --name $DOCKER_NAME odfe-http:security + sleep 30 + echo "Temp Solution to remove the wrong configuration. need be fixed in building stage" + docker exec -t $DOCKER_NAME /bin/bash -c "sed -i /^node.max_local_storage_nodes/d /usr/share/elasticsearch/config/elasticsearch.yml" + docker stop $DOCKER_NAME else sudo mkdir -p /home/repo sudo chmod 777 /home/repo @@ -111,6 +133,14 @@ then then cd $ES_ROOT nohup ./opendistro-tar-install.sh > /dev/null 2>&1 & + sleep 30 + kill -9 `ps -ef | grep [e]lasticsearch | awk '{print $2}'` + sed -i /^node.max_local_storage_nodes/d ./config/elasticsearch.yml + nohup ./opendistro-tar-install.sh > /dev/null 2>&1 & + elif [ "$SETUP_DISTRO" = "docker" ] + then + docker restart $DOCKER_NAME + docker ps else sudo systemctl restart elasticsearch.service fi @@ -132,8 +162,14 @@ then if [ "$SETUP_DISTRO" = "zip" ] then + sed -i /install_demo_configuration/d $ES_ROOT/opendistro-tar-install.sh $ES_ROOT/bin/elasticsearch-plugin remove opendistro_security sed -i '/http\.port/s/^# *//' $ES_ROOT/config/elasticsearch.yml + elif [ "$SETUP_DISTRO" = "docker" ] + then + echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security" >> Dockerfile + docker build -t odfe-http:no-security -f Dockerfile . + sleep 5 else sudo /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security sudo sed -i '/http\.port/s/^# *//' /etc/elasticsearch/elasticsearch.yml @@ -149,6 +185,10 @@ then then cd $ES_ROOT nohup ./opendistro-tar-install.sh > /dev/null 2>&1 & + elif [ "$SETUP_DISTRO" = "docker" ] + then + docker run -d -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" --name $DOCKER_NAME_NoSec odfe-http:no-security + docker ps else sudo systemctl restart elasticsearch.service fi @@ -172,7 +212,14 @@ then then mkdir -p $KIBANA_ROOT aws s3 cp s3://$S3_BUCKET/downloads/tarball/$KIBANA_PACKAGE_NAME/$KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz . --quiet; echo $? - tar -zxf $KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz -C $KIBANA_ROOT --strip 1 + tar -zxf $KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz -C $KIBANA_ROOT --strip-components 1 + elif [ "$SETUP_DISTRO" = "docker" ] + then + echo "FROM opendistroforelasticsearch/opendistroforelasticsearch-kibana:$OD_VERSION" >> Dockerfile.kibana + echo "RUN sed -i /^opendistro_security/d /usr/share/kibana/config/kibana.yml" >> Dockerfile.kibana + echo "RUN sed -i 's/https/http/' /usr/share/kibana/config/kibana.yml" >> Dockerfile.kibana + docker build -t odfe-kibana-http:security -f Dockerfile.kibana . + sleep 5 else sudo apt install $KIBANA_PACKAGE_NAME=$OD_VERSION* -y || sudo yum install $KIBANA_PACKAGE_NAME-$OD_VERSION -y fi @@ -184,8 +231,17 @@ then then cd $ES_ROOT nohup ./opendistro-tar-install.sh > /dev/null 2>&1 & + sleep 30 + kill -9 `ps -ef | grep [e]lasticsearch | awk '{print $2}'` + sed -i /^node.max_local_storage_nodes/d ./config/elasticsearch.yml + nohup ./opendistro-tar-install.sh > /dev/null 2>&1 & cd $KIBANA_ROOT nohup ./bin/kibana > /dev/null 2>&1 & + elif [ "$SETUP_DISTRO" = "docker" ] + then + docker restart $DOCKER_NAME + docker run -d -p 5601:5601 --name $DOCKER_NAME_KIBANA --network="host" odfe-kibana-http:security + docker ps else sudo systemctl restart elasticsearch.service sudo systemctl restart kibana.service @@ -218,6 +274,14 @@ then nohup ./opendistro-tar-install.sh > /dev/null 2>&1 & cd $KIBANA_ROOT nohup ./bin/kibana > /dev/null 2>&1 & + elif [ "$SETUP_DISTRO" = "docker" ] + then + echo "RUN /usr/share/kibana/bin/kibana-plugin remove opendistro_security" >> Dockerfile.kibana + docker build -t odfe-kibana-http:no-security -f Dockerfile.kibana . + sleep 5 + docker run -d -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" --name $DOCKER_NAME_NoSec odfe-http:no-security + docker run -d -p 5601:5601 --name $DOCKER_NAME_KIBANA_NoSec --network="host" odfe-kibana-http:no-security + docker ps else sudo /usr/share/kibana/bin/kibana-plugin remove opendistro_security --allow-root sudo sed -i /^opendistro_security/d /etc/kibana/kibana.yml diff --git a/.github/workflows/staging-build-docker.yml b/.github/workflows/staging-build-docker.yml index 183fcb04b..88c12ff77 100644 --- a/.github/workflows/staging-build-docker.yml +++ b/.github/workflows/staging-build-docker.yml @@ -87,209 +87,6 @@ jobs: # Optional content type: content_type: text/html - test-es-plugins-without-security: - needs: [build-es-docker] - strategy: - matrix: - java: [14] - name: test-es-plugins-without-security - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - - name: Retrieve plugin tags - run: | - echo ::set-env name=p_tag_ism::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/index-management) - echo ::set-env name=p_tag_alerting::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/alerting) - echo ::set-env name=p_tag_sql::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql) - echo ::set-env name=p_tag_knn::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/k-NN) - echo ::set-env name=p_tag_ad::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection) - - - name: Checkout ISM - uses: actions/checkout@v1 - with: - repository: opendistro-for-elasticsearch/index-management - ref: ${{env.p_tag_ism}} - - - name: Checkout Alerting - uses: actions/checkout@v1 - with: - repository: opendistro-for-elasticsearch/alerting - ref: ${{env.p_tag_alerting}} - - - name: Checkout SQL - uses: actions/checkout@v1 - with: - repository: opendistro-for-elasticsearch/sql - ref: ${{env.p_tag_sql}} - - - name: Checkout k-NN - uses: actions/checkout@v1 - with: - repository: opendistro-for-elasticsearch/k-NN - ref: ${{env.p_tag_knn}} - - - name: Checkout AD - uses: actions/checkout@v1 - with: - repository: opendistro-for-elasticsearch/anomaly-detection - ref: ${{env.p_tag_ad}} - - - name: Configure AWS - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Set Up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - - name: Start ES Container without Security - run: | - ODFE_VER=`./bin/version-info --od` - echo "ODFE VERSION $ODFE_VER" - cd .. - DOCKER_NAME=odfe-test-$ODFE_VER - echo "Running newly tagged image with no security" - echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$ODFE_VER" >> Dockerfile - echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security" >> Dockerfile - echo "RUN echo '' >> /usr/share/elasticsearch/config/elasticsearch.yml" >> Dockerfile - echo "RUN echo \"path.repo: [\\\"/usr/share/elasticsearch\\\"]\" >> /usr/share/elasticsearch/config/elasticsearch.yml" >> Dockerfile - docker build -t odfe-http:no-security . - sleep 5 - - docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" --name $DOCKER_NAME odfe-http:no-security - sleep 30 - curl -XGET http://localhost:9200/_cat/plugins - - - name: Test ISM Plugin - if: always() - run: | - cd ../index-management - ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest - - - name: Test Alerting Plugin - if: always() - run: | - cd ../alerting/alerting - ../gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest - - - name: Test SQL Plugin - if: always() - run: | - cd ../sql - ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest - - - name: Test k-NN Plugin - if: always() - run: | - cd ../k-NN - ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest - - - name: Test AD Plugin - if: always() - run: | - cd ../anomaly-detection - ./gradlew :integTestRunner --tests "com.amazon.opendistroforelasticsearch.ad.rest.*IT" --tests "com.amazon.opendistroforelasticsearch.ad.e2e.*IT" -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest - - test-es-plugins-with-security: - needs: [build-es-docker] - strategy: - matrix: - java: [14] - name: test-es-plugins-with-security - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - - name: Retrieve plugin tags - run: | - echo ::set-env name=p_tag_ism::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/index-management) - echo ::set-env name=p_tag_alerting::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/alerting) - echo ::set-env name=p_tag_sql::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql) - echo ::set-env name=p_tag_knn::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/k-NN) - echo ::set-env name=p_tag_ad::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection) - - - name: Checkout ISM - uses: actions/checkout@v1 - with: - repository: opendistro-for-elasticsearch/index-management - ref: ${{env.p_tag_ism}} - - - name: Checkout Alerting - uses: actions/checkout@v1 - with: - repository: opendistro-for-elasticsearch/alerting - ref: ${{env.p_tag_alerting}} - - - name: Checkout SQL - uses: actions/checkout@v1 - with: - repository: opendistro-for-elasticsearch/sql - ref: ${{env.p_tag_sql}} - - - name: Checkout k-NN - uses: actions/checkout@v1 - with: - repository: opendistro-for-elasticsearch/k-NN - ref: ${{env.p_tag_knn}} - - - name: Checkout AD - uses: actions/checkout@v1 - with: - repository: opendistro-for-elasticsearch/anomaly-detection - ref: ${{env.p_tag_ad}} - - - name: Configure AWS - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Set Up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - - name: Start ES Container with Security - run: | - ODFE_VER=`./bin/version-info --od` - echo "ODFE VERSION $ODFE_VER" - cd .. - DOCKER_NAME=odfe-test-$ODFE_VER - echo "Running newly tagged image with security" - echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$ODFE_VER" >> Dockerfile - echo "RUN echo '' >> /usr/share/elasticsearch/config/elasticsearch.yml" >> Dockerfile - echo "RUN echo \"path.repo: [\\\"/usr/share/elasticsearch\\\"]\" >> /usr/share/elasticsearch/config/elasticsearch.yml" >> Dockerfile - docker build -t odfe-http:security . - sleep 5 - - docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" --name $DOCKER_NAME odfe-http:security - sleep 30 - curl -XGET https://localhost:9200/_cat/plugins -u admin:admin --insecure - - echo "Temp Solution to remove the wrong configuration. need be fixed in building stage" - docker exec -t $DOCKER_NAME /bin/bash -c "sed -i /^node.max_local_storage_nodes/d /usr/share/elasticsearch/config/elasticsearch.yml" - docker restart $DOCKER_NAME - sleep 30 - curl -XGET https://localhost:9200/_cat/plugins -u admin:admin --insecure - - - name: Test SQL Plugin - if: always() - run: | - cd ../sql - ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest -Dhttps=true -Duser=admin -Dpassword=admin - - - name: Test AD Plugin - if: always() - run: | - cd ../anomaly-detection - ./gradlew :integTestRunner --tests "com.amazon.opendistroforelasticsearch.ad.rest.*IT" --tests "com.amazon.opendistroforelasticsearch.ad.e2e.*IT" -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="es-integrationtest" -Dhttps=true -Duser=admin -Dpassword=admin - build-kibana-docker: runs-on: [ubuntu-16.04] name: Build Kibana Docker @@ -346,6 +143,206 @@ jobs: # Optional content type: content_type: text/html + Test-ISM-NoSec: + needs: [build-es-docker] + runs-on: [ubuntu-latest] + name: Test-ISM-NoSec + strategy: + matrix: + java: [14] + steps: + - uses: actions/checkout@v1 + + - name: Retrieve plugin tags + run: echo ::set-env name=p_tag_ism::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/index-management) + + - uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/index-management + ref: ${{env.p_tag_ism}} + + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: IT + run: | + .github/scripts/setup_runners_service.sh docker --es-nosec + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../index-management; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest + + + Test-ALERTING-NoSec: + needs: [build-es-docker] + runs-on: [ubuntu-latest] + name: Test-ALERTING-NoSec + strategy: + matrix: + java: [14] + steps: + - uses: actions/checkout@v1 + + - name: Retrieve plugin tags + run: echo ::set-env name=p_tag_alerting::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/alerting) + + - uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/alerting + ref: ${{env.p_tag_alerting}} + + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: IT + run: | + .github/scripts/setup_runners_service.sh docker --es-nosec + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../alerting/alerting; pwd + ../gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest + + Test-SQL-NoSec: + needs: [build-es-docker] + runs-on: [ubuntu-latest] + name: Test-SQL-NoSec + strategy: + matrix: + java: [14] + steps: + - uses: actions/checkout@v1 + + - name: Retrieve plugin tags + run: echo ::set-env name=p_tag_sql::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql) + + - uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/sql + ref: ${{env.p_tag_sql}} + + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: IT + run: | + .github/scripts/setup_runners_service.sh docker --es-nosec + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../sql; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest + + Test-KNN-NoSec: + needs: [build-es-docker] + runs-on: [ubuntu-latest] + name: Test-KNN-NoSec + strategy: + matrix: + java: [14] + steps: + - uses: actions/checkout@v1 + - name: Retrieve plugin tags + run: echo ::set-env name=p_tag_knn::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/k-NN) + - uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/k-NN + ref: ${{env.p_tag_knn}} + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - name: IT + run: | + .github/scripts/setup_runners_service.sh docker --es-nosec + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../k-NN; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest + + Test-AD-NoSec: + needs: [build-es-docker] + runs-on: [ubuntu-latest] + name: Test-AD-NoSec + strategy: + matrix: + java: [14] + steps: + - uses: actions/checkout@v1 + + - name: Retrieve plugin tags + run: echo ::set-env name=p_tag_ad::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection) + + - uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/anomaly-detection + ref: ${{env.p_tag_ad}} + + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: IT + run: | + .github/scripts/setup_runners_service.sh docker --es-nosec + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../anomaly-detection; pwd + ./gradlew :integTestRunner --tests "com.amazon.opendistroforelasticsearch.ad.rest.*IT" --tests "com.amazon.opendistroforelasticsearch.ad.e2e.*IT" -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="es-integrationtest" + + Test-SQL: + needs: [build-es-docker] + runs-on: [ubuntu-latest] + name: Test-SQL + strategy: + matrix: + java: [14] + steps: + - uses: actions/checkout@v1 + + - name: Retrieve plugin tags + run: echo ::set-env name=p_tag_sql::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/sql) + + - uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/sql + ref: ${{env.p_tag_sql}} + + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: IT + run: | + .github/scripts/setup_runners_service.sh docker --es + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../sql; pwd + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=es-integrationtest -Dhttps=true -Duser=admin -Dpassword=admin + + Test-AD: + needs: [build-es-docker] + runs-on: [ubuntu-latest] + name: Test-AD + strategy: + matrix: + java: [14] + steps: + - uses: actions/checkout@v1 + + - name: Retrieve plugin tags + run: echo ::set-env name=p_tag_ad::$(.github/scripts/plugin_tag.sh opendistro-for-elasticsearch/anomaly-detection) + + - uses: actions/checkout@v1 + with: + repository: opendistro-for-elasticsearch/anomaly-detection + ref: ${{env.p_tag_ad}} + + - name: Setup Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: IT + run: | + .github/scripts/setup_runners_service.sh docker --es + export PATH=$JAVA_HOME:$PATH; cd $GITHUB_WORKSPACE/../anomaly-detection; pwd + ./gradlew :integTestRunner --tests "com.amazon.opendistroforelasticsearch.ad.rest.*IT" --tests "com.amazon.opendistroforelasticsearch.ad.e2e.*IT" -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="es-integrationtest" -Dhttps=true -Duser=admin -Dpassword=admin + Test-AD-Kibana-Plugin: needs: [build-es-docker, build-kibana-docker] runs-on: [ubuntu-16.04] @@ -403,32 +400,7 @@ jobs: yarn kbn bootstrap - name: Start ES and Kibana - run: | - ODFE_VER=`./bin/version-info --od` - echo "ODFE VERSION $ODFE_VER" - cd .. - DOCKER_NAME=odfe-test-$ODFE_VER - echo "Running newly tagged es image with no security" - echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$ODFE_VER" >> Dockerfile - echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security" >> Dockerfile - echo "RUN echo '' >> /usr/share/elasticsearch/config/elasticsearch.yml" >> Dockerfile - echo "RUN echo \"path.repo: [\\\"/usr/share/elasticsearch\\\"]\" >> /usr/share/elasticsearch/config/elasticsearch.yml" >> Dockerfile - docker build -t odfe-http:no-security . - sleep 5 - docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" --name $DOCKER_NAME odfe-http:no-security - sleep 30 - curl -XGET http://localhost:9200/ - - echo "Running newly tagged kibana image with no security" - echo "FROM opendistroforelasticsearch/opendistroforelasticsearch-kibana:$ODFE_VER" >> Dockerfile - echo "RUN /usr/share/kibana/bin/kibana-plugin remove opendistro_security" >> Dockerfile - echo "RUN sed -i /^opendistro_security/d /usr/share/kibana/config/kibana.yml" >> Dockerfile - echo "RUN sed -i 's/https/http/' /usr/share/kibana/config/kibana.yml" >> Dockerfile - docker build -t odfe-kibana-http:no-security . - sleep 5 - docker run -d --name odfe-kibana-it --network="host" odfe-kibana-http:no-security - sleep 120 - curl -v http://127.0.0.1:5601 + run: .github/scripts/setup_runners_service.sh docker --kibana-nosec - name: run IT uses: cypress-io/github-action@v1