Workflow Consistency / Simplification Updates in All Workflows - Part2 (

#335) * Improve rpm windows and adding deb tar simplifications * rename release workflows to prod workflows * Delete setup_runners_service.sh_knnlib * Update setup_runners_service_windows.ps1 * Update setup_runners_service.sh * Update setup_runners.sh
opendistro-for-elasticsearch · Aug 11, 2020 · 4a634d4 · 4a634d4
1 parent 9f21d2c
commit 4a634d4
Show file tree

Hide file tree

Showing 15 changed files with 289 additions and 575 deletions.
diff --git a/.github/scripts/setup_runners.sh b/.github/scripts/setup_runners.sh
@@ -47,7 +47,7 @@
 #                }
 #
 # Starting Date: 2020-07-27
-# Modified Date: 2020-07-30
+# Modified Date: 2020-08-02
 ###############################################################################################
 
 set -e
@@ -68,7 +68,7 @@ fi
 SETUP_ACTION=$1
 SETUP_INSTANCE=`echo $2 | sed 's/,/ /g'`
 SETUP_TOKEN=$3
-SETUP_AMI_ID="ami-0397210a1b7e7e74c"
+SETUP_AMI_ID="ami-0655d7056d3941cd2"
 SETUP_AMI_USER="ec2-user"
 SETUP_INSTANCE_TYPE="m5.xlarge"
 SETUP_INSTANCE_SIZE=20 #GiB

diff --git a/.github/scripts/setup_runners_service.sh b/.github/scripts/setup_runners_service.sh
@@ -8,19 +8,21 @@
 # About:         Setup ES/KIBANA for integTests on *NIX based ODFE distros w/wo Security
 #
 # Usage:         ./setup_runners_service.sh $SETUP_DISTRO $SETUP_ACTION
-#                $SETUP_DISTRO: tar | deb | rpm (required)
+#                $SETUP_DISTRO: zip | deb | rpm (required)
 #                $SETUP_ACTION: --es | --es-nosec | --kibana | --kibana-nosec (required)
 #
+# Requirements:  This script assumes java 14 is already installed on the servers
+#
 # Starting Date: 2020-07-27
-# Modified Date: 2020-07-30
+# Modified Date: 2020-08-02
 ###############################################################################################
 
 # This script allows users to manually assign parameters
 if [ "$#" -ne 2 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ]
 then
   echo "Please assign 2 parameters when running this script"
   echo "Example: $0 \$SETUP_DISTRO \$SETUP_ACTION"
-  echo "Example: $0 \"tar | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec\""
+  echo "Example: $0 \"zip | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec\""
   exit 1
 fi
 
@@ -36,6 +38,13 @@ ROOT=`dirname $(realpath $0)`; cd $ROOT
 OD_VERSION=`python $REPO_ROOT/bin/version-info --od`
 ES_VERSION=`python $REPO_ROOT/bin/version-info --es`
 
+ES_PACKAGE_NAME="opendistroforelasticsearch-${OD_VERSION}"
+ES_ROOT="${ROOT}/odfe-testing/${ES_PACKAGE_NAME}"
+KIBANA_PACKAGE_NAME="opendistroforelasticsearch-kibana"
+KIBANA_ROOT="${ROOT}/kibana-testing/${KIBANA_PACKAGE_NAME}"
+
+S3_BUCKET="artifacts.opendistroforelasticsearch.amazon.com"
+
 #####################################################################################################
 
 echo "############################################################"
@@ -45,59 +54,70 @@ echo "############################################################"
 
 echo "setup parameters"
 echo $JAVA_HOME
-export PATH=$PATH:$JAVA_HOME
+export PATH=$JAVA_HOME:$PATH
+which java
+java -version
 sudo sysctl -w vm.max_map_count=262144
 
-if [ "$SETUP_DISTRO" = "tar" ]
+if [ "$SETUP_DISTRO" = "zip" ]
 then
-  mkdir -p odfe-testing
-  cd odfe-testing
-  aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/tarball/opendistro-elasticsearch/opendistroforelasticsearch-$OD_VERSION.tar.gz . --quiet; echo $?
-  tar -zxf opendistroforelasticsearch-$OD_VERSION.tar.gz
-  cd opendistroforelasticsearch-$OD_VERSION
+  mkdir -p $ES_ROOT
+  aws s3 cp s3://$S3_BUCKET/downloads/tarball/opendistro-elasticsearch/$ES_PACKAGE_NAME.tar.gz . --quiet; echo $?
+  tar -zxf $ES_PACKAGE_NAME.tar.gz -C $ES_ROOT --strip 1
 fi
 
 if [ "$SETUP_DISTRO" = "deb" ]
 then
+  sudo add-apt-repository ppa:openjdk-r/ppa
+  sudo apt update
   sudo sudo apt install -y net-tools
   wget -qO - https://d3g5vo6xdbdb9a.cloudfront.net/GPG-KEY-opendistroforelasticsearch | sudo apt-key add -
   echo "deb https://d3g5vo6xdbdb9a.cloudfront.net/staging/apt stable main" | sudo tee -a /etc/apt/sources.list.d/opendistroforelasticsearch.list
-  wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-$ES_VERSION-amd64.deb
+  wget -q https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-$ES_VERSION-amd64.deb
   sudo dpkg -i elasticsearch-oss-$ES_VERSION-amd64.deb
   sudo apt update -y
-  sudo apt install -y opendistroforelasticsearch-$OD_VERSION
+  sudo apt install -y opendistroforelasticsearch=$OD_VERSION*
 fi
 
 if [ "$SETUP_DISTRO" = "rpm" ]
 then
   sudo curl https://d3g5vo6xdbdb9a.cloudfront.net/yum/staging-opendistroforelasticsearch-artifacts.repo  -o /etc/yum.repos.d/staging-opendistroforelasticsearch-artifacts.repo
   sudo yum update
-  sudo yum install opendistroforelasticsearch-$OD_VERSION -y
+  sudo yum install $ES_PACKAGE_NAME -y
 fi
 
 #####################################################################################################
 
 echo "setup es"
 
 # everything needs es
-if [ "$SETUP_DISTRO" = "tar" ]
+if [ "$SETUP_DISTRO" = "zip" ]
 then
-  sed -i /install_demo_configuration/d opendistro-tar-install.sh
-  sed -i '/http\.port/s/^# *//' config/elasticsearch.yml
-  nohup ./opendistro-tar-install.sh &          
+  sed -i /install_demo_configuration/d $ES_ROOT/opendistro-tar-install.sh
+  sed -i /^node.max_local_storage_nodes/d $ES_ROOT/config/elasticsearch.yml
+  mkdir -p snapshots
+  echo "path.repo: [\"$PWD/snapshots\"]" >> $ES_ROOT/config/elasticsearch.yml
 else
   sudo mkdir -p /home/repo
   sudo chmod 777 /home/repo
   sudo chmod 777 /etc/elasticsearch/elasticsearch.yml
   sudo sed -i '/path.logs/a path.repo: ["/home/repo"]' /etc/elasticsearch/elasticsearch.yml
   sudo sed -i /^node.max_local_storage_nodes/d /etc/elasticsearch/elasticsearch.yml
-  sudo systemctl restart elasticsearch.service
 fi
 
 if [ "$SETUP_ACTION" = "--es" ]
 then
-  sleep 60
+  if [ "$SETUP_DISTRO" = "zip" ]
+  then
+    cd $ES_ROOT
+    nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
+  else
+    sudo systemctl restart elasticsearch.service
+  fi
+  echo "Sleep 120 seconds"
+  sleep 120
   curl -XGET https://localhost:9200 -u admin:admin --insecure
+  curl -XGET https://localhost:9200/_cluster/health?pretty -u admin:admin --insecure
   echo "es start"
   cd $REPO_ROOT
   exit 0
@@ -110,23 +130,32 @@ if [ "$SETUP_ACTION" = "--es-nosec" ] || [ "$SETUP_ACTION" = "--kibana-nosec" ]
 then
   echo "remove es security"
 
-  if [ "$SETUP_DISTRO" = "tar" ]
+  if [ "$SETUP_DISTRO" = "zip" ]
   then
-    ./bin/elasticsearch-plugin remove opendistro_security
-    nohup ./opendistro-tar-install.sh &          
+    $ES_ROOT/bin/elasticsearch-plugin remove opendistro_security
+    sed -i '/http\.port/s/^# *//' $ES_ROOT/config/elasticsearch.yml
   else
     sudo /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security 
+    sudo sed -i '/http\.port/s/^# *//' /etc/elasticsearch/elasticsearch.yml
     sudo sed -i /^opendistro_security/d /etc/elasticsearch/elasticsearch.yml
     sudo sed -i /CN=kirk/d /etc/elasticsearch/elasticsearch.yml
     sudo sed -i /^cluster.routing.allocation.disk.threshold_enabled/d /etc/elasticsearch/elasticsearch.yml
-    sudo systemctl restart elasticsearch.service
   fi
 fi
 
 if [ "$SETUP_ACTION" = "--es-nosec" ]
 then
-  sleep 60
+  if [ "$SETUP_DISTRO" = "zip" ]
+  then
+    cd $ES_ROOT
+    nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
+  else
+    sudo systemctl restart elasticsearch.service
+  fi
+  echo "Sleep 120 seconds"
+  sleep 120
   curl -XGET http://localhost:9200
+  curl -XGET http://localhost:9200/_cluster/health?pretty
   echo "es-nosec start"
   cd $REPO_ROOT
   exit 0
@@ -139,31 +168,34 @@ if [ "$SETUP_ACTION" = "--kibana" ] || [ "$SETUP_ACTION" = "--kibana-nosec" ]
 then
   echo "setup kibana"
 
-  if [ "$SETUP_DISTRO" = "tar" ]
+  if [ "$SETUP_DISTRO" = "zip" ]
   then
-    cd ../../
-    mkdir -p kibana-testing
-    cd kibana-testing
-    aws s3 cp s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/tarball/opendistroforelasticsearch-kibana/opendistroforelasticsearch-kibana-$odfe_version.tar.gz . --quiet; echo $?
-    tar -zxf opendistroforelasticsearch-kibana-$odfe_version.tar.gz
-    cd opendistroforelasticsearch-kibana
-    sed -i /install_demo_configuration/d opendistro-tar-install.sh
-    cd ../../odfe-testing
-    nohup ./opendistro-tar-install.sh &
-    cd ../../kibana-testing
-    nohup ./bin/kibana &
+    mkdir -p $KIBANA_ROOT
+    aws s3 cp s3://$S3_BUCKET/downloads/tarball/$KIBANA_PACKAGE_NAME/$KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz . --quiet; echo $?
+    tar -zxf $KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz -C $KIBANA_ROOT --strip 1
   else
-    sudo apt install opendistroforelasticsearch-kibana-$OD_VERSION -y || sudo yum install opendistroforelasticsearch-kibana-$OD_VERSION -y
-    sudo systemctl restart elasticsearch.service
-    sudo systemctl restart kibana.service
+    sudo apt install $KIBANA_PACKAGE_NAME=$OD_VERSION* -y || sudo yum install $KIBANA_PACKAGE_NAME-$OD_VERSION -y
   fi
 fi
 
 if [ "$SETUP_ACTION" = "--kibana" ]
 then
+  if [ "$SETUP_DISTRO" = "zip" ]
+  then
+    cd $ES_ROOT
+    nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
+    cd $KIBANA_ROOT
+    nohup ./bin/kibana > /dev/null 2>&1 &
+  else
+    sudo systemctl restart elasticsearch.service
+    sudo systemctl restart kibana.service
+  fi
+  echo "Sleep 120 seconds"
   sleep 120
   curl -XGET https://localhost:9200 -u admin:admin --insecure
+  curl -XGET https://localhost:9200/_cluster/health?pretty -u admin:admin --insecure
   curl -v -XGET https://localhost:5601 --insecure
+  curl -v -XGET https://localhost:5601/api/status --insecure
   echo "es & kibana start"
   cd $REPO_ROOT
   exit 0
@@ -176,28 +208,31 @@ if [ "$SETUP_ACTION" = "--kibana-nosec" ]
 then
   echo "remove kibana security"
 
-  if [ "$SETUP_DISTRO" = "tar" ]
+  if [ "$SETUP_DISTRO" = "zip" ]
   then
-    ./bin/kibana-plugin remove opendistro_security 
-    sed -i /^opendistro_security/d ./config/kibana.yml
-    sed -i 's/https/http/' ./config/kibana.yml
-    cd ../../odfe-testing
-    nohup ./opendistro-tar-install.sh &
-    cd ../../kibana-testing
-    nohup ./bin/kibana &
+    $KIBANA_ROOT/bin/kibana-plugin remove opendistro_security 
+    sed -i /^opendistro_security/d $KIBANA_ROOT/config/kibana.yml
+    sed -i 's/https/http/' $KIBANA_ROOT/config/kibana.yml
+
+    cd $ES_ROOT
+    nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
+    cd $KIBANA_ROOT
+    nohup ./bin/kibana > /dev/null 2>&1 &
   else
     sudo /usr/share/kibana/bin/kibana-plugin remove opendistro_security --allow-root
     sudo sed -i /^opendistro_security/d /etc/kibana/kibana.yml
     sudo sed -i 's/https/http/' /etc/kibana/kibana.yml
     sudo systemctl restart elasticsearch.service
     sudo systemctl restart kibana.service
   fi
-
-    sleep 120
-    curl -XGET http://localhost:9200
-    curl -v -XGET http://localhost:5601
-    echo "es & kibana-nosec start"
+  echo "Sleep 120 seconds"
+  sleep 120
+  curl -XGET http://localhost:9200
+  curl -XGET http://localhost:9200/_cluster/health?pretty
+  curl -v -XGET http://localhost:5601
+  curl -v -XGET http://localhost:5601/api/status
+  echo "es & kibana-nosec start"
   cd $REPO_ROOT
-    exit 0
+  exit 0
 fi
 
diff --git a/.github/scripts/setup_runners_service_windows.ps1 b/.github/scripts/setup_runners_service_windows.ps1
@@ -10,8 +10,10 @@
 # Usage:         ./setup_runners_service_windows.ps1 $SETUP_ACTION
 #                $SETUP_ACTION: --es | --es-nosec | --kibana | --kibana-nosec (required)
 #
+# Requirements:  This script assumes java 14 is already installed on the servers
+#
 # Starting Date: 2020-07-27
-# Modified Date: 2020-07-30
+# Modified Date: 2020-08-02
 ###############################################################################################
 
 # Keep the pwsh script running even with errors
@@ -31,6 +33,7 @@ echo "User enters $SETUP_ACTION"
 echo "############################################################"
 
 echo "setup es"
+java -version
 dir
 python -m pip install --upgrade pip
 echo pip3 -version
@@ -58,9 +61,10 @@ if ($SETUP_ACTION -eq "--es"){
   echo "running es"
   nohup .\$PACKAGE-$OD_VERSION\bin\elasticsearch.bat &
 
-  echo "Waiting for 60 seconds"
-  ping -n 60 127.0.0.1 >.\out.txt
-  curl -XGET https://localhost:9200 -u admin:admin --insecure
+  echo "Waiting for 160 seconds"
+  ping -n 160 127.0.0.1 >.\out.txt
+  #curl -XGET https://localhost:9200 -u admin:admin --insecure
+  #curl -XGET https://localhost:9200/_cluster/health?pretty -u admin:admin --insecure
 
   echo "es start"
   exit 0
@@ -84,9 +88,10 @@ if ($SETUP_ACTION -eq "--es-nosec"){
   echo "running es"
   nohup .\$PACKAGE-$OD_VERSION\bin\elasticsearch.bat &
 
-  echo "Waiting for 60 seconds"
-  ping -n 60 127.0.0.1 >.\out.txt
-  curl -XGET http://localhost:9200
+  echo "Waiting for 160 seconds"
+  ping -n 160 127.0.0.1 >.\out.txt
+  #curl -XGET http://localhost:9200
+  #curl -XGET http://localhost:9200/_cluster/health?pretty
 
   echo "es-nosec start"
   exit 0
@@ -115,10 +120,12 @@ if ($SETUP_ACTION -eq "--kibana"){
   nohup .\bin\kibana.bat &
   cd ..\..
 
-  echo "Waiting for 120 seconds"
-  ping -n 120 127.0.0.1 >.\out.txt
-  curl -XGET https://localhost:9200 -u admin:admin --insecure
-  curl -v -XGET https://localhost:5601 --insecure
+  echo "Waiting for 160 seconds"
+  ping -n 160 127.0.0.1 >.\out.txt
+  #curl -XGET https://localhost:9200 -u admin:admin --insecure
+  #curl -XGET https://localhost:9200/_cluster/health?pretty -u admin:admin --insecure
+  #curl -v -XGET https://localhost:5601 --insecure
+  #curl -v -XGET https://localhost:5601/api/status --insecure
 
   echo "kibana start"
   exit 0
@@ -144,10 +151,12 @@ if ($SETUP_ACTION -eq "--kibana-nosec"){
   nohup .\bin\kibana.bat &
   cd ..\..
 
-  echo "Waiting for 120 seconds"
-  ping -n 120 127.0.0.1 >.\out.txt
-  curl -XGET http://localhost:9200
-  curl -v -XGET http://localhost:5601
+  echo "Waiting for 160 seconds"
+  ping -n 160 127.0.0.1 >.\out.txt
+  #curl -XGET http://localhost:9200
+  #curl -XGET http://localhost:9200/_cluster/health?pretty
+  #curl -v -XGET http://localhost:5601
+  #curl -v -XGET http://localhost:5601/api/status
 
   echo "kibana-nosec start"
   exit 0

diff --git a/.github/workflows/release-sync-ami.yml → .github/workflows/prod-sync-ami.yml b/.github/workflows/release-sync-ami.yml → .github/workflows/prod-sync-ami.yml
@@ -2,7 +2,7 @@ name: Sync ami artifacts to Prod
 
 on: 
   repository_dispatch:
-    types: [release-sync-ami]
+    types: [prod-sync-ami]
 
 jobs:
   Sync-AMI-Prod:

diff --git a/.github/workflows/release-sync-deb.yml → .github/workflows/prod-sync-deb.yml b/.github/workflows/release-sync-deb.yml → .github/workflows/prod-sync-deb.yml
@@ -2,7 +2,7 @@ name: Sync Deb artifacts to Prod
 
 on: 
   repository_dispatch:
-    types: [release-sync-deb]
+    types: [prod-sync-deb]
 
 jobs:
   Sync-Deb-Prod:

diff --git a/.github/workflows/release-sync-rpm.yml → .github/workflows/prod-sync-rpm.yml b/.github/workflows/release-sync-rpm.yml → .github/workflows/prod-sync-rpm.yml
@@ -2,7 +2,7 @@ name: Sync RPM artifacts to Production
 
 on:
   repository_dispatch:
-    types: [release-sync-rpm]
+    types: [prod-sync-rpm]
 
 jobs:
   signing-artifacts:   

diff --git a/.github/workflows/release-sync-tar.yml → .github/workflows/prod-sync-tar.yml b/.github/workflows/release-sync-tar.yml → .github/workflows/prod-sync-tar.yml
@@ -2,7 +2,7 @@ name: Sync tar artifacts to Prod
 
 on: 
   repository_dispatch:
-    types: [release-sync-tar]
+    types: [prod-sync-tar]
 
 jobs:
   Sync-Tarball-Prod: