From 84633813fe339e5e8d85b60eb373c5fa691aa4f1 Mon Sep 17 00:00:00 2001
From: Kawika Avilla <kavilla414@gmail.com>
Date: Sat, 12 Mar 2022 07:09:50 +0000
Subject: [PATCH 1/6] [Tests] BWC test improvements for CI

There were a couple issues related the BWC that was not supported
within the current iteration of the BWC script.

This fixes the following:
* Format of files being saved with color looks unreadable in txt form
* Exit code not being cypress exit code which would be the amount of errors.
  * Issue because texts would fail but would return 0 to test runner because the script worked
* CI test runner expects to be able to call the same files twice (security:no-security)
  * Issue because results for both were saved at the same time while original it was wiped clean
* Support from 'Playground' environments
* Source to actually call the script
* wget not available in CI runner
* Track PID of process instead of greping since greping was not reliable.
* OpenSearch config setup

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
---
 bwctest.sh             |  7 ++-
 scripts/bwctest_osd.sh | 98 +++++++++++++++++++++++++++++++-----------
 2 files changed, 75 insertions(+), 30 deletions(-)

diff --git a/bwctest.sh b/bwctest.sh
index 9f2934d62526..cd19906f976e 100755
--- a/bwctest.sh
+++ b/bwctest.sh
@@ -13,8 +13,7 @@
 
 set -e
 
-# TODO: Update to include all known BWC of data
-DEFAULT_VERSIONS="osd-1.1.0"
+DEFAULT_VERSIONS="osd-1.1.0,odfe-1.13.2"
 
 function usage() {
     echo ""
@@ -80,7 +79,7 @@ done
 if [ -z "$OPENSEARCH" ]; then
     IFS='/' read -ra SLASH_ARR <<< "$DASHBOARDS"
     # Expected to be opensearch-x.y.z-platform-arch.tar.gz
-    TARBALL="${SLASH_ARR[12]}"
+    [[ "$DASHBOARDS" == *"Playground"* ]] && TARBALL="${SLASH_ARR[13]}" || TARBALL="${SLASH_ARR[12]}"
     IFS='-' read -ra DASH_ARR <<< "$TARBALL"
     # Expected to be arch.tar.gz
     DOTS="${DASH_ARR[4]}"
@@ -93,4 +92,4 @@ if [ -z "$OPENSEARCH" ]; then
     OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz"
 fi
 
-./scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS
+source scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS
diff --git a/scripts/bwctest_osd.sh b/scripts/bwctest_osd.sh
index 9c131e50fc10..6ed14c3e19a7 100755
--- a/scripts/bwctest_osd.sh
+++ b/scripts/bwctest_osd.sh
@@ -101,19 +101,20 @@ done
 [ $SECURITY_ENABLED == "false" ] && releases_array=() || IFS=',' read -r -a releases_array <<<"$RELEASES"
 [ -z "$CREDENTIAL" ] && CREDENTIAL="admin:admin"
 
+total_test_failures=0
+# OpenSearch and OpenSearch Dashboards Process IDs
+parent_pid_list=()
 # define test path
 cwd=$(pwd)
 dir="$cwd/bwc_tmp"
 test_dir="$dir/test"
 opensearch_dir="$dir/opensearch"
 dashboards_dir="$dir/opensearch-dashboards"
-if [ -d "$dir" ]; then
-  echo "Temporary directory exists. Removing."   
-  rm -rf "$dir"
-fi
-mkdir "$dir"
-mkdir "$test_dir"
+[ ! -d "$dir" ] && mkdir "$dir"
+[ ! -d "$test_dir" ] && mkdir "$test_dir"
+[ -d "$opensearch_dir" ] && rm -rf "$opensearch_dir"
 mkdir "$opensearch_dir"
+[ -d "$dashboards_dir" ] && rm -rf "$dashboards_dir"
 mkdir "$dashboards_dir"
 
 function open_artifact {
@@ -122,8 +123,8 @@ function open_artifact {
   cd $artifact_dir
   
   # check if artifact provided is URL or attempt if passing by absolute path
-  if wget -q --method=HEAD $artifact; then
-    wget -c $artifact -O - | tar -xz --strip-components=1
+  if curl -I -L $artifact; then
+    curl -L $artifact | tar -xz --strip-components=1
   else
     tar -xf $artifact --strip-components=1
   fi
@@ -132,6 +133,7 @@ function open_artifact {
 
 # un-tar OpenSearch and OpenSearch Dashboards
 echo "[ unzip OpenSearch and OpenSearch Dashboards ]"
+echo $OPENSEARCH
 open_artifact $opensearch_dir $OPENSEARCH
 open_artifact $dashboards_dir $DASHBOARDS
 
@@ -173,12 +175,21 @@ test_suites=(
 
 # remove the running opensearch process
 function clean {
-  echo "Closing the running OpenSearch"
-  process=($(ps -ef | grep "Dopensearch" | awk '{print $2}'))
-  kill ${process[0]}
-  echo "Closing any usage on port $BIND_PORT"
-  process=($(lsof -i -P -n | grep $BIND_PORT | awk '{print $2}'))
-  kill ${process[0]}
+  echo "Attempt to Terminate Process with PID: ${parent_pid_list[*]}"
+  for pid_kill in "${parent_pid_list[@]}"
+  do
+    echo "Closing PID $pid_kill"
+    kill $pid_kill
+  done
+  parent_pid_list=()
+}
+
+function spawn_process_and_save_PID {
+    echo "Spawn '$@'"
+    eval $@
+    curr_pid=$!
+    echo "PID: $curr_pid"
+    parent_pid_list+=( $curr_pid )
 }
 
 # Print out a textfile line by line
@@ -215,6 +226,18 @@ function setup_cypress {
   echo "Cypress is ready!"
 }
 
+function run_cypress {
+    [ $1 == "core" ] && is_core=true || is_core=false
+    [ ! $is_core ] && echo "Running tests from plugins"
+    [ $is_core ] && spec="$test_dir/cypress/integration/$dashboards_type/$test.js" || "$test_dir/cypress/integration/$dashboards_type/plugins/*.js"
+    [ $is_core ] && success_msg="BWC tests for core passed ($spec)" || success_msg="BWC tests for plugin passed ($spec)"
+    [ $is_core ] && error_msg="BWC tests for core failed ($spec)" || error_msg="BWC tests for plugin failed ($spec)"
+    env NO_COLOR=1 npx cypress run --headless --spec $spec
+    test_failures=$?
+    [ $test_failures == 0 ] && echo $success_msg || echo $error_msg
+    total_test_failures=$(( $total_test_failures + $test_failures ))
+}
+
 # this function copies the tested data for the required version to the opensearch data folder
 # $1 is the required version
 function upload_data {
@@ -226,19 +249,38 @@ function upload_data {
   echo "Data has been uploaded and ready to test"
 }
 
+function setup_opensearch {
+  cd "$opensearch_dir"
+  # Required for IM
+  [ -d "plugins/opensearch-index-management" ] && echo "path.repo: [/tmp]" >> config/opensearch.yml
+  # Required for Alerting
+  [ -d "plugins/opensearch-alerting" ] && echo "plugins.destination.host.deny_list: [\"10.0.0.0/8\", \"127.0.0.1\"]" >> config/opensearch.yml
+  # Required for SQL
+  [ -d "plugins/opensearch-sql" ] && echo "script.context.field.max_compilations_rate: 1000/1m" >> config/opensearch.yml
+  # Required for PA
+  [ -d "plugins/opensearch-performance-analyzer" ] && echo "webservice-bind-host = 0.0.0.0" >> plugins/opensearch-performance-analyzer/pa_config/performance-analyzer.properties  
+  [ $SECURITY_ENABLED == "false" ] && echo "plugins.security.disabled: true" >> config/opensearch.yml
+}
+
 # Starts OpenSearch, if verifying a distribution it will install the certs then start.
 function run_opensearch {
   echo "[ Attempting to start OpenSearch... ]"
   cd "$opensearch_dir"
-  [ $SECURITY_ENABLED == "false" ] && ./bin/opensearch || ./opensearch-tar-install.sh
+  spawn_process_and_save_PID "./opensearch-tar-install.sh > ./bin/opensearch.log 2>&1 &"
+}
+
+function setup_dashboards {
+  cd "$dashboards_dir"
+  [ $SECURITY_ENABLED == "false" ] && ./bin/opensearch-dashboards-plugin remove securityDashboards
+  [ $SECURITY_ENABLED == "false" ] && rm config/opensearch_dashboards.yml && touch config/opensearch_dashboards.yml
+  [ $SECURITY_ENABLED == "false" ] && echo "server.host: 0.0.0.0" >> config/opensearch_dashboards.yml
 }
 
 # Starts OpenSearch Dashboards
 function run_dashboards {
   echo "[ Attempting to start OpenSearch Dashboards... ]"
   cd "$dashboards_dir"
-  [ $SECURITY_ENABLED == "false" ] && rm config/opensearch_dashboards.yml && touch config/opensearch_dashboards.yml
-  ./bin/opensearch-dashboards 
+  spawn_process_and_save_PID "./bin/opensearch-dashboards > ./bin/opensearch-dashboards.log 2>&1 &"
 }
 
 # Checks the running status of OpenSearch
@@ -247,7 +289,7 @@ function run_dashboards {
 function check_opensearch_status {
   echo "Checking the status OpenSearch..."
   cd "$dir"
-  check_status $opensearch_path "$opensearch_msg" $opensearch_url "$opensearch_args" >> /dev/null 2>&1
+  check_status $opensearch_path "$opensearch_msg" $opensearch_url "$opensearch_args" >> /dev/null 2>&1  &
   echo "OpenSearch is up!" 
 } 
 
@@ -269,17 +311,19 @@ function run_bwc {
   IFS=',' read -r -a tests <<<"$test_suite"
   for test in "${tests[@]}"
   do
-    npx cypress run --spec "$test_dir/cypress/integration/$dashboards_type/$test.js" || echo "backwards compatibility tests have issue"
+    run_cypress "core"
   done
   # Check if $dashboards_type/plugins has tests in them to execute
   if [ "$(ls -A $test_dir/cypress/integration/$dashboards_type/plugins | wc -l)" -gt 1 ]; then
-    echo "Running tests from plugins"
-    npx cypress run --spec "$test_dir/cypress/integration/$dashboards_type/plugins/*.js" || echo "backwards compatibility plugins tests have issue"
+    run_cypress "plugins"
   fi
 }
 
 # Main function
 function execute_tests {
+  setup_opensearch >> /dev/null 2>&1  &
+  setup_dashboards >> /dev/null 2>&1  &
+
   # for each required testing version, do the following
   # first run opensearch and check the status
   # second run dashboards and check the status
@@ -290,9 +334,9 @@ function execute_tests {
     echo "[ Setting up the OpenSearch environment for $version ]"
     upload_data $version
     
-    run_opensearch >> /dev/null 2>&1  &  
+    run_opensearch
     check_opensearch_status
-    run_dashboards >> /dev/null 2>&1 & 
+    run_dashboards
     check_dashboards_status
     
     echo "[ Run the backwards compatibility tests for $version ]"
@@ -318,15 +362,17 @@ function execute_mismatch_tests {
     (
       rm -rf $opensearch_dir && mkdir "$opensearch_dir"
       # TODO: support multiple platforms and architectures
-      cd $opensearch_dir && wget -c https://artifacts.opensearch.org/releases/bundle/opensearch/$release/opensearch-$release-linux-x64.tar.gz -O - | tar -xz --strip-components=1
+      cd $opensearch_dir && curl https://artifacts.opensearch.org/releases/bundle/opensearch/$release/opensearch-$release-linux-x64.tar.gz | tar -xz --strip-components=1
     )
     execute_tests
   done
 }
  
 # setup the cypress test env
-setup_cypress
+[ ! -d "$test_dir/cypress" ] && setup_cypress
 execute_tests
 (( ${#releases_array[@]} )) && execute_mismatch_tests
 
-rm -rf "$dir"
\ No newline at end of file
+echo "Completed BWC tests for $test_array [$dashboards_type]"
+echo "Total test failures: $total_test_failures"
+exit $total_test_failures
\ No newline at end of file

From 93c2cdecacf4f56e07b19e28209121681f7ce2dc Mon Sep 17 00:00:00 2001
From: Kawika Avilla <kavilla414@gmail.com>
Date: Thu, 24 Mar 2022 05:19:49 +0000
Subject: [PATCH 2/6] Cleanup

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
---
 bwctest.sh                                   |   4 +-
 scripts/bwc/opensearch_dashboards_service.sh |  32 +++
 scripts/bwc/opensearch_service.sh            |  38 +++
 scripts/bwc/utils.sh                         |  69 +++++
 scripts/bwctest_osd.sh                       | 257 ++++++-------------
 5 files changed, 217 insertions(+), 183 deletions(-)
 create mode 100755 scripts/bwc/opensearch_dashboards_service.sh
 create mode 100755 scripts/bwc/opensearch_service.sh
 create mode 100755 scripts/bwc/utils.sh

diff --git a/bwctest.sh b/bwctest.sh
index cd19906f976e..2ca6478dbb4b 100755
--- a/bwctest.sh
+++ b/bwctest.sh
@@ -13,7 +13,7 @@
 
 set -e
 
-DEFAULT_VERSIONS="osd-1.1.0,odfe-1.13.2"
+DEFAULT_VERSIONS="osd-1.1.0,odfe-1.13.2,odfe-0.10.0"
 
 function usage() {
     echo ""
@@ -92,4 +92,6 @@ if [ -z "$OPENSEARCH" ]; then
     OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz"
 fi
 
+# CI=1
+
 source scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS
diff --git a/scripts/bwc/opensearch_dashboards_service.sh b/scripts/bwc/opensearch_dashboards_service.sh
new file mode 100755
index 000000000000..e3016433b492
--- /dev/null
+++ b/scripts/bwc/opensearch_dashboards_service.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+function setup_dashboards() {
+  cd "$DASHBOARDS_DIR"
+  [ $SECURITY_ENABLED == "false" ] && [ -d "plugins/securityDashboards" ] && ./bin/opensearch-dashboards-plugin remove securityDashboards
+  [ $SECURITY_ENABLED == "false" ] && rm config/opensearch_dashboards.yml && touch config/opensearch_dashboards.yml
+  [ $SECURITY_ENABLED == "false" ] && echo "server.host: 0.0.0.0" >> config/opensearch_dashboards.yml
+  echo "csp.warnLegacyBrowsers: false" >> config/opensearch_dashboards.yml
+  echo "--max-old-space-size=5120" >> config/node.options
+}
+
+# Starts OpenSearch Dashboards
+function run_dashboards() {
+  echo "[ Attempting to start OpenSearch Dashboards... ]"
+  cd "$DASHBOARDS_DIR"
+  spawn_process_and_save_PID "./bin/opensearch-dashboards > ${LOGS_DIR}/opensearch_dashboards.log 2>&1 &"
+}
+
+# Checks the running status of OpenSearch Dashboards
+# it calls check_status and passes the OpenSearch Dashboards tmp file path, error msg, url, and arguments
+# if success, the while loop in the check_status will end and it prints out "OpenSearch Dashboards is up!"
+function check_dashboards_status {
+  echo "Checking the OpenSearch Dashboards..."
+  cd "$DIR"
+  check_status $DASHBOARDS_PATH "$DASHBOARDS_MSG" $DASHBOARDS_URL "" >> /dev/null 2>&1
+  echo "OpenSearch Dashboards is up!"
+} 
diff --git a/scripts/bwc/opensearch_service.sh b/scripts/bwc/opensearch_service.sh
new file mode 100755
index 000000000000..d0051b5271bc
--- /dev/null
+++ b/scripts/bwc/opensearch_service.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+function setup_opensearch() {
+  cd "$OPENSEARCH_DIR"
+  # Required for IM
+  [ -d "plugins/opensearch-index-management" ] && echo "path.repo: [/tmp]" >> config/opensearch.yml
+  # Required for Alerting
+  [ -d "plugins/opensearch-alerting" ] && echo "plugins.destination.host.deny_list: [\"10.0.0.0/8\", \"127.0.0.1\"]" >> config/opensearch.yml
+  # Required for SQL
+  [ -d "plugins/opensearch-sql" ] && echo "script.context.field.max_compilations_rate: 1000/1m" >> config/opensearch.yml
+  # Required for PA
+  [ -d "plugins/opensearch-performance-analyzer" ] && echo "webservice-bind-host = 0.0.0.0" >> plugins/opensearch-performance-analyzer/pa_config/performance-analyzer.properties
+  echo "network.host: 0.0.0.0" >> config/opensearch.yml
+  echo "discovery.type: single-node" >> config/opensearch.yml
+  [ $SECURITY_ENABLED == "false" ] && [ -d "plugins/opensearch-security" ] && echo "plugins.security.disabled: true" >> config/opensearch.yml
+}
+
+# Starts OpenSearch, if verifying a distribution it will install the certs then start.
+function run_opensearch() {
+  echo "[ Attempting to start OpenSearch... ]"
+  cd "$OPENSEARCH_DIR"
+  spawn_process_and_save_PID "./opensearch-tar-install.sh > ${LOGS_DIR}/opensearch.log 2>&1 &"
+}
+
+# Checks the running status of OpenSearch
+# it calls check_status and passes the OpenSearch tmp file path, error msg, url, and arguments
+# if success, the while loop in the check_status will end and it prints out "OpenSearch is up!"
+function check_opensearch_status() {
+  echo "Checking the status OpenSearch..."
+  cd "$DIR"
+  check_status $OPENSEARCH_PATH "$OPENSEARCH_MSG" $OPENSEARCH_URL "$OPENSEARCH_ARGS" >> /dev/null 2>&1  &
+  echo "OpenSearch is up!" 
+} 
diff --git a/scripts/bwc/utils.sh b/scripts/bwc/utils.sh
new file mode 100755
index 000000000000..5400a91a9ef4
--- /dev/null
+++ b/scripts/bwc/utils.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+function open_artifact() {
+  artifact_dir=$1
+  artifact=$2
+  cd $artifact_dir
+  
+  # check if artifact provided is URL or attempt if passing by absolute path
+  if curl -I -L $artifact; then
+    curl -L $artifact | tar -xz --strip-components=1
+  else
+    tar -xf $artifact --strip-components=1
+  fi
+}
+
+# remove the running opensearch process
+function clean() {
+  echo "Attempt to Terminate Process with PID: ${PARENT_PID_LIST[*]}"
+  for pid_kill in "${PARENT_PID_LIST[@]}"
+  do
+    echo "Closing PID $pid_kill"
+    kill $pid_kill || true
+  done
+  PARENT_PID_LIST=()
+}
+
+function spawn_process_and_save_PID() {
+    echo "Spawn '$@'"
+    eval $@
+    curr_pid=$!
+    echo "PID: $curr_pid"
+    PARENT_PID_LIST+=( $curr_pid )
+}
+
+# Print out a textfile line by line
+function print_txt() {
+  while IFS= read -r line; do
+    echo "text read from $1: $line"
+  done < $1
+}
+
+# this function is used to check the running status of OpenSearch or OpenSearch Dashboards
+# $1 is the path to the tmp file which saves the running status 
+# $2 is the error msg to check
+# $3 is the url to curl
+# $4 contains arguments that need to be passed to the curl command
+function check_status() {
+  while [ ! -f $1 ] || ! grep -q "$2" $1; do 
+     if [ -f $1 ]; then rm $1; fi  
+     curl $3 $4 > $1 || true
+  done
+  rm $1
+}
+
+# this function copies the tested data for the required version to the opensearch data folder
+# $1 is the required version
+function upload_data() {
+  rm -rf "$OPENSEARCH_DIR/data"
+  cd $OPENSEARCH_DIR
+  cp "$CWD/cypress/test-data/$DASHBOARDS_TYPE/$1.tar.gz" . 
+  tar -xvf "$OPENSEARCH_DIR/$1.tar.gz" >> /dev/null 2>&1
+  rm "$1.tar.gz"
+  echo "Data has been uploaded and ready to test"
+}
diff --git a/scripts/bwctest_osd.sh b/scripts/bwctest_osd.sh
index 6ed14c3e19a7..8a6cddcd11de 100755
--- a/scripts/bwctest_osd.sh
+++ b/scripts/bwctest_osd.sh
@@ -5,6 +5,10 @@
 
 set -e
 
+. scripts/bwc/utils.sh
+. scripts/bwc/opensearch_service.sh
+. scripts/bwc/opensearch_dashboards_service.sh
+
 # For every release, add sample data and new version below:
 DEFAULT_VERSIONS=(
   "odfe-0.10.0"
@@ -97,67 +101,54 @@ done
 [ -z "$BIND_PORT" ] && BIND_PORT="5601"
 [ -z "$VERSIONS" ] && test_array=("${DEFAULT_VERSIONS[@]}") || IFS=',' read -r -a test_array <<<"$VERSIONS"
 [ -z "$SECURITY_ENABLED" ] && SECURITY_ENABLED="false"
-[ $SECURITY_ENABLED == "false" ] && dashboards_type="without-security" || dashboards_type="with-security"
+[ $SECURITY_ENABLED == "false" ] && DASHBOARDS_TYPE="without-security" || DASHBOARDS_TYPE="with-security"
 [ $SECURITY_ENABLED == "false" ] && releases_array=() || IFS=',' read -r -a releases_array <<<"$RELEASES"
 [ -z "$CREDENTIAL" ] && CREDENTIAL="admin:admin"
 
-total_test_failures=0
+TOTAL_TEST_FAILURES=0
 # OpenSearch and OpenSearch Dashboards Process IDs
-parent_pid_list=()
+PARENT_PID_LIST=()
 # define test path
-cwd=$(pwd)
-dir="$cwd/bwc_tmp"
-test_dir="$dir/test"
-opensearch_dir="$dir/opensearch"
-dashboards_dir="$dir/opensearch-dashboards"
-[ ! -d "$dir" ] && mkdir "$dir"
-[ ! -d "$test_dir" ] && mkdir "$test_dir"
-[ -d "$opensearch_dir" ] && rm -rf "$opensearch_dir"
-mkdir "$opensearch_dir"
-[ -d "$dashboards_dir" ] && rm -rf "$dashboards_dir"
-mkdir "$dashboards_dir"
-
-function open_artifact {
-  artifact_dir=$1
-  artifact=$2
-  cd $artifact_dir
-  
-  # check if artifact provided is URL or attempt if passing by absolute path
-  if curl -I -L $artifact; then
-    curl -L $artifact | tar -xz --strip-components=1
-  else
-    tar -xf $artifact --strip-components=1
-  fi
-
-}
+CWD=$(pwd)
+DIR="$CWD/bwc_tmp"
+TEST_DIR="$DIR/test"
+LOGS_DIR="$TEST_DIR/cypress/results/local-cluster-logs"
+OPENSEARCH_DIR="$DIR/opensearch"
+DASHBOARDS_DIR="$DIR/opensearch-dashboards"
+[ ! -d "$DIR" ] && mkdir "$DIR"
+[ ! -d "$TEST_DIR" ] && mkdir "$TEST_DIR"
+[ -d "$OPENSEARCH_DIR" ] && rm -rf "$OPENSEARCH_DIR"
+mkdir "$OPENSEARCH_DIR"
+[ -d "$DASHBOARDS_DIR" ] && rm -rf "$DASHBOARDS_DIR"
+mkdir "$DASHBOARDS_DIR"
 
 # un-tar OpenSearch and OpenSearch Dashboards
 echo "[ unzip OpenSearch and OpenSearch Dashboards ]"
 echo $OPENSEARCH
-open_artifact $opensearch_dir $OPENSEARCH
-open_artifact $dashboards_dir $DASHBOARDS
+open_artifact $OPENSEARCH_DIR $OPENSEARCH
+open_artifact $DASHBOARDS_DIR $DASHBOARDS
 
 # define other paths and tmp files
-opensearch_file='opensearch.txt'
-dashboards_file='dashboards.txt'
-opensearch_path="$dir/$opensearch_file"
-dashboards_path="$dir/$dashboards_file"
-dashboards_msg="\"state\":\"green\",\"title\":\"Green\",\"nickname\":\"Looking good\",\"icon\":\"success\""
-dashboards_url="http://$BIND_ADDRESS:$BIND_PORT/api/status"
+OPENSEARCH_FILE='opensearch.txt'
+DASHBOARDS_FILE='dashboards.txt'
+OPENSEARCH_PATH="$DIR/$OPENSEARCH_FILE"
+DASHBOARDS_PATH="$DIR/$DASHBOARDS_FILE"
+DASHBOARDS_MSG="\"state\":\"green\",\"title\":\"Green\",\"nickname\":\"Looking good\",\"icon\":\"success\""
+DASHBOARDS_URL="http://$BIND_ADDRESS:$BIND_PORT/api/status"
 if [ $SECURITY_ENABLED == "false" ]; 
 then 
-  opensearch_msg="\"status\":\"green\""
-  opensearch_url="http://$BIND_ADDRESS:9200/_cluster/health"
-  opensearch_args=""
+  OPENSEARCH_MSG="\"status\":\"green\""
+  OPENSEARCH_URL="http://$BIND_ADDRESS:9200/_cluster/health"
+  OPENSEARCH_ARGS=""
 else 
-  opensearch_msg="\"status\":\"yellow\""
-  opensearch_url="https://$BIND_ADDRESS:9200/_cluster/health"
-  opensearch_args="-u $CREDENTIAL --insecure"
+  OPENSEARCH_MSG="\"status\":\"yellow\""
+  OPENSEARCH_URL="https://$BIND_ADDRESS:9200/_cluster/health"
+  OPENSEARCH_ARGS="-u $CREDENTIAL --insecure"
 fi
 
 # define test groups to test suites
-declare -A test_suites
-test_suites=(
+declare -A TEST_SUITES
+TEST_SUITES=(
   ["odfe-0.10.0"]=$TEST_GROUP_1 
   ["odfe-1.0.2"]=$TEST_GROUP_2 
   ["odfe-1.1.0"]=$TEST_GROUP_2
@@ -173,154 +164,57 @@ test_suites=(
   ["osd-1.1.0"]=$TEST_GROUP_4
 )
 
-# remove the running opensearch process
-function clean {
-  echo "Attempt to Terminate Process with PID: ${parent_pid_list[*]}"
-  for pid_kill in "${parent_pid_list[@]}"
-  do
-    echo "Closing PID $pid_kill"
-    kill $pid_kill
-  done
-  parent_pid_list=()
-}
-
-function spawn_process_and_save_PID {
-    echo "Spawn '$@'"
-    eval $@
-    curr_pid=$!
-    echo "PID: $curr_pid"
-    parent_pid_list+=( $curr_pid )
-}
-
-# Print out a textfile line by line
-function print_txt {
-  while IFS= read -r line; do
-    echo "text read from $1: $line"
-  done < $1
-}
-
-# this function is used to check the running status of OpenSearch or OpenSearch Dashboards
-# $1 is the path to the tmp file which saves the running status 
-# $2 is the error msg to check
-# $3 is the url to curl
-# $4 contains arguments that need to be passed to the curl command
-function check_status {
-  while [ ! -f $1 ] || ! grep -q "$2" $1; do 
-     if [ -f $1 ]; then rm $1; fi  
-     curl $3 $4 > $1 || true
-  done
-  rm $1
-}
-
 # this function sets up the cypress env
 # it first clones the opensearch-dashboards-functional-test library
 # then it removes the tests into the cypress integration folder 
 # and copies the backwards compatibility tests into the folder
-function setup_cypress {
+function setup_cypress() {
   echo "[ Setup the cypress test environment ]"
-  git clone https://github.com/opensearch-project/opensearch-dashboards-functional-test "$test_dir"
-  rm -rf "$test_dir/cypress/integration"
-  cp -r "$cwd/cypress/integration" "$test_dir/cypress"
-  cd "$test_dir"
+  git clone --depth=1 https://github.com/opensearch-project/opensearch-dashboards-functional-test "$TEST_DIR"
+  rm -rf "$TEST_DIR/cypress/integration"
+  cp -r "$CWD/cypress/integration" "$TEST_DIR/cypress"
+  [ ! -d "$LOGS_DIR" ] && mkdir -p "$LOGS_DIR"
+  cd "$TEST_DIR"
   npm install
   echo "Cypress is ready!"
 }
 
-function run_cypress {
-    [ $1 == "core" ] && is_core=true || is_core=false
-    [ ! $is_core ] && echo "Running tests from plugins"
-    [ $is_core ] && spec="$test_dir/cypress/integration/$dashboards_type/$test.js" || "$test_dir/cypress/integration/$dashboards_type/plugins/*.js"
-    [ $is_core ] && success_msg="BWC tests for core passed ($spec)" || success_msg="BWC tests for plugin passed ($spec)"
-    [ $is_core ] && error_msg="BWC tests for core failed ($spec)" || error_msg="BWC tests for plugin failed ($spec)"
-    env NO_COLOR=1 npx cypress run --headless --spec $spec
-    test_failures=$?
-    [ $test_failures == 0 ] && echo $success_msg || echo $error_msg
-    total_test_failures=$(( $total_test_failures + $test_failures ))
+function run_cypress() {
+    [ $1 == "core" ] && IS_CORE=true || IS_CORE=false
+    TEST_ARRAY=("$@")
+    SPEC_FILES=""
+    for test in "${TEST_ARRAY[@]}"
+    do
+      SPEC_FILES+="$TEST_DIR/cypress/integration/$DASHBOARDS_TYPE/*$test.js,"
+    done
+    [ ! $IS_CORE ] && echo "Running tests from plugins"
+    [ $IS_CORE ] && spec="$SPEC_FILES" || "$TEST_DIR/cypress/integration/$DASHBOARDS_TYPE/plugins/*.js"
+    [ $IS_CORE ] && success_msg="BWC tests for core passed ($spec)" || success_msg="BWC tests for plugin passed ($spec)"
+    [ $IS_CORE ] && error_msg="BWC tests for core failed ($spec)" || error_msg="BWC tests for plugin failed ($spec)"
+    [[ ! -z $CI ]] && cypress_args="--browser chromium" || cypress_args=""
+    env NO_COLOR=1 npx cypress run $cypress_args --headless --spec $spec || test_failures=$?
+    [ -z $test_failures ] && test_failures=0
+    [ $test_failures == 0 ] && echo $success_msg || echo "$error_msg::TEST_FAILURES: $test_failures"
+    TOTAL_TEST_FAILURES=$(( $TOTAL_TEST_FAILURES + $test_failures ))
 }
 
-# this function copies the tested data for the required version to the opensearch data folder
-# $1 is the required version
-function upload_data {
-  rm -rf "$opensearch_dir/data"
-  cd $opensearch_dir
-  cp "$cwd/cypress/test-data/$dashboards_type/$1.tar.gz" . 
-  tar -xvf "$opensearch_dir/$1.tar.gz" >> /dev/null 2>&1
-  rm "$1.tar.gz"
-  echo "Data has been uploaded and ready to test"
-}
-
-function setup_opensearch {
-  cd "$opensearch_dir"
-  # Required for IM
-  [ -d "plugins/opensearch-index-management" ] && echo "path.repo: [/tmp]" >> config/opensearch.yml
-  # Required for Alerting
-  [ -d "plugins/opensearch-alerting" ] && echo "plugins.destination.host.deny_list: [\"10.0.0.0/8\", \"127.0.0.1\"]" >> config/opensearch.yml
-  # Required for SQL
-  [ -d "plugins/opensearch-sql" ] && echo "script.context.field.max_compilations_rate: 1000/1m" >> config/opensearch.yml
-  # Required for PA
-  [ -d "plugins/opensearch-performance-analyzer" ] && echo "webservice-bind-host = 0.0.0.0" >> plugins/opensearch-performance-analyzer/pa_config/performance-analyzer.properties  
-  [ $SECURITY_ENABLED == "false" ] && echo "plugins.security.disabled: true" >> config/opensearch.yml
-}
-
-# Starts OpenSearch, if verifying a distribution it will install the certs then start.
-function run_opensearch {
-  echo "[ Attempting to start OpenSearch... ]"
-  cd "$opensearch_dir"
-  spawn_process_and_save_PID "./opensearch-tar-install.sh > ./bin/opensearch.log 2>&1 &"
-}
-
-function setup_dashboards {
-  cd "$dashboards_dir"
-  [ $SECURITY_ENABLED == "false" ] && ./bin/opensearch-dashboards-plugin remove securityDashboards
-  [ $SECURITY_ENABLED == "false" ] && rm config/opensearch_dashboards.yml && touch config/opensearch_dashboards.yml
-  [ $SECURITY_ENABLED == "false" ] && echo "server.host: 0.0.0.0" >> config/opensearch_dashboards.yml
-}
-
-# Starts OpenSearch Dashboards
-function run_dashboards {
-  echo "[ Attempting to start OpenSearch Dashboards... ]"
-  cd "$dashboards_dir"
-  spawn_process_and_save_PID "./bin/opensearch-dashboards > ./bin/opensearch-dashboards.log 2>&1 &"
-}
-
-# Checks the running status of OpenSearch
-# it calls check_status and passes the OpenSearch tmp file path, error msg, url, and arguments
-# if success, the while loop in the check_status will end and it prints out "OpenSearch is up!"
-function check_opensearch_status {
-  echo "Checking the status OpenSearch..."
-  cd "$dir"
-  check_status $opensearch_path "$opensearch_msg" $opensearch_url "$opensearch_args" >> /dev/null 2>&1  &
-  echo "OpenSearch is up!" 
-} 
-
-# Checks the running status of OpenSearch Dashboards
-# it calls check_status and passes the OpenSearch Dashboards tmp file path, error msg, url, and arguments
-# if success, the while loop in the check_status will end and it prints out "OpenSearch Dashboards is up!"
-function check_dashboards_status {
-  echo "Checking the OpenSearch Dashboards..."
-  cd "$dir"
-  check_status $dashboards_path "$dashboards_msg" $dashboards_url "" >> /dev/null 2>&1
-  echo "OpenSearch Dashboards is up!"
-} 
-
 # Runs the backwards compatibility test using cypress for the required version
 # $1 is the requested version 
-function run_bwc {
-  cd "$test_dir"
-  [ -z "${test_suites[$1]}" ] && test_suite=$TEST_GROUP_DEFAULT || test_suite="${test_suites[$1]}"
+function run_bwc() {
+  cd "$TEST_DIR"
+  [ -z "${TEST_SUITES[$1]}" ] && test_suite=$TEST_GROUP_DEFAULT || test_suite="${TEST_SUITES[$1]}"
   IFS=',' read -r -a tests <<<"$test_suite"
-  for test in "${tests[@]}"
-  do
-    run_cypress "core"
-  done
-  # Check if $dashboards_type/plugins has tests in them to execute
-  if [ "$(ls -A $test_dir/cypress/integration/$dashboards_type/plugins | wc -l)" -gt 1 ]; then
+
+  # Check if $DASHBOARDS_TYPE/plugins has tests in them to execute
+  if [ "$(ls -A $TEST_DIR/cypress/integration/$DASHBOARDS_TYPE/plugins | wc -l)" -gt 1 ]; then
     run_cypress "plugins"
+  else 
+    run_cypress "core" "${tests[@]}"
   fi
 }
 
 # Main function
-function execute_tests {
+function execute_tests() {
   setup_opensearch >> /dev/null 2>&1  &
   setup_dashboards >> /dev/null 2>&1  &
 
@@ -348,8 +242,8 @@ function execute_tests {
 }
 
 # Executes the main function with different versions of OpenSearch downloaded
-function execute_mismatch_tests {
-  PACKAGE_VERSION=$(cat $dashboards_dir/package.json \
+function execute_mismatch_tests() {
+  PACKAGE_VERSION=$(cat $DASHBOARDS_DIR/package.json \
   | grep version \
   | head -1 \
   | awk -F: '{ print $2 }' \
@@ -360,19 +254,18 @@ function execute_mismatch_tests {
   do
     echo "Running tests with OpenSearch Dashboards $PACKAGE_VERSION and OpenSearch $release"
     (
-      rm -rf $opensearch_dir && mkdir "$opensearch_dir"
+      rm -rf $OPENSEARCH_DIR && mkdir "$OPENSEARCH_DIR"
       # TODO: support multiple platforms and architectures
-      cd $opensearch_dir && curl https://artifacts.opensearch.org/releases/bundle/opensearch/$release/opensearch-$release-linux-x64.tar.gz | tar -xz --strip-components=1
+      cd $OPENSEARCH_DIR && curl https://artifacts.opensearch.org/releases/bundle/opensearch/$release/opensearch-$release-linux-x64.tar.gz | tar -xz --strip-components=1
     )
     execute_tests
   done
 }
  
 # setup the cypress test env
-[ ! -d "$test_dir/cypress" ] && setup_cypress
+[ ! -d "$TEST_DIR/cypress" ] && setup_cypress
 execute_tests
 (( ${#releases_array[@]} )) && execute_mismatch_tests
-
-echo "Completed BWC tests for $test_array [$dashboards_type]"
-echo "Total test failures: $total_test_failures"
-exit $total_test_failures
\ No newline at end of file
+echo "Completed BWC tests for $test_array [$DASHBOARDS_TYPE]"
+echo "Total test failures: $TOTAL_TEST_FAILURES"
+exit $TOTAL_TEST_FAILURES
\ No newline at end of file

From 33bc9c322831a341801c875f5ef18e64140d9bc1 Mon Sep 17 00:00:00 2001
From: Kawika Avilla <kavilla414@gmail.com>
Date: Tue, 12 Apr 2022 09:07:27 +0000
Subject: [PATCH 3/6] Fix path of distribution

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
---
 bwctest.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bwctest.sh b/bwctest.sh
index 2ca6478dbb4b..77da1360e31e 100755
--- a/bwctest.sh
+++ b/bwctest.sh
@@ -79,7 +79,7 @@ done
 if [ -z "$OPENSEARCH" ]; then
     IFS='/' read -ra SLASH_ARR <<< "$DASHBOARDS"
     # Expected to be opensearch-x.y.z-platform-arch.tar.gz
-    [[ "$DASHBOARDS" == *"Playground"* ]] && TARBALL="${SLASH_ARR[13]}" || TARBALL="${SLASH_ARR[12]}"
+    [[ "$DASHBOARDS" == *"Playground"* ]] && TARBALL="${SLASH_ARR[14]}" || TARBALL="${SLASH_ARR[13]}"
     IFS='-' read -ra DASH_ARR <<< "$TARBALL"
     # Expected to be arch.tar.gz
     DOTS="${DASH_ARR[4]}"
@@ -89,7 +89,7 @@ if [ -z "$OPENSEARCH" ]; then
     PLATFORM="${DASH_ARR[3]}"
     ARCH="${DOTS_ARR[0]}"
 
-    OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz"
+    OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/tar/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz"
 fi
 
 # CI=1

From 00e5375a17fdcd14312bf32b50a8de8bd82cc8b6 Mon Sep 17 00:00:00 2001
From: Kawika Avilla <kavilla414@gmail.com>
Date: Thu, 14 Apr 2022 06:34:11 +0000
Subject: [PATCH 4/6] set CI and clean up more vars

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
---
 bwctest.sh             |  5 ++---
 scripts/bwctest_osd.sh | 14 +++++++-------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/bwctest.sh b/bwctest.sh
index 77da1360e31e..ef6d9bdfd794 100755
--- a/bwctest.sh
+++ b/bwctest.sh
@@ -13,7 +13,7 @@
 
 set -e
 
-DEFAULT_VERSIONS="osd-1.1.0,odfe-1.13.2,odfe-0.10.0"
+DEFAULT_VERSIONS="osd-1.1.0"
 
 function usage() {
     echo ""
@@ -74,6 +74,7 @@ done
 [ -z "$BIND_PORT" ] && BIND_PORT="5601"
 [ -z "$SECURITY_ENABLED" ] && SECURITY_ENABLED="false"
 [ -z "$CREDENTIAL" ] && CREDENTIAL="admin:admin"
+[ -z "$CI" ] && CI=1
 
 # If no OpenSearch build was passed then this constructs the version
 if [ -z "$OPENSEARCH" ]; then
@@ -92,6 +93,4 @@ if [ -z "$OPENSEARCH" ]; then
     OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/tar/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz"
 fi
 
-# CI=1
-
 source scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS
diff --git a/scripts/bwctest_osd.sh b/scripts/bwctest_osd.sh
index 8a6cddcd11de..a6775e54a858 100755
--- a/scripts/bwctest_osd.sh
+++ b/scripts/bwctest_osd.sh
@@ -99,10 +99,10 @@ done
 
 [ -z "$BIND_ADDRESS" ] && BIND_ADDRESS="localhost"
 [ -z "$BIND_PORT" ] && BIND_PORT="5601"
-[ -z "$VERSIONS" ] && test_array=("${DEFAULT_VERSIONS[@]}") || IFS=',' read -r -a test_array <<<"$VERSIONS"
+[ -z "$VERSIONS" ] && TEST_VERSIONS=("${DEFAULT_VERSIONS[@]}") || IFS=',' read -r -a TEST_VERSIONS <<<"$VERSIONS"
 [ -z "$SECURITY_ENABLED" ] && SECURITY_ENABLED="false"
 [ $SECURITY_ENABLED == "false" ] && DASHBOARDS_TYPE="without-security" || DASHBOARDS_TYPE="with-security"
-[ $SECURITY_ENABLED == "false" ] && releases_array=() || IFS=',' read -r -a releases_array <<<"$RELEASES"
+[ $SECURITY_ENABLED == "false" ] && RELEASES_ARRAY=() || IFS=',' read -r -a RELEASES_ARRAY <<<"$RELEASES"
 [ -z "$CREDENTIAL" ] && CREDENTIAL="admin:admin"
 
 TOTAL_TEST_FAILURES=0
@@ -191,7 +191,7 @@ function run_cypress() {
     [ $IS_CORE ] && spec="$SPEC_FILES" || "$TEST_DIR/cypress/integration/$DASHBOARDS_TYPE/plugins/*.js"
     [ $IS_CORE ] && success_msg="BWC tests for core passed ($spec)" || success_msg="BWC tests for plugin passed ($spec)"
     [ $IS_CORE ] && error_msg="BWC tests for core failed ($spec)" || error_msg="BWC tests for plugin failed ($spec)"
-    [[ ! -z $CI ]] && cypress_args="--browser chromium" || cypress_args=""
+    [ $CI == 1 ] && cypress_args="--browser chromium" || cypress_args=""
     env NO_COLOR=1 npx cypress run $cypress_args --headless --spec $spec || test_failures=$?
     [ -z $test_failures ] && test_failures=0
     [ $test_failures == 0 ] && echo $success_msg || echo "$error_msg::TEST_FAILURES: $test_failures"
@@ -222,7 +222,7 @@ function execute_tests() {
   # first run opensearch and check the status
   # second run dashboards and check the status
   # run the backwards compatibility tests
-  for version in "${test_array[@]}"
+  for version in "${TEST_VERSIONS[@]}"
   do
     # copy and un-tar data into the OpenSearch data folder
     echo "[ Setting up the OpenSearch environment for $version ]"
@@ -250,7 +250,7 @@ function execute_mismatch_tests() {
   | sed 's/[",]//g' \
   | tr -d [:space:])
 
-  for release in "${releases_array[@]}"
+  for release in "${RELEASES_ARRAY[@]}"
   do
     echo "Running tests with OpenSearch Dashboards $PACKAGE_VERSION and OpenSearch $release"
     (
@@ -265,7 +265,7 @@ function execute_mismatch_tests() {
 # setup the cypress test env
 [ ! -d "$TEST_DIR/cypress" ] && setup_cypress
 execute_tests
-(( ${#releases_array[@]} )) && execute_mismatch_tests
-echo "Completed BWC tests for $test_array [$DASHBOARDS_TYPE]"
+(( ${#RELEASES_ARRAY[@]} )) && execute_mismatch_tests
+echo "Completed BWC tests for $TEST_VERSIONS [$DASHBOARDS_TYPE]"
 echo "Total test failures: $TOTAL_TEST_FAILURES"
 exit $TOTAL_TEST_FAILURES
\ No newline at end of file

From 121dc654c78456a65dc2312fff522d23d67aef00 Mon Sep 17 00:00:00 2001
From: Kawika Avilla <kavilla414@gmail.com>
Date: Fri, 15 Apr 2022 00:32:51 +0000
Subject: [PATCH 5/6] Add playground comment

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
---
 bwctest.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bwctest.sh b/bwctest.sh
index ef6d9bdfd794..867ba4e631b3 100755
--- a/bwctest.sh
+++ b/bwctest.sh
@@ -80,6 +80,7 @@ done
 if [ -z "$OPENSEARCH" ]; then
     IFS='/' read -ra SLASH_ARR <<< "$DASHBOARDS"
     # Expected to be opensearch-x.y.z-platform-arch.tar.gz
+    # Playground is supported path to enable sandbox testing
     [[ "$DASHBOARDS" == *"Playground"* ]] && TARBALL="${SLASH_ARR[14]}" || TARBALL="${SLASH_ARR[13]}"
     IFS='-' read -ra DASH_ARR <<< "$TARBALL"
     # Expected to be arch.tar.gz

From b5c4356bac7678fd37ba1505c4bb6fe2c59567e2 Mon Sep 17 00:00:00 2001
From: Kawika Avilla <kavilla414@gmail.com>
Date: Fri, 15 Apr 2022 00:35:55 +0000
Subject: [PATCH 6/6] Readd tests

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
---
 bwctest.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bwctest.sh b/bwctest.sh
index 867ba4e631b3..b4d30ef64425 100755
--- a/bwctest.sh
+++ b/bwctest.sh
@@ -13,7 +13,7 @@
 
 set -e
 
-DEFAULT_VERSIONS="osd-1.1.0"
+DEFAULT_VERSIONS="osd-1.1.0,odfe-1.13.2,odfe-0.10.0"
 
 function usage() {
     echo ""