[Tests] BWC test improvements

bwctest.sh

@@ -13,7 +13,6 @@
 
 set -e
 
-# TODO: Update to include all known BWC of data
 DEFAULT_VERSIONS="osd-1.1.0"
 
 function usage() {
@@ -75,12 +74,13 @@ done
 [ -z "$BIND_PORT" ] && BIND_PORT="5601"
 [ -z "$SECURITY_ENABLED" ] && SECURITY_ENABLED="false"
 [ -z "$CREDENTIAL" ] && CREDENTIAL="admin:admin"
+[ -z "$CI" ] && CI=1
 
 # If no OpenSearch build was passed then this constructs the version
 if [ -z "$OPENSEARCH" ]; then
     IFS='/' read -ra SLASH_ARR <<< "$DASHBOARDS"
     # Expected to be opensearch-x.y.z-platform-arch.tar.gz
-    TARBALL="${SLASH_ARR[12]}"
+    [[ "$DASHBOARDS" == *"Playground"* ]] && TARBALL="${SLASH_ARR[14]}" || TARBALL="${SLASH_ARR[13]}"
     IFS='-' read -ra DASH_ARR <<< "$TARBALL"
     # Expected to be arch.tar.gz
     DOTS="${DASH_ARR[4]}"
@@ -90,7 +90,7 @@ if [ -z "$OPENSEARCH" ]; then
     PLATFORM="${DASH_ARR[3]}"
     ARCH="${DOTS_ARR[0]}"
 
-    OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz"
+    OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/tar/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz"
 fi
 
-./scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS
+source scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS

scripts/bwc/opensearch_dashboards_service.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+function setup_dashboards() {
+  cd "$DASHBOARDS_DIR"
+  [ $SECURITY_ENABLED == "false" ] && [ -d "plugins/securityDashboards" ] && ./bin/opensearch-dashboards-plugin remove securityDashboards
+  [ $SECURITY_ENABLED == "false" ] && rm config/opensearch_dashboards.yml && touch config/opensearch_dashboards.yml
+  [ $SECURITY_ENABLED == "false" ] && echo "server.host: 0.0.0.0" >> config/opensearch_dashboards.yml
+  echo "csp.warnLegacyBrowsers: false" >> config/opensearch_dashboards.yml
+  echo "--max-old-space-size=5120" >> config/node.options
+}
+
+# Starts OpenSearch Dashboards
+function run_dashboards() {
+  echo "[ Attempting to start OpenSearch Dashboards... ]"
+  cd "$DASHBOARDS_DIR"
+  spawn_process_and_save_PID "./bin/opensearch-dashboards > ${LOGS_DIR}/opensearch_dashboards.log 2>&1 &"
+}
+
+# Checks the running status of OpenSearch Dashboards
+# it calls check_status and passes the OpenSearch Dashboards tmp file path, error msg, url, and arguments
+# if success, the while loop in the check_status will end and it prints out "OpenSearch Dashboards is up!"
+function check_dashboards_status {
+  echo "Checking the OpenSearch Dashboards..."
+  cd "$DIR"
+  check_status $DASHBOARDS_PATH "$DASHBOARDS_MSG" $DASHBOARDS_URL "" >> /dev/null 2>&1
+  echo "OpenSearch Dashboards is up!"
+} 

scripts/bwc/opensearch_service.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+function setup_opensearch() {
+  cd "$OPENSEARCH_DIR"
+  # Required for IM
+  [ -d "plugins/opensearch-index-management" ] && echo "path.repo: [/tmp]" >> config/opensearch.yml
+  # Required for Alerting
+  [ -d "plugins/opensearch-alerting" ] && echo "plugins.destination.host.deny_list: [\"10.0.0.0/8\", \"127.0.0.1\"]" >> config/opensearch.yml
+  # Required for SQL
+  [ -d "plugins/opensearch-sql" ] && echo "script.context.field.max_compilations_rate: 1000/1m" >> config/opensearch.yml
+  # Required for PA
+  [ -d "plugins/opensearch-performance-analyzer" ] && echo "webservice-bind-host = 0.0.0.0" >> plugins/opensearch-performance-analyzer/pa_config/performance-analyzer.properties
+  echo "network.host: 0.0.0.0" >> config/opensearch.yml
+  echo "discovery.type: single-node" >> config/opensearch.yml
+  [ $SECURITY_ENABLED == "false" ] && [ -d "plugins/opensearch-security" ] && echo "plugins.security.disabled: true" >> config/opensearch.yml
+}
+
+# Starts OpenSearch, if verifying a distribution it will install the certs then start.
+function run_opensearch() {
+  echo "[ Attempting to start OpenSearch... ]"
+  cd "$OPENSEARCH_DIR"
+  spawn_process_and_save_PID "./opensearch-tar-install.sh > ${LOGS_DIR}/opensearch.log 2>&1 &"
+}
+
+# Checks the running status of OpenSearch
+# it calls check_status and passes the OpenSearch tmp file path, error msg, url, and arguments
+# if success, the while loop in the check_status will end and it prints out "OpenSearch is up!"
+function check_opensearch_status() {
+  echo "Checking the status OpenSearch..."
+  cd "$DIR"
+  check_status $OPENSEARCH_PATH "$OPENSEARCH_MSG" $OPENSEARCH_URL "$OPENSEARCH_ARGS" >> /dev/null 2>&1  &
+  echo "OpenSearch is up!" 
+} 

scripts/bwc/utils.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+function open_artifact() {
+  artifact_dir=$1
+  artifact=$2
+  cd $artifact_dir
+
+  # check if artifact provided is URL or attempt if passing by absolute path
+  if curl -I -L $artifact; then
+    curl -L $artifact | tar -xz --strip-components=1
+  else
+    tar -xf $artifact --strip-components=1
+  fi
+}
+
+# remove the running opensearch process
+function clean() {
+  echo "Attempt to Terminate Process with PID: ${PARENT_PID_LIST[*]}"
+  for pid_kill in "${PARENT_PID_LIST[@]}"
+  do
+    echo "Closing PID $pid_kill"
+    kill $pid_kill || true
+  done
+  PARENT_PID_LIST=()
+}
+
+function spawn_process_and_save_PID() {
+    echo "Spawn '$@'"
+    eval $@
+    curr_pid=$!
+    echo "PID: $curr_pid"
+    PARENT_PID_LIST+=( $curr_pid )
+}
+
+# Print out a textfile line by line
+function print_txt() {
+  while IFS= read -r line; do
+    echo "text read from $1: $line"
+  done < $1
+}
+
+# this function is used to check the running status of OpenSearch or OpenSearch Dashboards
+# $1 is the path to the tmp file which saves the running status 
+# $2 is the error msg to check
+# $3 is the url to curl
+# $4 contains arguments that need to be passed to the curl command
+function check_status() {
+  while [ ! -f $1 ] || ! grep -q "$2" $1; do 
+     if [ -f $1 ]; then rm $1; fi  
+     curl $3 $4 > $1 || true
+  done
+  rm $1
+}
+
+# this function copies the tested data for the required version to the opensearch data folder
+# $1 is the required version
+function upload_data() {
+  rm -rf "$OPENSEARCH_DIR/data"
+  cd $OPENSEARCH_DIR
+  cp "$CWD/cypress/test-data/$DASHBOARDS_TYPE/$1.tar.gz" . 
+  tar -xvf "$OPENSEARCH_DIR/$1.tar.gz" >> /dev/null 2>&1
+  rm "$1.tar.gz"
+  echo "Data has been uploaded and ready to test"
+}