From 1f373e1588a4d29ef4b7b2c7e6f4dcf8df3e9d0f Mon Sep 17 00:00:00 2001
From: Nathan Cutler <ncutler@suse.com>
Date: Tue, 24 Jul 2018 16:24:08 +0200
Subject: [PATCH 1/9] qa: refactor health-ok.sh, drop health-{mds,rgw}.sh

Now that we have the ceph.restart orchestration for testing the restart
functionality, we can simplify the integration testing scripts.

The actual triggering of the ceph.restart orchestration will be done
in the teuthology test yaml, which lives in the Ceph repo.

Signed-off-by: Nathan Cutler <ncutler@suse.com>
---
 qa/common/common.sh                   | 45 -------------
 qa/suites/basic/health-mds.sh         | 77 ----------------------
 qa/suites/basic/health-nfs-ganesha.sh | 12 +---
 qa/suites/basic/health-ok.sh          | 53 +++++++--------
 qa/suites/basic/health-rgw.sh         | 94 ---------------------------
 5 files changed, 27 insertions(+), 254 deletions(-)
 delete mode 100755 qa/suites/basic/health-mds.sh
 delete mode 100755 qa/suites/basic/health-rgw.sh

diff --git a/qa/common/common.sh b/qa/common/common.sh
index e3d791fd7..d71775e88 100644
--- a/qa/common/common.sh
+++ b/qa/common/common.sh
@@ -205,51 +205,6 @@ mgr initial modules = dashboard
 EOF
 }
 
-#
-# functions for testing ceph.restart orchestration
-#
-
-function restart_services {
-  rm -rf /srv/modules/runners/__pycache__
-  salt-run state.orch ceph.restart
-}
-
-function mon_restarted {
-  local expected_return=$1
-  local mon_hosts=$(salt --static --out json -C "I@roles:mon" test.ping | jq -r 'keys[]')
-  set +e
-  for minion in ${mon_hosts}; do
-    salt "${minion}*" cmd.shell "journalctl -u ceph-mon@*" | grep -i terminated
-    test $? = ${expected_return}
-  done
-  set -e
-}
-
-function osd_restarted {
-    local expected_return=$1
-    osd_hosts=$(salt --static --out json -C "I@roles:storage" test.ping | jq -r 'keys[]')
-    set +e
-    for host in ${osd_hosts}; do
-        osds=$(salt --static --out json ${host} osd.list | jq .[][])
-        for osd in ${osds}; do
-            salt ${host} cmd.shell "journalctl -u ceph-osd@${osd}" | grep -i terminated
-            test $? = ${expected_return}
-        done
-    done
-    set -e
-}
-
-function rgw_restarted {
-  local expected_return=$1
-  rgw_hosts=$(salt --static --out json -C "I@roles:rgw" test.ping | jq -r 'keys[]')
-  set +e
-  for host in ${rgw_hosts}; do
-    salt ${host} cmd.shell "journalctl -u ceph-radosgw@*" | grep -i terminated
-    test $? = ${expected_return}
-  done
-  set -e
-}
-
 
 #
 # functions for creating pools
diff --git a/qa/suites/basic/health-mds.sh b/qa/suites/basic/health-mds.sh
deleted file mode 100755
index 02d873b85..000000000
--- a/qa/suites/basic/health-mds.sh
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/bin/bash
-#
-# DeepSea integration test "suites/basic/health-mds.sh"
-#
-# This script runs DeepSea stages 0-4 to deploy a Ceph cluster with MDS.
-# After stage 4 completes, it mounts the cephfs on the client node,
-# touches a file, and asserts that it exists.
-#
-# The script makes no assumptions beyond those listed in qa/README.
-#
-# On success, the script returns 0. On failure, for whatever reason, the script
-# returns non-zero.
-#
-# The script produces verbose output on stdout, which can be captured for later
-# forensic analysis.
-#
-
-set -ex
-
-SCRIPTNAME=$(basename ${0})
-BASEDIR=$(readlink -f "$(dirname ${0})/../..")
-test -d $BASEDIR
-[[ $BASEDIR =~ \/qa$ ]]
-
-source $BASEDIR/common/common.sh
-
-function usage {
-    set +x
-    echo "$SCRIPTNAME - script for testing CephFS deployment"
-    echo "for use in SUSE Enterprise Storage testing"
-    echo
-    echo "Usage:"
-    echo "  $SCRIPTNAME [-h,--help] [--cli] [--mini]"
-    echo
-    echo "Options:"
-    echo "    --cli      Use DeepSea CLI"
-    echo "    --help     Display this usage message"
-    echo "    --mini     Omit long-running tests"
-    exit 1
-}
-
-TEMP=$(getopt -o h --long "cli,help,mini,smoke" \
-     -n 'health-mds.sh' -- "$@")
-
-if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
-
-# Note the quotes around TEMP': they are essential!
-eval set -- "$TEMP"
-
-# process options
-CLI=""
-MINI=""
-while true ; do
-    case "$1" in
-        --cli) CLI="$1" ; shift ;;
-        -h|--help) usage ;;    # does not return
-        --mini|--smoke) MINI="$1" ; shift ;;
-        --) shift ; break ;;
-        *) echo "Internal error" ; exit 1 ;;
-    esac
-done
-
-# deploy phase
-MIN_NODES=2
-CLIENT_NODES=1
-STORAGE_PROFILE="default"
-MDS="--mds"
-deploy_ceph
-
-# test phase
-ceph_health_test
-cephfs_mount_and_sanity_test
-if [ -z "$MINI" ] ; then
-    run_stage_0 "$CLI"
-fi
-
-echo "OK"
diff --git a/qa/suites/basic/health-nfs-ganesha.sh b/qa/suites/basic/health-nfs-ganesha.sh
index 8bfd0c2ef..9b6091bb3 100755
--- a/qa/suites/basic/health-nfs-ganesha.sh
+++ b/qa/suites/basic/health-nfs-ganesha.sh
@@ -36,17 +36,15 @@ function usage {
     echo
     echo "Usage:"
     echo "  $SCRIPTNAME [-h,--help] [--cli] [--fsal={cephfs,rgw,both}]"
-    echo "              [--mini]"
     echo
     echo "Options:"
     echo "    --cli      Use DeepSea CLI"
     echo "    --fsal     Defaults to cephfs"
     echo "    --help     Display this usage message"
-    echo "    --mini     Omit long-running tests"
     exit 1
 }
 
-TEMP=$(getopt -o h --long "cli,fsal:,help,mini,smoke" \
+TEMP=$(getopt -o h --long "cli,fsal:,help" \
      -n 'health-nfs-ganesha.sh' -- "$@")
 
 if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
@@ -57,13 +55,11 @@ eval set -- "$TEMP"
 # process options
 CLI=""
 FSAL="cephfs"
-MINI=""
 while true ; do
     case "$1" in
         --cli) CLI="cli" ; shift ;;
         --fsal) FSAL=$2 ; shift ; shift ;;
         -h|--help) usage ;;    # does not return
-        --mini|--smoke) MINI="$1" ; shift ;;
         --) shift ; break ;;
         *) echo "Internal error" ; exit 1 ;;
     esac
@@ -111,9 +107,7 @@ for v in "" "3" "4" ; do
     nfs_ganesha_umount
     sleep 10
 done
-
-if [ -z "$MINI" ] ; then
-    run_stage_0 "$CLI"
-fi
+# exercise ceph.restart orchestration
+run_stage_0 "$CLI"
 
 echo "OK"
diff --git a/qa/suites/basic/health-ok.sh b/qa/suites/basic/health-ok.sh
index f6a4384e4..e0064f6e4 100755
--- a/qa/suites/basic/health-ok.sh
+++ b/qa/suites/basic/health-ok.sh
@@ -2,16 +2,15 @@
 #
 # DeepSea integration test "suites/basic/health-ok.sh"
 #
-# This script runs DeepSea stages 0-3 to deploy a Ceph cluster, optionally
-# with data-at-rest encryption of OSDs, on all the nodes that have at least
-# one external disk drive. After stage 3 completes, the script checks for
-# HEALTH_OK and tests the "ceph.restart" orchestration if --mini is not provided.
+# This script runs DeepSea stages 0-3 (or 0-4, depending on options) to deploy
+# a Ceph cluster, optionally with RGW, MDS, and/or data-at-rest encryption of
+# OSDs, on all the nodes that have at least one external disk drive. After the
+# last stage completes, the script checks for HEALTH_OK.
 #
 # The script makes no assumptions beyond those listed in qa/README.
 #
-# On success (HEALTH_OK is reached and optionally "ceph.restart" orchestration behaves
-# as expected), the script returns 0. On failure, for whatever reason, the
-# script returns non-zero.
+# On success (HEALTH_OK is reached), the script returns 0. On failure, for
+# whatever reason, the script returns non-zero.
 #
 # The script produces verbose output on stdout, which can be captured for later
 # forensic analysis.
@@ -32,20 +31,22 @@ function usage {
     echo "for use in SUSE Enterprise Storage testing"
     echo
     echo "Usage:"
-    echo "  $SCRIPTNAME [-h,--help] [--cli] [--mini]"
+    echo "  $SCRIPTNAME [-h,--help] [--cli] [--mds] [--rgw] [--ssl]"
     echo
     echo "Options:"
     echo "    --cli         Use DeepSea CLI"
     echo "    --encryption  Deploy OSDs with data-at-rest encryption"
-    echo "    --mini        Omit long-running tests"
+    echo "    --mds         Deploy MDS"
     echo "    --help        Display this usage message"
+    echo "    --rgw         Deploy RGW"
+    echo "    --ssl         Deploy RGW with SSL"
     exit 1
 }
 
 assert_enhanced_getopt
 
 TEMP=$(getopt -o h \
---long "cli,encrypted,encryption,help,mini,smoke" \
+--long "cli,encrypted,encryption,help,mds,rgw,ssl" \
 -n 'health-ok.sh' -- "$@")
 
 if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
@@ -56,19 +57,23 @@ eval set -- "$TEMP"
 # process command-line options
 CLI=""
 STORAGE_PROFILE="default"
-MINI=""
+MDS=""
+RGW=""
+SSL=""
 while true ; do
     case "$1" in
         --cli) CLI="$1" ; shift ;;
         --encrypted|--encryption) STORAGE_PROFILE="dmcrypt" ; shift ;;
-        --mini|--smoke) MINI="$1" ; shift ;;
+        --mds) MDS="$1" ; shift ;;
         -h|--help) usage ;;    # does not return
+        --rgw) RGW="$1" ; shift ;;
+        --ssl) SSL="$1" ; shift ;;
         --) shift ; break ;;
         *) echo "Internal error" ; exit 1 ;;
     esac
 done
 echo "WWWW"
-echo "Running health-ok.sh with options $CLI $ENCRYPTION $MINI"
+echo "Running health-ok.sh with options $CLI $ENCRYPTION $MDS $RGW $SSL"
 
 # deploy phase
 MIN_NODES=1
@@ -81,22 +86,12 @@ ceph_log_grep_enoent_eaccess
 test_systemd_ceph_osd_target_wants
 rados_write_test
 ceph_version_test
-if [ -z "$MINI" ] ; then
-    run_stage_0 "$CLI"
-    restart_services
-    mon_restarted "1" # 1 means not restarted
-    osd_restarted "1"
-    # apply config change
-    change_osd_conf
-    change_mon_conf
-    # construct and spread config
-    run_stage_3 "$CLI"
-    restart_services
-    mon_restarted "0" # 0 means restarted
-    osd_restarted "0"
-    # make sure still in HEALTH_OK
-    ceph_cluster_status
-    ceph_health_test
+if [ -n "$RGW" ] ; then
+    rgw_curl_test
+    test -n "$SSL" && validate_rgw_cert_perm
+    rgw_user_and_bucket_list
+    rgw_validate_system_user
 fi
+test -n "$MDS" && cephfs_mount_and_sanity_test
 
 echo "health-ok test result: PASS"
diff --git a/qa/suites/basic/health-rgw.sh b/qa/suites/basic/health-rgw.sh
deleted file mode 100755
index 1f3fc7272..000000000
--- a/qa/suites/basic/health-rgw.sh
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/bin/bash
-#
-# DeepSea integration test "suites/basic/health-rgw.sh"
-#
-# This script runs DeepSea stages 0-4 to deploy a Ceph cluster with RGW (and
-# optionally RGW+SSL). After stage 4 completes, it sends a GET request to the
-# RGW node using curl (optionally using SSL endpoint), and tests that: (a) the
-# response contains the string "anonymous" and (b) the response is legal XML.
-# The script also deploys some RGW demo users.
-#
-# The script makes no assumptions beyond those listed in qa/README.
-#
-# On success, the script returns 0. On failure, for whatever reason, the script
-# returns non-zero.
-#
-# The script produces verbose output on stdout, which can be captured for later
-# forensic analysis.
-#
-
-set -ex
-
-SCRIPTNAME=$(basename ${0})
-BASEDIR=$(readlink -f "$(dirname ${0})/../..")
-test -d $BASEDIR
-[[ $BASEDIR =~ \/qa$ ]]
-
-source $BASEDIR/common/common.sh
-
-function usage {
-    set +x
-    echo "$SCRIPTNAME - script for testing RADOS Gateway deployment"
-    echo "for use in SUSE Enterprise Storage testing"
-    echo
-    echo "Usage:"
-    echo "  $SCRIPTNAME [-h,--help] [--cli] [--mini] [--ssl]"
-    echo
-    echo "Options:"
-    echo "    --cli      Use DeepSea CLI"
-    echo "    --help     Display this usage message"
-    echo "    --mini     Omit long-running tests"
-    echo "    --ssl      Use SSL (https, port 443) with RGW"
-    exit 1
-}
-
-TEMP=$(getopt -o h --long "cli,help,mini,smoke,ssl" \
-     -n 'health-rgw.sh' -- "$@")
-
-if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
-
-# Note the quotes around TEMP': they are essential!
-eval set -- "$TEMP"
-
-# process command-line options
-CLI=""
-MINI=""
-SSL=""
-while true ; do
-    case "$1" in
-        --cli) CLI="$1" ; shift ;;
-        -h|--help) usage ;;    # does not return
-        --mini|--smoke) MINI="$1" ; shift ;;
-        --ssl) SSL="$1" ; shift ;;
-        --) shift ; break ;;
-        *) echo "Internal error" ; exit 1 ;;
-    esac
-done
-
-# deploy phase
-MIN_NODES=1
-CLIENT_NODES=0
-STORAGE_PROFILE="default"
-RGW="--rgw"
-deploy_ceph
-
-# test phase
-ceph_health_test
-rgw_curl_test
-test -n "$SSL" && validate_rgw_cert_perm
-rgw_user_and_bucket_list
-rgw_validate_system_user
-ceph_health_test
-if [ -z "$MINI" ] ; then
-    run_stage_0 "$CLI"
-    restart_services
-    rgw_restarted "1" # 1 means not restarted
-    # apply config change
-    change_rgw_conf
-    # construct and spread config
-    run_stage_3 "$CLI"
-    restart_services
-    rgw_restarted "0" # 0 means restarted
-fi
-
-echo "health-rgw test result: PASS"

From 3e7eaab178b9d8dc13e4b0369a6d8715c2c33537 Mon Sep 17 00:00:00 2001
From: Nathan Cutler <ncutler@suse.com>
Date: Tue, 24 Jul 2018 16:32:39 +0200
Subject: [PATCH 2/9] qa/common/helper.sh: drop workaround for bsc#1087232

Signed-off-by: Nathan Cutler <ncutler@suse.com>
---
 qa/common/helper.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/qa/common/helper.sh b/qa/common/helper.sh
index a76f0ab4a..4e3fd432f 100644
--- a/qa/common/helper.sh
+++ b/qa/common/helper.sh
@@ -25,10 +25,6 @@ function _run_stage {
   local deepsea_cli_output_path="/tmp/deepsea.${stage_num}.log"
   local deepsea_exit_status=""
 
-  # workaround for https://bugzilla.suse.com/show_bug.cgi?id=1087232
-  # delete Salt runner __pycache__
-  rm -rf /srv/modules/runners/__pycache__
-
   set +x
   echo ""
   echo "*********************************************"

From 10fb280ed50257db34c48c501a29ba9234c27ca1 Mon Sep 17 00:00:00 2001
From: Nathan Cutler <ncutler@suse.com>
Date: Tue, 24 Jul 2018 17:32:17 +0200
Subject: [PATCH 3/9] qa: make MDS deploy even without client nodes

Signed-off-by: Nathan Cutler <ncutler@suse.com>
---
 qa/common/policy.sh          | 20 ++++++++++++++++----
 qa/suites/basic/health-ok.sh |  2 +-
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/qa/common/policy.sh b/qa/common/policy.sh
index 1e7239fd0..4b7852d51 100644
--- a/qa/common/policy.sh
+++ b/qa/common/policy.sh
@@ -83,14 +83,26 @@ EOF
     else
         echo "Unexpected number of client nodes ->$CLIENT_NODES<-; bailing out!"
         exit 1
-  fi
+    fi
 }
 
 function policy_cfg_mds {
-    cat <<EOF >> /srv/pillar/ceph/proposals/policy.cfg
-# Role assignment - mds (all but last node)
-role-mds/cluster/*.sls slice=[:-1]
+    test -n "$CLIENT_NODES"
+
+    if [ "$CLIENT_NODES" -eq 0 ] ; then
+        cat <<EOF >> /srv/pillar/ceph/proposals/policy.cfg
+# Role assignment - mds (all nodes)
+role-mds/cluster/*.sls
 EOF
+    elif [ "$CLIENT_NODES" -ge 1 ] ; then
+        cat <<EOF >> /srv/pillar/ceph/proposals/policy.cfg
+# Role assignment - mds (all non-client nodes)
+role-mds/cluster/*.sls slice=[:-$CLIENT_NODES]
+EOF
+    else
+        echo "Unexpected number of client nodes ->$CLIENT_NODES<-; bailing out!"
+        exit 1
+    fi
 }
 
 function policy_cfg_rgw {
diff --git a/qa/suites/basic/health-ok.sh b/qa/suites/basic/health-ok.sh
index e0064f6e4..10ea04f21 100755
--- a/qa/suites/basic/health-ok.sh
+++ b/qa/suites/basic/health-ok.sh
@@ -92,6 +92,6 @@ if [ -n "$RGW" ] ; then
     rgw_user_and_bucket_list
     rgw_validate_system_user
 fi
-test -n "$MDS" && cephfs_mount_and_sanity_test
+test -n "$MDS" -a "$CLIENT_NODES" -ge 1 && cephfs_mount_and_sanity_test
 
 echo "health-ok test result: PASS"

From 4fc7e8547b1e88f85b62e23a74727f0886e8763c Mon Sep 17 00:00:00 2001
From: Nathan Cutler <ncutler@suse.com>
Date: Wed, 25 Jul 2018 16:39:10 +0200
Subject: [PATCH 4/9] qa: fix ceph_version_test

This function could have produced a false positive if both the RPM and Ceph version strings
failed the regex match. (In that case, both the variables would have been empty and the
equality test might have passed.)

With this commit, the test explicity checks that the regex match actually produced a string
that can be usefully compared.

Signed-off-by: Nathan Cutler <ncutler@suse.com>
---
 qa/common/common.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/qa/common/common.sh b/qa/common/common.sh
index d71775e88..ffc12be77 100644
--- a/qa/common/common.sh
+++ b/qa/common/common.sh
@@ -297,11 +297,13 @@ function ceph_version_test {
   rpm -q ceph
   local RPM_NAME=$(rpm -q ceph)
   local RPM_CEPH_VERSION=$(perl -e '"'"$RPM_NAME"'" =~ m/ceph-(\d+\.\d+\.\d+)/; print "$1\n";')
-  echo "According to RPM, the ceph upstream version is $RPM_CEPH_VERSION"
+  echo "According to RPM, the ceph upstream version is ->$RPM_CEPH_VERSION<-"
+  test -n "$RPM_CEPH_VERSION"
   ceph --version
   local BUFFER=$(ceph --version)
   local CEPH_CEPH_VERSION=$(perl -e '"'"$BUFFER"'" =~ m/ceph version (\d+\.\d+\.\d+)/; print "$1\n";')
-  echo "According to \"ceph --version\", the ceph upstream version is $CEPH_CEPH_VERSION"
+  echo "According to \"ceph --version\", the ceph upstream version is ->$CEPH_CEPH_VERSION<-"
+  test -n "$RPM_CEPH_VERSION"
   test "$RPM_CEPH_VERSION" = "$CEPH_CEPH_VERSION"
 }
 

From e6a8a3aef58a4bad8b5507f3e02a022df90e2701 Mon Sep 17 00:00:00 2001
From: Nathan Cutler <ncutler@suse.com>
Date: Thu, 26 Jul 2018 12:56:02 +0200
Subject: [PATCH 5/9] qa: health-ok.sh: add a top-secret undocumented
 navigation code

This secret code will make it easier to navigate teuthology logs,
for tests that use health-ok.sh to deploy the cluster.

Just search for YYYY to get to the end of the deployment phase.

Signed-off-by: Nathan Cutler <ncutler@suse.com>
---
 qa/suites/basic/health-ok.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/suites/basic/health-ok.sh b/qa/suites/basic/health-ok.sh
index 10ea04f21..99037d8c2 100755
--- a/qa/suites/basic/health-ok.sh
+++ b/qa/suites/basic/health-ok.sh
@@ -94,4 +94,5 @@ if [ -n "$RGW" ] ; then
 fi
 test -n "$MDS" -a "$CLIENT_NODES" -ge 1 && cephfs_mount_and_sanity_test
 
+echo "YYYY"
 echo "health-ok test result: PASS"

From d6bab016f6b3e37fce5ed1c88ce7913d543f6e51 Mon Sep 17 00:00:00 2001
From: Nathan Cutler <ncutler@suse.com>
Date: Thu, 26 Jul 2018 13:07:51 +0200
Subject: [PATCH 6/9] qa/common/helper.sh: refactor _run_stage

Signed-off-by: Nathan Cutler <ncutler@suse.com>
---
 qa/common/helper.sh | 61 ++++++++++++++++++++++++---------------------
 1 file changed, 33 insertions(+), 28 deletions(-)

diff --git a/qa/common/helper.sh b/qa/common/helper.sh
index 4e3fd432f..a8a073c44 100644
--- a/qa/common/helper.sh
+++ b/qa/common/helper.sh
@@ -19,11 +19,6 @@ function _report_stage_failure_and_die {
 
 function _run_stage {
   local stage_num=$1
-  local cli=$2
-  test -z "$cli" && cli="classic"
-  local stage_log_path="/tmp/stage.${stage_num}.log"
-  local deepsea_cli_output_path="/tmp/deepsea.${stage_num}.log"
-  local deepsea_exit_status=""
 
   set +x
   echo ""
@@ -33,34 +28,44 @@ function _run_stage {
   set -x
 
   # CLI case
-  if [ "x$cli" = "xcli" ] ; then
-      echo "using DeepSea CLI"
-      set +e
-      deepsea \
-          --log-file=/var/log/salt/deepsea.log \
-          --log-level=debug \
-          stage \
-          run \
-          ceph.stage.${stage_num} \
-          --simple-output \
-          2>&1 | tee $deepsea_cli_output_path
-      deepsea_exit_status="${PIPESTATUS[0]}"
-      echo "deepsea exit status: $deepsea_exit_status"
-      if [ "$deepsea_exit_status" = "0" ] ; then
-          if grep -q -F "failed=0" $deepsea_cli_output_path ; then
-              echo "DeepSea stage OK"
-          else
-              echo "ERROR: deepsea stage returned exit status 0, yet one or more steps failed. Bailing out!"
-              _report_stage_failure_and_die $stage_num
-          fi
+  test -n "$CLI" && _run_stage_cli $stage_num || _run_stage_non_cli $stage_num
+}
+
+function _run_stage_cli {
+  local stage_num=$1
+  local deepsea_cli_output_path="/tmp/deepsea.${stage_num}.log"
+  local deepsea_exit_status=""
+
+  echo "using DeepSea CLI"
+  set +e
+  deepsea \
+      --log-file=/var/log/salt/deepsea.log \
+      --log-level=debug \
+      stage \
+      run \
+      ceph.stage.${stage_num} \
+      --simple-output \
+      2>&1 | tee $deepsea_cli_output_path
+  deepsea_exit_status="${PIPESTATUS[0]}"
+  echo "deepsea exit status: $deepsea_exit_status"
+  echo "WWWW"
+  if [ "$deepsea_exit_status" = "0" ] ; then
+      if grep -q -F "failed=0" $deepsea_cli_output_path ; then
+          echo "********** Stage $stage_num completed successfully **********"
       else
+          echo "ERROR: deepsea stage returned exit status 0, yet one or more steps failed. Bailing out!"
           _report_stage_failure_and_die $stage_num
       fi
-      set -e
-      return
+  else
+      _report_stage_failure_and_die $stage_num
   fi
+  set -e
+}
+
+function _run_stage_non_cli {
+  local stage_num=$1
+  local stage_log_path="/tmp/stage.${stage_num}.log"
 
-  # non-CLI ("classic") case
   echo -n "" > $stage_log_path
   salt-run --no-color state.orch ceph.stage.${stage_num} 2>&1 | tee $stage_log_path
   STAGE_FINISHED=$(grep -F 'Total states run' $stage_log_path)

From 74ab84417dc34cd2079d98a1fc7ff835c69473a1 Mon Sep 17 00:00:00 2001
From: Nathan Cutler <ncutler@suse.com>
Date: Thu, 26 Jul 2018 13:53:51 +0200
Subject: [PATCH 7/9] qa/health-ok: add --min-nodes and --client-nodes options

Signed-off-by: Nathan Cutler <ncutler@suse.com>
---
 qa/suites/basic/health-ok.sh | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/qa/suites/basic/health-ok.sh b/qa/suites/basic/health-ok.sh
index 99037d8c2..ea026483d 100755
--- a/qa/suites/basic/health-ok.sh
+++ b/qa/suites/basic/health-ok.sh
@@ -31,22 +31,25 @@ function usage {
     echo "for use in SUSE Enterprise Storage testing"
     echo
     echo "Usage:"
-    echo "  $SCRIPTNAME [-h,--help] [--cli] [--mds] [--rgw] [--ssl]"
+    echo "  $SCRIPTNAME [-h,--help] [--cli] [--client-nodes=X]"
+    echo "  [--mds] [--min-nodes=X] [--rgw] [--ssl]"
     echo
     echo "Options:"
-    echo "    --cli         Use DeepSea CLI"
-    echo "    --encryption  Deploy OSDs with data-at-rest encryption"
-    echo "    --mds         Deploy MDS"
-    echo "    --help        Display this usage message"
-    echo "    --rgw         Deploy RGW"
-    echo "    --ssl         Deploy RGW with SSL"
+    echo "    --cli           Use DeepSea CLI"
+    echo "    --client-nodes  Number of client (non-cluster) nodes"
+    echo "    --encryption    Deploy OSDs with data-at-rest encryption"
+    echo "    --mds           Deploy MDS"
+    echo "    --min-nodes     Minimum number of nodes"
+    echo "    --help          Display this usage message"
+    echo "    --rgw           Deploy RGW"
+    echo "    --ssl           Deploy RGW with SSL"
     exit 1
 }
 
 assert_enhanced_getopt
 
 TEMP=$(getopt -o h \
---long "cli,encrypted,encryption,help,mds,rgw,ssl" \
+--long "cli,client-nodes:,encrypted,encryption,help,mds,min-nodes:,rgw,ssl" \
 -n 'health-ok.sh' -- "$@")
 
 if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
@@ -56,15 +59,19 @@ eval set -- "$TEMP"
 
 # process command-line options
 CLI=""
+CLIENT_NODES=0
 STORAGE_PROFILE="default"
 MDS=""
+MIN_NODES=1
 RGW=""
 SSL=""
 while true ; do
     case "$1" in
         --cli) CLI="$1" ; shift ;;
+        --client-nodes) shift ; CLIENT_NODES=$1 ; shift ;;
         --encrypted|--encryption) STORAGE_PROFILE="dmcrypt" ; shift ;;
         --mds) MDS="$1" ; shift ;;
+        --min-nodes) shift ; MIN_NODES=$1 ; shift ;;
         -h|--help) usage ;;    # does not return
         --rgw) RGW="$1" ; shift ;;
         --ssl) SSL="$1" ; shift ;;
@@ -76,8 +83,6 @@ echo "WWWW"
 echo "Running health-ok.sh with options $CLI $ENCRYPTION $MDS $RGW $SSL"
 
 # deploy phase
-MIN_NODES=1
-CLIENT_NODES=0
 deploy_ceph
 
 # test phase

From 09f8d1f066162cc19d481fdc4d2f4155b3fe5d42 Mon Sep 17 00:00:00 2001
From: Nathan Cutler <ncutler@suse.com>
Date: Thu, 26 Jul 2018 14:24:58 +0200
Subject: [PATCH 8/9] qa: rename ceph.smoketests to ceph.functests.1node

Signed-off-by: Nathan Cutler <ncutler@suse.com>
---
 Makefile                                      | 38 +++++++++---------
 deepsea.spec.in                               | 39 ++++++++++---------
 .../1node}/apparmor/init.sls                  |  0
 .../{smoketests => functests/1node}/init.sls  |  0
 .../1node}/keyrings/init.sls                  |  0
 .../1node}/macros/os_switch/init.sls          |  0
 .../1node}/openstack/default.sls              |  0
 .../1node}/openstack/init.sls                 |  0
 .../1node}/openstack/prefix.sls               |  0
 .../1node}/quiescent/init.sls                 |  0
 .../1node}/restart/common.sls                 |  0
 .../1node}/restart/mds.sls                    |  0
 .../1node}/restart/mgr.sls                    |  0
 .../1node}/restart/mon.sls                    |  0
 .../1node}/restart/rgw.sls                    |  0
 .../1node}/tuned/init.sls                     |  0
 .../1node}/tuned/off/init.sls                 |  0
 17 files changed, 39 insertions(+), 38 deletions(-)
 rename srv/salt/ceph/{smoketests => functests/1node}/apparmor/init.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/init.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/keyrings/init.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/macros/os_switch/init.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/openstack/default.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/openstack/init.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/openstack/prefix.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/quiescent/init.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/restart/common.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/restart/mds.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/restart/mgr.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/restart/mon.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/restart/rgw.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/tuned/init.sls (100%)
 rename srv/salt/ceph/{smoketests => functests/1node}/tuned/off/init.sls (100%)

diff --git a/Makefile b/Makefile
index bd04c1f6e..bc26fd37d 100644
--- a/Makefile
+++ b/Makefile
@@ -126,25 +126,25 @@ copy-files:
 	install -m 644 srv/salt/ceph/tests/restart/rgw/nochange/*.sls $(DESTDIR)/srv/salt/ceph/tests/restart/rgw/nochange
 	install -m 644 srv/salt/ceph/tests/tuned/*.sls $(DESTDIR)/srv/salt/ceph/tests/tuned
 	install -m 644 srv/salt/ceph/tests/tuned/off/*.sls $(DESTDIR)/srv/salt/ceph/tests/tuned/off
-	# smoketests
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests
-	install -m 644 srv/salt/ceph/smoketests/*.sls $(DESTDIR)/srv/salt/ceph/smoketests
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/apparmor
-	install -m 644 srv/salt/ceph/smoketests/apparmor/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/apparmor
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/keyrings
-	install -m 644 srv/salt/ceph/smoketests/keyrings/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/keyrings
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/macros
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/macros/os_switch
-	install -m 644 srv/salt/ceph/smoketests/macros/os_switch/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/macros/os_switch
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/openstack
-	install -m 644 srv/salt/ceph/smoketests/openstack/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/openstack
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/quiescent
-	install -m 644 srv/salt/ceph/smoketests/quiescent/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/quiescent
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/restart
-	install -m 644 srv/salt/ceph/smoketests/restart/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/restart
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/tuned/off
-	install -m 644 srv/salt/ceph/smoketests/tuned/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/tuned
-	install -m 644 srv/salt/ceph/smoketests/tuned/off/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/tuned/off
+	# functests/1node
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node
+	install -m 644 srv/salt/ceph/functests/1node/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/apparmor
+	install -m 644 srv/salt/ceph/functests/1node/apparmor/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/apparmor
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/keyrings
+	install -m 644 srv/salt/ceph/functests/1node/keyrings/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/keyrings
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/macros
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/macros/os_switch
+	install -m 644 srv/salt/ceph/functests/1node/macros/os_switch/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/macros/os_switch
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/openstack
+	install -m 644 srv/salt/ceph/functests/1node/openstack/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/openstack
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/quiescent
+	install -m 644 srv/salt/ceph/functests/1node/quiescent/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/quiescent
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/restart
+	install -m 644 srv/salt/ceph/functests/1node/restart/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/restart
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/tuned/off
+	install -m 644 srv/salt/ceph/functests/1node/tuned/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/tuned
+	install -m 644 srv/salt/ceph/functests/1node/tuned/off/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/tuned/off
 	# docs
 	install -d -m 755 $(DESTDIR)$(DOCDIR)/deepsea
 	install -m 644 LICENSE $(DESTDIR)$(DOCDIR)/deepsea/
diff --git a/deepsea.spec.in b/deepsea.spec.in
index de977ed8a..768499102 100644
--- a/deepsea.spec.in
+++ b/deepsea.spec.in
@@ -745,16 +745,17 @@ the README for more information.
 
 %files qa
 %{_libexecdir}/deepsea/qa
-%dir /srv/salt/ceph/smoketests
-%dir /srv/salt/ceph/smoketests/apparmor
-%dir /srv/salt/ceph/smoketests/keyrings
-%dir /srv/salt/ceph/smoketests/macros
-%dir /srv/salt/ceph/smoketests/macros/os_switch
-%dir /srv/salt/ceph/smoketests/openstack
-%dir /srv/salt/ceph/smoketests/quiescent
-%dir /srv/salt/ceph/smoketests/restart
-%dir /srv/salt/ceph/smoketests/tuned
-%dir /srv/salt/ceph/smoketests/tuned/off
+%dir /srv/salt/ceph/functests
+%dir /srv/salt/ceph/functests/1node
+%dir /srv/salt/ceph/functests/1node/apparmor
+%dir /srv/salt/ceph/functests/1node/keyrings
+%dir /srv/salt/ceph/functests/1node/macros
+%dir /srv/salt/ceph/functests/1node/macros/os_switch
+%dir /srv/salt/ceph/functests/1node/openstack
+%dir /srv/salt/ceph/functests/1node/quiescent
+%dir /srv/salt/ceph/functests/1node/restart
+%dir /srv/salt/ceph/functests/1node/tuned
+%dir /srv/salt/ceph/functests/1node/tuned/off
 %dir /srv/salt/ceph/tests
 %dir /srv/salt/ceph/tests/keyrings
 %dir /srv/salt/ceph/tests/openstack
@@ -780,17 +781,17 @@ the README for more information.
 %dir /srv/salt/ceph/tests/restart/rgw/change
 %dir /srv/salt/ceph/tests/restart/rgw/forced
 %dir /srv/salt/ceph/tests/restart/rgw/nochange
-/srv/salt/ceph/smoketests/*.sls
-/srv/salt/ceph/smoketests/apparmor/*.sls
+/srv/salt/ceph/functests/1node/*.sls
+/srv/salt/ceph/functests/1node/apparmor/*.sls
 %dir /srv/salt/ceph/tests/tuned
 %dir /srv/salt/ceph/tests/tuned/off
-/srv/salt/ceph/smoketests/keyrings/*.sls
-/srv/salt/ceph/smoketests/macros/os_switch/*.sls
-/srv/salt/ceph/smoketests/openstack/*.sls
-/srv/salt/ceph/smoketests/quiescent/*.sls
-/srv/salt/ceph/smoketests/restart/*.sls
-/srv/salt/ceph/smoketests/tuned/*.sls
-/srv/salt/ceph/smoketests/tuned/off/*.sls
+/srv/salt/ceph/functests/1node/keyrings/*.sls
+/srv/salt/ceph/functests/1node/macros/os_switch/*.sls
+/srv/salt/ceph/functests/1node/openstack/*.sls
+/srv/salt/ceph/functests/1node/quiescent/*.sls
+/srv/salt/ceph/functests/1node/restart/*.sls
+/srv/salt/ceph/functests/1node/tuned/*.sls
+/srv/salt/ceph/functests/1node/tuned/off/*.sls
 /srv/salt/ceph/tests/keyrings/*.sls
 /srv/salt/ceph/tests/openstack/*.sls
 /srv/salt/ceph/tests/os_switch/*.sls
diff --git a/srv/salt/ceph/smoketests/apparmor/init.sls b/srv/salt/ceph/functests/1node/apparmor/init.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/apparmor/init.sls
rename to srv/salt/ceph/functests/1node/apparmor/init.sls
diff --git a/srv/salt/ceph/smoketests/init.sls b/srv/salt/ceph/functests/1node/init.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/init.sls
rename to srv/salt/ceph/functests/1node/init.sls
diff --git a/srv/salt/ceph/smoketests/keyrings/init.sls b/srv/salt/ceph/functests/1node/keyrings/init.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/keyrings/init.sls
rename to srv/salt/ceph/functests/1node/keyrings/init.sls
diff --git a/srv/salt/ceph/smoketests/macros/os_switch/init.sls b/srv/salt/ceph/functests/1node/macros/os_switch/init.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/macros/os_switch/init.sls
rename to srv/salt/ceph/functests/1node/macros/os_switch/init.sls
diff --git a/srv/salt/ceph/smoketests/openstack/default.sls b/srv/salt/ceph/functests/1node/openstack/default.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/openstack/default.sls
rename to srv/salt/ceph/functests/1node/openstack/default.sls
diff --git a/srv/salt/ceph/smoketests/openstack/init.sls b/srv/salt/ceph/functests/1node/openstack/init.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/openstack/init.sls
rename to srv/salt/ceph/functests/1node/openstack/init.sls
diff --git a/srv/salt/ceph/smoketests/openstack/prefix.sls b/srv/salt/ceph/functests/1node/openstack/prefix.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/openstack/prefix.sls
rename to srv/salt/ceph/functests/1node/openstack/prefix.sls
diff --git a/srv/salt/ceph/smoketests/quiescent/init.sls b/srv/salt/ceph/functests/1node/quiescent/init.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/quiescent/init.sls
rename to srv/salt/ceph/functests/1node/quiescent/init.sls
diff --git a/srv/salt/ceph/smoketests/restart/common.sls b/srv/salt/ceph/functests/1node/restart/common.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/restart/common.sls
rename to srv/salt/ceph/functests/1node/restart/common.sls
diff --git a/srv/salt/ceph/smoketests/restart/mds.sls b/srv/salt/ceph/functests/1node/restart/mds.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/restart/mds.sls
rename to srv/salt/ceph/functests/1node/restart/mds.sls
diff --git a/srv/salt/ceph/smoketests/restart/mgr.sls b/srv/salt/ceph/functests/1node/restart/mgr.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/restart/mgr.sls
rename to srv/salt/ceph/functests/1node/restart/mgr.sls
diff --git a/srv/salt/ceph/smoketests/restart/mon.sls b/srv/salt/ceph/functests/1node/restart/mon.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/restart/mon.sls
rename to srv/salt/ceph/functests/1node/restart/mon.sls
diff --git a/srv/salt/ceph/smoketests/restart/rgw.sls b/srv/salt/ceph/functests/1node/restart/rgw.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/restart/rgw.sls
rename to srv/salt/ceph/functests/1node/restart/rgw.sls
diff --git a/srv/salt/ceph/smoketests/tuned/init.sls b/srv/salt/ceph/functests/1node/tuned/init.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/tuned/init.sls
rename to srv/salt/ceph/functests/1node/tuned/init.sls
diff --git a/srv/salt/ceph/smoketests/tuned/off/init.sls b/srv/salt/ceph/functests/1node/tuned/off/init.sls
similarity index 100%
rename from srv/salt/ceph/smoketests/tuned/off/init.sls
rename to srv/salt/ceph/functests/1node/tuned/off/init.sls

From d84877a934a2486affef5004cfef166293cbeb9e Mon Sep 17 00:00:00 2001
From: Nathan Cutler <ncutler@suse.com>
Date: Thu, 26 Jul 2018 15:46:29 +0200
Subject: [PATCH 9/9] qa/common/deploy.sh: fix redirect of stderr to /dev/null

Signed-off-by: Nathan Cutler <ncutler@suse.com>
---
 qa/common/deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/common/deploy.sh b/qa/common/deploy.sh
index a3ec630c9..4b8b124c8 100644
--- a/qa/common/deploy.sh
+++ b/qa/common/deploy.sh
@@ -51,7 +51,7 @@ function vet_nodes {
 }
 
 function ceph_cluster_running {
-    ceph status 2>&1 >/dev/null
+    ceph status >/dev/null 2>&1
 }
 
 function deploy_ceph {