Merge pull request #1233 from smithfarm/wip-qa-new-health-ok

qa: refactor health-ok.sh, drop health-{mds,rgw}.sh

Makefile

@@ -126,25 +126,25 @@ copy-files:
 	install -m 644 srv/salt/ceph/tests/restart/rgw/nochange/*.sls $(DESTDIR)/srv/salt/ceph/tests/restart/rgw/nochange
 	install -m 644 srv/salt/ceph/tests/tuned/*.sls $(DESTDIR)/srv/salt/ceph/tests/tuned
 	install -m 644 srv/salt/ceph/tests/tuned/off/*.sls $(DESTDIR)/srv/salt/ceph/tests/tuned/off
-	# smoketests
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests
-	install -m 644 srv/salt/ceph/smoketests/*.sls $(DESTDIR)/srv/salt/ceph/smoketests
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/apparmor
-	install -m 644 srv/salt/ceph/smoketests/apparmor/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/apparmor
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/keyrings
-	install -m 644 srv/salt/ceph/smoketests/keyrings/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/keyrings
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/macros
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/macros/os_switch
-	install -m 644 srv/salt/ceph/smoketests/macros/os_switch/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/macros/os_switch
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/openstack
-	install -m 644 srv/salt/ceph/smoketests/openstack/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/openstack
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/quiescent
-	install -m 644 srv/salt/ceph/smoketests/quiescent/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/quiescent
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/restart
-	install -m 644 srv/salt/ceph/smoketests/restart/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/restart
-	install -d -m 755 $(DESTDIR)/srv/salt/ceph/smoketests/tuned/off
-	install -m 644 srv/salt/ceph/smoketests/tuned/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/tuned
-	install -m 644 srv/salt/ceph/smoketests/tuned/off/*.sls $(DESTDIR)/srv/salt/ceph/smoketests/tuned/off
+	# functests/1node
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node
+	install -m 644 srv/salt/ceph/functests/1node/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/apparmor
+	install -m 644 srv/salt/ceph/functests/1node/apparmor/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/apparmor
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/keyrings
+	install -m 644 srv/salt/ceph/functests/1node/keyrings/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/keyrings
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/macros
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/macros/os_switch
+	install -m 644 srv/salt/ceph/functests/1node/macros/os_switch/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/macros/os_switch
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/openstack
+	install -m 644 srv/salt/ceph/functests/1node/openstack/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/openstack
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/quiescent
+	install -m 644 srv/salt/ceph/functests/1node/quiescent/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/quiescent
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/restart
+	install -m 644 srv/salt/ceph/functests/1node/restart/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/restart
+	install -d -m 755 $(DESTDIR)/srv/salt/ceph/functests/1node/tuned/off
+	install -m 644 srv/salt/ceph/functests/1node/tuned/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/tuned
+	install -m 644 srv/salt/ceph/functests/1node/tuned/off/*.sls $(DESTDIR)/srv/salt/ceph/functests/1node/tuned/off
 	# docs
 	install -d -m 755 $(DESTDIR)$(DOCDIR)/deepsea
 	install -m 644 LICENSE $(DESTDIR)$(DOCDIR)/deepsea/

deepsea.spec.in

@@ -745,16 +745,17 @@ the README for more information.
 
 %files qa
 %{_libexecdir}/deepsea/qa
-%dir /srv/salt/ceph/smoketests
-%dir /srv/salt/ceph/smoketests/apparmor
-%dir /srv/salt/ceph/smoketests/keyrings
-%dir /srv/salt/ceph/smoketests/macros
-%dir /srv/salt/ceph/smoketests/macros/os_switch
-%dir /srv/salt/ceph/smoketests/openstack
-%dir /srv/salt/ceph/smoketests/quiescent
-%dir /srv/salt/ceph/smoketests/restart
-%dir /srv/salt/ceph/smoketests/tuned
-%dir /srv/salt/ceph/smoketests/tuned/off
+%dir /srv/salt/ceph/functests
+%dir /srv/salt/ceph/functests/1node
+%dir /srv/salt/ceph/functests/1node/apparmor
+%dir /srv/salt/ceph/functests/1node/keyrings
+%dir /srv/salt/ceph/functests/1node/macros
+%dir /srv/salt/ceph/functests/1node/macros/os_switch
+%dir /srv/salt/ceph/functests/1node/openstack
+%dir /srv/salt/ceph/functests/1node/quiescent
+%dir /srv/salt/ceph/functests/1node/restart
+%dir /srv/salt/ceph/functests/1node/tuned
+%dir /srv/salt/ceph/functests/1node/tuned/off
 %dir /srv/salt/ceph/tests
 %dir /srv/salt/ceph/tests/keyrings
 %dir /srv/salt/ceph/tests/openstack
@@ -780,17 +781,17 @@ the README for more information.
 %dir /srv/salt/ceph/tests/restart/rgw/change
 %dir /srv/salt/ceph/tests/restart/rgw/forced
 %dir /srv/salt/ceph/tests/restart/rgw/nochange
-/srv/salt/ceph/smoketests/*.sls
-/srv/salt/ceph/smoketests/apparmor/*.sls
+/srv/salt/ceph/functests/1node/*.sls
+/srv/salt/ceph/functests/1node/apparmor/*.sls
 %dir /srv/salt/ceph/tests/tuned
 %dir /srv/salt/ceph/tests/tuned/off
-/srv/salt/ceph/smoketests/keyrings/*.sls
-/srv/salt/ceph/smoketests/macros/os_switch/*.sls
-/srv/salt/ceph/smoketests/openstack/*.sls
-/srv/salt/ceph/smoketests/quiescent/*.sls
-/srv/salt/ceph/smoketests/restart/*.sls
-/srv/salt/ceph/smoketests/tuned/*.sls
-/srv/salt/ceph/smoketests/tuned/off/*.sls
+/srv/salt/ceph/functests/1node/keyrings/*.sls
+/srv/salt/ceph/functests/1node/macros/os_switch/*.sls
+/srv/salt/ceph/functests/1node/openstack/*.sls
+/srv/salt/ceph/functests/1node/quiescent/*.sls
+/srv/salt/ceph/functests/1node/restart/*.sls
+/srv/salt/ceph/functests/1node/tuned/*.sls
+/srv/salt/ceph/functests/1node/tuned/off/*.sls
 /srv/salt/ceph/tests/keyrings/*.sls
 /srv/salt/ceph/tests/openstack/*.sls
 /srv/salt/ceph/tests/os_switch/*.sls

qa/common/common.sh

@@ -205,51 +205,6 @@ mgr initial modules = dashboard
 EOF
 }
 
-#
-# functions for testing ceph.restart orchestration
-#
-
-function restart_services {
-  rm -rf /srv/modules/runners/__pycache__
-  salt-run state.orch ceph.restart
-}
-
-function mon_restarted {
-  local expected_return=$1
-  local mon_hosts=$(salt --static --out json -C "I@roles:mon" test.ping | jq -r 'keys[]')
-  set +e
-  for minion in ${mon_hosts}; do
-    salt "${minion}*" cmd.shell "journalctl -u ceph-mon@*" | grep -i terminated
-    test $? = ${expected_return}
-  done
-  set -e
-}
-
-function osd_restarted {
-    local expected_return=$1
-    osd_hosts=$(salt --static --out json -C "I@roles:storage" test.ping | jq -r 'keys[]')
-    set +e
-    for host in ${osd_hosts}; do
-        osds=$(salt --static --out json ${host} osd.list | jq .[][])
-        for osd in ${osds}; do
-            salt ${host} cmd.shell "journalctl -u ceph-osd@${osd}" | grep -i terminated
-            test $? = ${expected_return}
-        done
-    done
-    set -e
-}
-
-function rgw_restarted {
-  local expected_return=$1
-  rgw_hosts=$(salt --static --out json -C "I@roles:rgw" test.ping | jq -r 'keys[]')
-  set +e
-  for host in ${rgw_hosts}; do
-    salt ${host} cmd.shell "journalctl -u ceph-radosgw@*" | grep -i terminated
-    test $? = ${expected_return}
-  done
-  set -e
-}
-
 
 #
 # functions for creating pools
@@ -342,11 +297,13 @@ function ceph_version_test {
   rpm -q ceph
   local RPM_NAME=$(rpm -q ceph)
   local RPM_CEPH_VERSION=$(perl -e '"'"$RPM_NAME"'" =~ m/ceph-(\d+\.\d+\.\d+)/; print "$1\n";')
-  echo "According to RPM, the ceph upstream version is $RPM_CEPH_VERSION"
+  echo "According to RPM, the ceph upstream version is ->$RPM_CEPH_VERSION<-"
+  test -n "$RPM_CEPH_VERSION"
   ceph --version
   local BUFFER=$(ceph --version)
   local CEPH_CEPH_VERSION=$(perl -e '"'"$BUFFER"'" =~ m/ceph version (\d+\.\d+\.\d+)/; print "$1\n";')
-  echo "According to \"ceph --version\", the ceph upstream version is $CEPH_CEPH_VERSION"
+  echo "According to \"ceph --version\", the ceph upstream version is ->$CEPH_CEPH_VERSION<-"
+  test -n "$RPM_CEPH_VERSION"
   test "$RPM_CEPH_VERSION" = "$CEPH_CEPH_VERSION"
 }
 

qa/common/deploy.sh

@@ -51,7 +51,7 @@ function vet_nodes {
 }
 
 function ceph_cluster_running {
-    ceph status 2>&1 >/dev/null
+    ceph status >/dev/null 2>&1
 }
 
 function deploy_ceph {

qa/common/helper.sh

@@ -19,15 +19,6 @@ function _report_stage_failure_and_die {
 
 function _run_stage {
   local stage_num=$1
-  local cli=$2
-  test -z "$cli" && cli="classic"
-  local stage_log_path="/tmp/stage.${stage_num}.log"
-  local deepsea_cli_output_path="/tmp/deepsea.${stage_num}.log"
-  local deepsea_exit_status=""
-
-  # workaround for https://bugzilla.suse.com/show_bug.cgi?id=1087232
-  # delete Salt runner __pycache__
-  rm -rf /srv/modules/runners/__pycache__
 
   set +x
   echo ""
@@ -37,34 +28,44 @@ function _run_stage {
   set -x
 
   # CLI case
-  if [ "x$cli" = "xcli" ] ; then
-      echo "using DeepSea CLI"
-      set +e
-      deepsea \
-          --log-file=/var/log/salt/deepsea.log \
-          --log-level=debug \
-          stage \
-          run \
-          ceph.stage.${stage_num} \
-          --simple-output \
-          2>&1 | tee $deepsea_cli_output_path
-      deepsea_exit_status="${PIPESTATUS[0]}"
-      echo "deepsea exit status: $deepsea_exit_status"
-      if [ "$deepsea_exit_status" = "0" ] ; then
-          if grep -q -F "failed=0" $deepsea_cli_output_path ; then
-              echo "DeepSea stage OK"
-          else
-              echo "ERROR: deepsea stage returned exit status 0, yet one or more steps failed. Bailing out!"
-              _report_stage_failure_and_die $stage_num
-          fi
+  test -n "$CLI" && _run_stage_cli $stage_num || _run_stage_non_cli $stage_num
+}
+
+function _run_stage_cli {
+  local stage_num=$1
+  local deepsea_cli_output_path="/tmp/deepsea.${stage_num}.log"
+  local deepsea_exit_status=""
+
+  echo "using DeepSea CLI"
+  set +e
+  deepsea \
+      --log-file=/var/log/salt/deepsea.log \
+      --log-level=debug \
+      stage \
+      run \
+      ceph.stage.${stage_num} \
+      --simple-output \
+      2>&1 | tee $deepsea_cli_output_path
+  deepsea_exit_status="${PIPESTATUS[0]}"
+  echo "deepsea exit status: $deepsea_exit_status"
+  echo "WWWW"
+  if [ "$deepsea_exit_status" = "0" ] ; then
+      if grep -q -F "failed=0" $deepsea_cli_output_path ; then
+          echo "********** Stage $stage_num completed successfully **********"
       else
+          echo "ERROR: deepsea stage returned exit status 0, yet one or more steps failed. Bailing out!"
           _report_stage_failure_and_die $stage_num
       fi
-      set -e
-      return
+  else
+      _report_stage_failure_and_die $stage_num
   fi
+  set -e
+}
+
+function _run_stage_non_cli {
+  local stage_num=$1
+  local stage_log_path="/tmp/stage.${stage_num}.log"
 
-  # non-CLI ("classic") case
   echo -n "" > $stage_log_path
   salt-run --no-color state.orch ceph.stage.${stage_num} 2>&1 | tee $stage_log_path
   STAGE_FINISHED=$(grep -F 'Total states run' $stage_log_path)

qa/common/policy.sh

@@ -83,14 +83,26 @@ EOF
     else
         echo "Unexpected number of client nodes ->$CLIENT_NODES<-; bailing out!"
         exit 1
-  fi
+    fi
 }
 
 function policy_cfg_mds {
-    cat <<EOF >> /srv/pillar/ceph/proposals/policy.cfg
-# Role assignment - mds (all but last node)
-role-mds/cluster/*.sls slice=[:-1]
+    test -n "$CLIENT_NODES"
+
+    if [ "$CLIENT_NODES" -eq 0 ] ; then
+        cat <<EOF >> /srv/pillar/ceph/proposals/policy.cfg
+# Role assignment - mds (all nodes)
+role-mds/cluster/*.sls
 EOF
+    elif [ "$CLIENT_NODES" -ge 1 ] ; then
+        cat <<EOF >> /srv/pillar/ceph/proposals/policy.cfg
+# Role assignment - mds (all non-client nodes)
+role-mds/cluster/*.sls slice=[:-$CLIENT_NODES]
+EOF
+    else
+        echo "Unexpected number of client nodes ->$CLIENT_NODES<-; bailing out!"
+        exit 1
+    fi
 }
 
 function policy_cfg_rgw {