Merge pull request ceph#60431 from VallariAg/fix-nvmeof-ns-test

qa: fix nvmeof scalability and namespace test

qa/suites/nvmeof/basic/workloads/nvmeof_initiator.yaml

@@ -1,7 +1,8 @@
+# runs on default nvmeof image (i.e. DEFAULT_NVMEOF_IMAGE)
 tasks:
 - nvmeof:
     installer: host.a
-    gw_image: quay.io/ceph/nvmeof:latest # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest"
+    gw_image: default # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest"
     rbd:
       pool_name: mypool
       image_name_prefix: myimage

qa/suites/nvmeof/basic/workloads/nvmeof_namespaces.yaml

@@ -18,6 +18,7 @@ tasks:
     clients:
       client.0:
         - nvmeof/setup_subsystem.sh
+        - nvmeof/basic_tests.sh
     env:
       RBD_POOL: mypool
       RBD_IMAGE_PREFIX: myimage
@@ -27,7 +28,6 @@ tasks:
     timeout: 30m
     clients:
       client.0:
-        - nvmeof/basic_tests.sh
         - nvmeof/fio_test.sh --rbd_iostat
       client.1:
         - nvmeof/basic_tests.sh

qa/suites/nvmeof/basic/workloads/nvmeof_scalability.yaml

@@ -31,8 +31,11 @@ tasks:
     no_coverage_and_limits: true
     timeout: 30m
     clients:
-      client.0:
+      client.3:
         - nvmeof/scalability_test.sh nvmeof.a,nvmeof.b
         - nvmeof/scalability_test.sh nvmeof.b,nvmeof.c,nvmeof.d
+        - nvmeof/scalability_test.sh nvmeof.b,nvmeof.c
     env:
       SCALING_DELAYS: '50'
+      RBD_POOL: mypool
+      NVMEOF_GROUP: mygroup0

qa/suites/nvmeof/thrash/gateway-initiator-setup/10-subsys-90-namespace-no_huge_pages.yaml

@@ -0,0 +1,37 @@
+tasks:
+- nvmeof:
+    installer: host.a
+    gw_image: quay.io/ceph/nvmeof:latest # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest"
+    rbd:
+      pool_name: mypool
+      image_name_prefix: myimage
+    gateway_config:
+      subsystems_count: 10 
+      namespaces_count: 90 # each subsystem
+      cli_image: quay.io/ceph/nvmeof-cli:latest
+
+- cephadm.wait_for_service:
+    service: nvmeof.mypool.mygroup0
+
+- cephadm.exec:
+    host.a:
+      - ceph orch ls nvmeof --export > /tmp/nvmeof-orig.yaml
+      - cp /tmp/nvmeof-orig.yaml /tmp/nvmeof-no-huge-page.yaml 
+      - "sed -i '/  pool: mypool/a\\  spdk_mem_size: 4096' /tmp/nvmeof-no-huge-page.yaml"
+      - cat /tmp/nvmeof-no-huge-page.yaml
+      - ceph orch ls --refresh
+      - ceph orch apply -i /tmp/nvmeof-no-huge-page.yaml
+      - ceph orch redeploy nvmeof.mypool.mygroup0
+
+- cephadm.wait_for_service:
+    service: nvmeof.mypool.mygroup0
+
+- workunit:
+    no_coverage_and_limits: true
+    clients:
+      client.0:
+        - nvmeof/setup_subsystem.sh
+        - nvmeof/basic_tests.sh
+    env:
+      RBD_POOL: mypool
+      RBD_IMAGE_PREFIX: myimage

qa/suites/nvmeof/thrash/gateway-initiator-setup/3-subsys-60-namespace.yaml → qa/suites/nvmeof/thrash/gateway-initiator-setup/120-subsys-8-namespace.yaml

@@ -6,8 +6,8 @@ tasks:
       pool_name: mypool
       image_name_prefix: myimage
     gateway_config:
-      subsystems_count: 3
-      namespaces_count: 20 # each subsystem
+      subsystems_count: 120
+      namespaces_count: 8 # each subsystem
       cli_image: quay.io/ceph/nvmeof-cli:latest
 
 - cephadm.wait_for_service:

qa/suites/nvmeof/thrash/thrashers/nvmeof_mon_thrash.yaml

@@ -11,6 +11,7 @@ overrides:
       - NVMEOF_SINGLE_GATEWAY
       - NVMEOF_GATEWAY_DOWN
       - are in unavailable state
+      - is unavailable
       - is in error state
       - failed cephadm daemon
 

qa/suites/nvmeof/thrash/thrashers/nvmeof_thrash.yaml

@@ -6,9 +6,11 @@ overrides:
       - NVMEOF_SINGLE_GATEWAY
       - NVMEOF_GATEWAY_DOWN
       - are in unavailable state
+      - is unavailable
       - is in error state
       - failed cephadm daemon
 
 tasks:
 - nvmeof.thrash:
     checker_host: 'client.0'
+    randomize: False

qa/suites/nvmeof/thrash/workloads/fio.yaml

@@ -1,11 +1,11 @@
 tasks:
 - workunit:
     no_coverage_and_limits: true
-    timeout: 30m
+    timeout: 60m
     clients:
       client.0:
-        - nvmeof/fio_test.sh --rbd_iostat
+        - nvmeof/fio_test.sh --random_devices 200
     env:
       RBD_POOL: mypool
       IOSTAT_INTERVAL: '10'
-      RUNTIME: '600'
+      RUNTIME: '1800'

qa/tasks/nvmeof.py

@@ -128,12 +128,11 @@ def deploy_nvmeof(self):
 
             total_images = int(self.namespaces_count) * int(self.subsystems_count)
             log.info(f'[nvmeof]: creating {total_images} images')
+            rbd_create_cmd = []
             for i in range(1, total_images + 1):
                 imagename = self.image_name_prefix + str(i)
-                log.info(f'[nvmeof]: rbd create {poolname}/{imagename} --size {self.rbd_size}')
-                _shell(self.ctx, self.cluster_name, self.remote, [
-                    'rbd', 'create', f'{poolname}/{imagename}', '--size', f'{self.rbd_size}'
-                ])
+                rbd_create_cmd += ['rbd', 'create', f'{poolname}/{imagename}', '--size', f'{self.rbd_size}', run.Raw(';')]           
+            _shell(self.ctx, self.cluster_name, self.remote, rbd_create_cmd)     
 
         for role, i in daemons.items():
             remote, id_ = i
@@ -251,9 +250,9 @@ class NvmeofThrasher(Thrasher, Greenlet):
 
     daemon_max_thrash_times: 
                         For now, NVMeoF daemons have limitation that each daemon can 
-                        be thrashed only 3 times in span of 30 mins. This option 
+                        be thrashed only 5 times in span of 30 mins. This option 
                         allows to set the amount of times it could be thrashed in a period
-                        of time. (default: 3)
+                        of time. (default: 5)
     daemon_max_thrash_period: 
                         This option goes with the above option. It sets the period of time
                         over which each daemons can be thrashed for daemon_max_thrash_times
@@ -306,12 +305,12 @@ def __init__(self, ctx, config, daemons) -> None:
         self.max_thrash_daemons = int(self.config.get('max_thrash', len(self.daemons) - 1))
 
         # Limits on thrashing each daemon
-        self.daemon_max_thrash_times = int(self.config.get('daemon_max_thrash_times', 3))
+        self.daemon_max_thrash_times = int(self.config.get('daemon_max_thrash_times', 5))
         self.daemon_max_thrash_period = int(self.config.get('daemon_max_thrash_period', 30 * 60)) # seconds
 
         self.min_thrash_delay = int(self.config.get('min_thrash_delay', 60))
         self.max_thrash_delay = int(self.config.get('max_thrash_delay', self.min_thrash_delay + 30))
-        self.min_revive_delay = int(self.config.get('min_revive_delay', 100))
+        self.min_revive_delay = int(self.config.get('min_revive_delay', 60))
         self.max_revive_delay = int(self.config.get('max_revive_delay', self.min_revive_delay + 30))
 
     def _get_devices(self, remote):
@@ -347,6 +346,7 @@ def do_checks(self):
             run.Raw('&&'), 'ceph', 'orch', 'ps', '--daemon-type', 'nvmeof',
             run.Raw('&&'), 'ceph', 'health', 'detail',
             run.Raw('&&'), 'ceph', '-s',
+            run.Raw('&&'), 'sudo', 'nvme', 'list',
         ]
         for dev in self.devices:
             check_cmd += [
@@ -421,13 +421,11 @@ def do_thrash(self):
         while not self.stopping.is_set():
             killed_daemons = defaultdict(list)
 
-            weight = 1.0 / len(self.daemons)
-            count = 0
+            thrash_daemon_num = self.rng.randint(1, self.max_thrash_daemons)
+            selected_daemons = self.rng.sample(self.daemons, thrash_daemon_num)
             for daemon in self.daemons:
-                skip = self.rng.uniform(0.0, 1.0)
-                if weight <= skip:
-                    self.log('skipping daemon {label} with skip ({skip}) > weight ({weight})'.format(
-                        label=daemon.id_, skip=skip, weight=weight))
+                if daemon not in selected_daemons:
+                    self.log(f'skipping daemon {daemon.id_} ...')
                     continue
 
                 # For now, nvmeof daemons can only be thrashed 3 times in last 30mins. 
@@ -445,17 +443,11 @@ def do_thrash(self):
                         continue
 
                 self.log('kill {label}'.format(label=daemon.id_))
-                # daemon.stop()
                 kill_method = self.kill_daemon(daemon)
 
                 killed_daemons[kill_method].append(daemon)
                 daemons_thrash_history[daemon.id_] += [datetime.now()]
 
-                # only thrash max_thrash_daemons amount of daemons
-                count += 1
-                if count >= self.max_thrash_daemons:
-                    break
-
             if killed_daemons:
                 iteration_summary = "thrashed- "
                 for kill_method in killed_daemons:
@@ -468,7 +460,7 @@ def do_thrash(self):
 
                 self.log(f'waiting for {revive_delay} secs before reviving')
                 time.sleep(revive_delay) # blocking wait
-                self.log('done waiting before reviving')
+                self.log(f'done waiting before reviving - iteration #{len(summary)}: {iteration_summary}')
 
                 self.do_checks()
                 self.switch_task()
@@ -487,7 +479,7 @@ def do_thrash(self):
                 if thrash_delay > 0.0:
                     self.log(f'waiting for {thrash_delay} secs before thrashing')
                     time.sleep(thrash_delay) # blocking
-                    self.log('done waiting before thrashing')
+                    self.log('done waiting before thrashing - everything should be up now')
 
                 self.do_checks()
                 self.switch_task()

qa/workunits/nvmeof/basic_tests.sh

@@ -38,8 +38,10 @@ disconnect_all() {
 connect_all() {
     sudo nvme connect-all --traddr=$NVMEOF_DEFAULT_GATEWAY_IP_ADDRESS --transport=tcp -l 3600
     sleep 5
-    output=$(sudo nvme list --output-format=json)
-    if ! echo "$output" | grep -q "$SPDK_CONTROLLER"; then
+    expected_devices_count=$1
+    actual_devices=$(sudo nvme list --output-format=json | grep -o "$SPDK_CONTROLLER" | wc -l) 
+    if [ "$actual_devices" -ne "$expected_devices_count" ]; then
+        sudo nvme list --output-format=json
         return 1
     fi
 }
@@ -72,11 +74,13 @@ test_run connect
 test_run list_subsys 1
 test_run disconnect_all
 test_run list_subsys 0
-test_run connect_all
+devices_count=$(( $NVMEOF_NAMESPACES_COUNT * $NVMEOF_SUBSYSTEMS_COUNT)) 
+test_run connect_all $devices_count
 gateways_count=$(( $(echo "$NVMEOF_GATEWAY_IP_ADDRESSES" | tr -cd ',' | wc -c) + 1 ))
 multipath_count=$(( $gateways_count * $NVMEOF_SUBSYSTEMS_COUNT)) 
 test_run list_subsys $multipath_count
 
 
+
 echo "-------------Test Summary-------------"
 echo "[nvmeof] All nvmeof basic tests passed!"

qa/workunits/nvmeof/fio_test.sh

@@ -5,6 +5,7 @@ sudo yum -y install sysstat
 
 namespace_range_start=
 namespace_range_end=
+random_devices_count=
 rbd_iostat=false
 
 while [[ $# -gt 0 ]]; do
@@ -17,6 +18,10 @@ while [[ $# -gt 0 ]]; do
             namespace_range_end=$2
             shift 2
             ;;
+        --random_devices)
+            random_devices_count=$2
+            shift 2
+            ;;
         --rbd_iostat)
             rbd_iostat=true
             shift
@@ -37,6 +42,8 @@ all_drives_list=$(sudo nvme list --output-format=json |
 # run on first 3 namespaces here.
 if [ "$namespace_range_start" ] || [ "$namespace_range_end" ]; then
     selected_drives=$(echo "${all_drives_list[@]}" | sed -n "${namespace_range_start},${namespace_range_end}p")
+elif [ "$random_devices_count" ]; then 
+    selected_drives=$(echo "${all_drives_list[@]}" | shuf -n $random_devices_count)
 else
     selected_drives="${all_drives_list[@]}"
 fi

qa/workunits/nvmeof/scalability_test.sh

@@ -3,37 +3,64 @@
 
 GATEWAYS=$1 # exmaple "nvmeof.a,nvmeof.b"
 DELAY="${SCALING_DELAYS:-50}"
+POOL="${RBD_POOL:-mypool}"
+GROUP="${NVMEOF_GROUP:-mygroup0}"
+source /etc/ceph/nvmeof.env
 
 if [ -z "$GATEWAYS" ]; then
     echo "At least one gateway needs to be defined for scalability test"
     exit 1
 fi
 
-pip3 install yq
-
 status_checks() {
-    ceph nvme-gw show mypool ''
-    ceph orch ls
-    ceph orch ps 
-    ceph -s
+    expected_count=$1
+
+    output=$(ceph nvme-gw show $POOL $GROUP) 
+    nvme_show=$(echo $output | grep -o '"AVAILABLE"' | wc -l)
+    if [ "$nvme_show" -ne "$expected_count" ]; then
+        return 1
+    fi
+
+    orch_ls=$(ceph orch ls)
+    if ! echo "$orch_ls" | grep -q "$expected_count/$expected_count"; then
+        return 1
+    fi
+
+    output=$(ceph orch ps --service-name nvmeof.$POOL.$GROUP)     
+    orch_ps=$(echo $output | grep -o 'running' | wc -l)
+    if [ "$orch_ps" -ne "$expected_count" ]; then
+        return 1
+    fi
+
+    ceph_status=$(ceph -s)
+    if ! echo "$ceph_status" | grep -q "HEALTH_OK"; then
+        return 1
+    fi
 }
 
+total_gateways_count=$(( $(echo "$NVMEOF_GATEWAY_IP_ADDRESSES" | tr -cd ',' | wc -c) + 1 ))
+scaled_down_gateways_count=$(( total_gateways_count - $(echo "$GATEWAYS" | tr -cd ',' | wc -c) - 1 ))
+
 
 echo "[nvmeof.scale] Setting up config to remove gateways ${GATEWAYS}"
+ceph orch ls --service-name nvmeof.$POOL.$GROUP --export > /tmp/nvmeof-gw.yaml
 ceph orch ls nvmeof --export > /tmp/nvmeof-gw.yaml
 cat /tmp/nvmeof-gw.yaml
-yq "del(.placement.hosts[] | select(. | test(\".*($(echo $GATEWAYS | sed 's/,/|/g'))\")))" /tmp/nvmeof-gw.yaml > /tmp/nvmeof-gw-new.yaml
+
+pattern=$(echo $GATEWAYS | sed 's/,/\\|/g')
+sed "/$pattern/d" /tmp/nvmeof-gw.yaml > /tmp/nvmeof-gw-new.yaml  
 cat /tmp/nvmeof-gw-new.yaml
 
 echo "[nvmeof.scale] Starting scale testing by removing ${GATEWAYS}"
-status_checks
-ceph orch rm nvmeof.mypool && sleep 20 # temp workaround
+status_checks $total_gateways_count 
 ceph orch apply -i /tmp/nvmeof-gw-new.yaml # downscale
+ceph orch redeploy nvmeof.$POOL.$GROUP 
 sleep $DELAY
-status_checks
-ceph orch rm nvmeof.mypool && sleep 20 # temp workaround
+status_checks $scaled_down_gateways_count
+echo "[nvmeof.scale] Downscale complete - removed gateways (${GATEWAYS}); now scaling back up"
 ceph orch apply -i /tmp/nvmeof-gw.yaml #upscale
+ceph orch redeploy nvmeof.$POOL.$GROUP 
 sleep $DELAY
-status_checks
+status_checks $total_gateways_count
 
 echo "[nvmeof.scale] Scale testing passed for ${GATEWAYS}"

qa/workunits/nvmeof/setup_subsystem.sh

@@ -26,14 +26,21 @@ list_subsystems () {
     done
 }
 
+list_namespaces () { 
+    for i in $(seq 1 $NVMEOF_SUBSYSTEMS_COUNT); do
+        subsystem_nqn="${NVMEOF_SUBSYSTEMS_PREFIX}${i}"
+        sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_DEFAULT_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT --format plain namespace list --subsystem $subsystem_nqn        
+    done
+}
+
+echo "[nvmeof] Starting subsystem setup..."
+
 # add all subsystems
 for i in $(seq 1 $NVMEOF_SUBSYSTEMS_COUNT); do
     subsystem_nqn="${NVMEOF_SUBSYSTEMS_PREFIX}${i}"
     sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_DEFAULT_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT subsystem add --subsystem $subsystem_nqn --no-group-append
 done
 
-list_subsystems
-
 # add all gateway listeners 
 for i in "${!gateway_ips[@]}"
 do
@@ -65,11 +72,5 @@ done
 
 list_subsystems
 
-# list namespaces
-for i in $(seq 1 $NVMEOF_SUBSYSTEMS_COUNT); do
-    subsystem_nqn="${NVMEOF_SUBSYSTEMS_PREFIX}${i}"
-    sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $NVMEOF_DEFAULT_GATEWAY_IP_ADDRESS --server-port $NVMEOF_SRPORT --format plain namespace list --subsystem $subsystem_nqn        
-done
-
 
 echo "[nvmeof] Subsystem setup done"