Implement inband authentication

.env

@@ -86,3 +86,15 @@ BDEVPERF_CONTAINER_NAME="${COMPOSE_PROJECT_NAME}-bdevperf-1"
 DISC1="${COMPOSE_PROJECT_NAME}-discovery-1"
 GW1="${COMPOSE_PROJECT_NAME}-nvmeof-1"
 GW2="${COMPOSE_PROJECT_NAME}-nvmeof-2"
+
+# Keys for security tests
+PSK_KEY1="NVMeTLSkey-1:01:YzrPElk4OYy1uUERriPwiiyEJE/+J5ckYpLB+5NHMsR2iBuT:"     # generated using "nvme gen-tls-key"
+PSK_KEY2="NVMeTLSkey-1:01:vUrPe33Auz/sgAAcYctjI0oOOEFM5lheeLy7U+yTsD/LHm9q:"
+PSK_KEY3="NVMeTLSkey-1:01:IuIuyghntsi1iX5LdnlRp7MjON1QuYe4hELKYr5VwsL4AgHU:"
+
+DHCHAP_KEY1="DHHC-1:01:rPTE0Q73nd3hEqqEuQNaPL11G/aFXpOHtldWXz9vNCeef4WV:"        # generated using "nvme gen-dhchap-key"
+DHCHAP_KEY2="DHHC-1:01:x7ecfGgIdOEl+J5cJ9JcZHOS2By2Me6eDJUnrsT9MVrCWRYV:"
+DHCHAP_KEY3="DHHC-1:01:eNNXGjidEHHStbUi2Gmpps0JcnofReFfy+NaulguGgt327hz:"
+DHCHAP_KEY4="DHHC-1:01:c8D8fVPP/wcuxxRCd8mdQQFjOWtjcS2KmspzvkeOEoF6SUm6:"
+DHCHAP_KEY5="DHHC-1:01:zNZ6nrs5JDIpqbH/ZP1VTAATxNf5i/rH44dci+vvjhsyI2ha:"
+DHCHAP_KEY6="DHHC-1:01:Bu4tZd7X2oW7XxmVH5tGCdoS30pDX6bZvexHYoudeVlJW9yz:"

.github/workflows/build-container.yml

@@ -133,7 +133,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: ["cli", "cli_change_lb", "state", "multi_gateway", "server", "grpc", "omap_lock", "old_omap", "log_files", "nsid", "psk"]
+        test: ["cli", "cli_change_lb", "state", "multi_gateway", "server", "grpc", "omap_lock", "old_omap", "log_files", "nsid", "psk", "dhchap"]
     runs-on: ubuntu-latest
     env:
       HUGEPAGES: 512  # for multi gateway test, approx 256 per gateway instance
@@ -248,6 +248,10 @@ jobs:
 
   demo:
     needs: [build, build-ceph]
+    strategy:
+      fail-fast: false
+      matrix:
+        security_protocol: ["unsecured", "psk", "dhchap"]
     runs-on: ubuntu-latest
     env:
       HUGEPAGES: 512
@@ -297,155 +301,15 @@ jobs:
 
       - name: Test
         run: |
-          make demo OPTS=-T
-
-      - name: List resources
-        run: |
-          # https://github.com/actions/toolkit/issues/766
-          shopt -s expand_aliases
-          eval $(make alias)
-          cephnvmf subsystem list
-          subs=$(cephnvmf --output stdio --format json subsystem list | grep nqn | sed 's/"nqn": "//' | sed 's/",$//')
-          for sub in $subs
-          do
-              cephnvmf namespace list --subsystem $sub
-              cephnvmf listener list --subsystem $sub
-              cephnvmf host list --subsystem $sub
-          done
-
-      - name: Run bdevperf
-        run: |
-          # see https://spdk.io/doc/nvmf_multipath_howto.html
-          shopt -s expand_aliases
-          eval $(make alias)
-          . .env
-          set -x
-          echo -n "ℹ️  Starting bdevperf container"
-          docker compose up -d bdevperf
-          sleep 10
-          echo "ℹ️  bdevperf start up logs"
-          make logs SVC=bdevperf
-          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' )
-
-          rpc="/usr/libexec/spdk/scripts/rpc.py"
-          echo "ℹ️  bdevperf bdev_nvme_set_options"
-          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1"
-          echo "ℹ️  bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: $NVMEOF_IO_PORT nqn: $NQN"
-          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -f ipv4 -n $NQN -q ${NQN}host -l -1 -o 10"
-          echo "ℹ️  verify connection list"
-          conns=$(cephnvmf --output stdio --format json connection list --subsystem $NQN)
-          echo $conns | grep -q '"status": 0'
-          echo $conns | grep -q "\"nqn\": \"${NQN}host\""
-          echo $conns | grep -q "\"trsvcid\": ${NVMEOF_IO_PORT}"
-          echo $conns | grep -q "\"traddr\": \"${NVMEOF_IP_ADDRESS}\""
-          echo $conns | grep -q "\"adrfam\": \"ipv4\""
-          echo $conns | grep -q "\"trtype\": \"TCP\""
-          con_cnt=$(echo $conns | xargs -n 1 | grep traddr | wc -l)
-          if [ $con_cnt -ne 1 ]; then
-              echo "Number of connections ${con_cnt}, expected 1 list: ${conns}"
-              exit 1
-          fi
-          echo $conns | grep -q "\"qpairs_count\": 1"
-          echo $conns | grep -q "\"connected\": true"
-          echo "ℹ️  bdevperf perform_tests"
-          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' )
-          timeout=$(expr $BDEVPERF_TEST_DURATION \* 2)
-          bdevperf="/usr/libexec/spdk/scripts/bdevperf.py"
-          make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests"
-
-      - name: Check coredump existence
-        if: success() || failure()
-        id: check_coredumps
-        uses: andstor/file-existence-action@20b4d2e596410855db8f9ca21e96fbe18e12930b  # v2, pinned to SHA for security reasons
-        with:
-          files: "/tmp/coredump/core.*"
-
-      - name: Upload demo core dumps
-        if: steps.check_coredumps.outputs.files_exists == 'true'
-        uses: actions/upload-artifact@v4
-        with:
-          name: core_demo
-          path: /tmp/coredump/core.*
-
-      # For debugging purposes (provides an SSH connection to the runner)
-      # - name: Setup tmate session
-      #   uses: mxschmitt/action-tmate@v3
-      #   with:
-      #     limit-access-to-actor: true
-
-      - name: Display logs
-        if: success() || failure()
-        run: make logs OPTS=''
-
-      - name: Tear down
-        if: success() || failure()
-        run: |
-          make down
-          make clean
-
-  demo-secure:
-    needs: [build, build-ceph]
-    runs-on: ubuntu-latest
-    env:
-      HUGEPAGES: 512
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Setup huge-pages
-        run: make setup HUGEPAGES=$HUGEPAGES
-
-      - name: Download container images
-        uses: actions/download-artifact@v4
-        with:
-          pattern: container_images*
-          merge-multiple: true
-
-      - name: Load container images
-        run: |
-          docker load < nvmeof.tar
-          docker load < nvmeof-cli.tar
-          docker load < ceph.tar
-          docker load < bdevperf.tar
-
-      - name: Start containers
-        timeout-minutes: 3
-        run: |
-          make up
-
-      - name: Wait for the Gateway to be listening
-        timeout-minutes: 3
-        run: |
-          . .env
-
-          echo using gateway $NVMEOF_IP_ADDRESS port $NVMEOF_GW_PORT
-          until nc -z $NVMEOF_IP_ADDRESS $NVMEOF_GW_PORT; do
-            echo -n .
-            sleep ${{ env.WAIT_INTERVAL_SECS }}
-          done
-
-      - name: List containers
-        if: success() || failure()
-        run: make ps
-
-      - name: List processes
-        if: success() || failure()
-        run: make top
-
-      - name: Test
-        run: |
-          . .env
-          port2=`expr ${NVMEOF_IO_PORT} + 10`
-          make demosecure OPTS=-T HOSTNQN="${NQN}host" HOSTNQN2="${NQN}host2" NVMEOF_IO_PORT2=${port2}
+          ./tests/ha/demo_test.sh test_${{ matrix.security_protocol }}
 
       - name: List resources
         run: |
           # https://github.com/actions/toolkit/issues/766
           shopt -s expand_aliases
           eval $(make alias)
-          cephnvmf get_subsystems
           cephnvmf subsystem list
-          subs=$(cephnvmf --output stdio --format json subsystem list | grep nqn | sed 's/"nqn": "//' | sed 's/",$//')
+          subs=$(cephnvmf --output stdio --format json subsystem list | jq -r '.subsystems[].nqn')
           for sub in $subs
           do
               cephnvmf namespace list --subsystem $sub
@@ -455,70 +319,7 @@ jobs:
 
       - name: Run bdevperf
         run: |
-          # see https://spdk.io/doc/nvmf_multipath_howto.html
-          shopt -s expand_aliases
-          eval $(make alias)
-          . .env
-          set -x
-          echo -n "ℹ️  Starting bdevperf container"
-          docker compose up -d bdevperf
-          sleep 10
-          echo "ℹ️  bdevperf start up logs"
-          make logs SVC=bdevperf
-          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' )
-          psk_path_prefix="/tmp/psk/"
-          psk_path="${psk_path_prefix}${NQN}"
-          mkdir -p ${psk_path}
-          echo -n "NVMeTLSkey-1:01:YzrPElk4OYy1uUERriPwiiyEJE/+J5ckYpLB+5NHMsR2iBuT:" > ${psk_path}/${NQN}host
-          chmod 600 ${psk_path}/${NQN}host
-          docker cp ${psk_path_prefix} ${BDEVPERF_CONTAINER_NAME}:${psk_path_prefix}
-
-          rpc="/usr/libexec/spdk/scripts/rpc.py"
-          port2=`expr ${NVMEOF_IO_PORT} + 10`
-          echo "ℹ️  bdevperf bdev_nvme_set_options"
-          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1"
-          echo "ℹ️  bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: $NVMEOF_IO_PORT nqn: $NQN"
-          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -f ipv4 -n $NQN -q ${NQN}host -l -1 -o 10 --psk ${psk_path}/${NQN}host"
-          echo "ℹ️  verify connection list"
-          conns=$(cephnvmf --output stdio --format json connection list --subsystem $NQN)
-          echo $conns | grep -q '"status": 0'
-          echo $conns | grep -q "\"nqn\": \"${NQN}host\""
-          echo $conns | grep -q "\"trsvcid\": ${NVMEOF_IO_PORT}"
-          echo $conns | grep -q "\"traddr\": \"${NVMEOF_IP_ADDRESS}\""
-          echo $conns | grep -q "\"adrfam\": \"ipv4\""
-          echo $conns | grep -q "\"trtype\": \"TCP\""
-          echo $conns | grep -q "\"qpairs_count\": 1"
-          echo $conns | grep -q "\"connected\": true"
-          echo $conns | grep -q "\"secure\": true"
-          echo $conns | grep -q -v "\"secure\": false"
-          echo $conns | grep -q "\"use_psk\": true"
-          echo $conns | grep -q "\"use_psk\": false"
-          con_cnt=$(echo $conns | xargs -n 2 | grep traddr | grep -v "n/a" | wc -l)
-          if [ $con_cnt -ne 1 ]; then
-              echo "Number of connections ${con_cnt}, expected 1, list: ${conns}"
-              exit 1
-          fi
-          echo "ℹ️  bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: ${port2} nqn: ${NQN}host2"
-          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme1 -t tcp -a $NVMEOF_IP_ADDRESS -s ${port2} -f ipv4 -n $NQN -q "${NQN}host2" -l -1 -o 10"
-          echo "ℹ️  verify connection list again"
-          conns=$(cephnvmf --output stdio --format json connection list --subsystem $NQN)
-          con_cnt=$(echo $conns | xargs -n 2 | grep traddr | grep -v "n/a" | wc -l)
-          if [ $con_cnt -ne 2 ]; then
-              echo "Number of connections ${con_cnt}, expected 2, list: ${conns}"
-              exit 1
-          fi
-          echo $conns | grep -q "\"nqn\": \"${NQN}host2\""
-          echo $conns | grep -q "\"trsvcid\": ${port2}"
-          echo $conns | grep -q "\"secure\": true"
-          echo $conns | grep -q "\"secure\": false"
-          echo $conns | grep -q "\"use_psk\": true"
-          echo $conns | grep -q "\"use_psk\": false"
-          echo "ℹ️  bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: ${port2} nqn: ${NQN}host2"
-          echo "ℹ️  bdevperf perform_tests"
-          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' )
-          timeout=$(expr $BDEVPERF_TEST_DURATION \* 2)
-          bdevperf="/usr/libexec/spdk/scripts/bdevperf.py"
-          make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests"
+          ./tests/ha/demo_test.sh bdevperf_${{ matrix.security_protocol }}
 
       - name: Check coredump existence
         if: success() || failure()
@@ -886,7 +687,7 @@ jobs:
 
   push-images-to-ceph-registry:
     if: github.event_name == 'release'
-    needs: [pytest, demo, demo-secure, discovery, ha, atom, build-arm64]
+    needs: [pytest, demo, discovery, ha, atom, build-arm64]
     runs-on: ubuntu-latest
 
     steps:

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "spdk"]
 	path = spdk
 	url = https://github.com/ceph/spdk.git
-	branch =  ceph-nvmeof-v24.05
+	branch = ceph-nvmeof-v24.05

Makefile

@@ -21,7 +21,8 @@ endif
 # Includes
 include mk/containerized.mk
 include mk/demo.mk
-include mk/demosecure.mk
+include mk/demosecurepsk.mk
+include mk/demosecuredhchap.mk
 include mk/misc.mk
 include mk/autohelp.mk
 

control/cli.py

@@ -1056,14 +1056,28 @@ def host_add(self, args):
         if args.psk:
             if len(args.host_nqn) > 1:
                 self.cli.parser.error(f"Can't have more than one host NQN when PSK keys are used")
+            if args.dhchap_key:
+                self.cli.parser.error(f"PSK and DH-HMAC-CHAP keys are mutually exclusive")
+
+        if args.dhchap_key:
+            if len(args.host_nqn) > 1:
+                self.cli.parser.error(f"Can't have more than one host NQN when DH-HMAC-CHAP keys are used")
+
+        if args.dhchap_ctrlr_key:
+            if not args.dhchap_key:
+                self.cli.parser.error(f"DH-HMAC-CHAP controller keys can not be used without DH-HMAC-CHAP keys")
 
         for i in range(len(args.host_nqn)):
             one_host_nqn = args.host_nqn[i]
 
             if one_host_nqn == "*" and args.psk:
-                self.cli.parser.error(f"PSK is only allowed for specific hosts")
+                self.cli.parser.error(f"PSK key is only allowed for specific hosts")
+
+            if one_host_nqn == "*" and args.dhchap_key:
+                self.cli.parser.error(f"DH-HMAC-CHAP key is only allowed for specific hosts")
 
-            req = pb2.add_host_req(subsystem_nqn=args.subsystem, host_nqn=one_host_nqn, psk=args.psk)
+            req = pb2.add_host_req(subsystem_nqn=args.subsystem, host_nqn=one_host_nqn,
+                                   psk=args.psk, dhchap_key=args.dhchap_key, dhchap_ctrlr_key=args.dhchap_ctrlr_key)
             try:
                 ret = self.stub.add_host(req)
             except Exception as ex:
@@ -1173,14 +1187,15 @@ def host_list(self, args):
                     hosts_list.append(["Any host", "n/a"])
                 for h in hosts_info.hosts:
                     use_psk = "Yes" if h.use_psk else "No"
-                    hosts_list.append([h.nqn, use_psk])
+                    use_dhchap = "Yes" if h.use_dhchap else "No"
+                    hosts_list.append([h.nqn, use_psk, use_dhchap])
                 if len(hosts_list) > 0:
                     if args.format == "text":
                         table_format = "fancy_grid"
                     else:
                         table_format = "plain"
                     hosts_out = tabulate(hosts_list,
-                                      headers = ["Host NQN", "Uses PSK"],
+                                      headers = ["Host NQN", "Uses PSK", "Uses DHCHAP"],
                                       tablefmt=table_format, stralign="center")
                     out_func(f"Hosts allowed to access {args.subsystem}:\n{hosts_out}")
                 else:
@@ -1210,7 +1225,9 @@ def host_list(self, args):
     ]
     host_add_args = host_common_args + [
         argument("--host-nqn", "-t", help="Host NQN list", nargs="+", required=True),
-        argument("--psk", help="Hosts PSK key list", required=False),
+        argument("--psk", help="Hosts PSK key", required=False),
+        argument("--dhchap-key", help="Host DH-HMAC-CHAP key", required=False),
+        argument("--dhchap-ctrlr-key", help="Host DH-HMAC-CHAP controller key", required=False),
     ]
     host_del_args = host_common_args + [
         argument("--host-nqn", "-t", help="Host NQN list", nargs="+", required=True),
@@ -1251,6 +1268,7 @@ def connection_list(self, args):
                 for conn in connections_info.connections:
                     conn_secure = "<n/a>"
                     conn_psk = "Yes" if conn.use_psk else "No"
+                    conn_dhchap = "Yes" if conn.use_dhchap else "No"
                     if conn.connected:
                         conn_secure = "Yes" if conn.secure else "No"
                     connections_list.append([conn.nqn,
@@ -1259,14 +1277,15 @@ def connection_list(self, args):
                                             conn.qpairs_count if conn.connected else "<n/a>",
                                             conn.controller_id if conn.connected else "<n/a>",
                                             conn_secure,
-                                            conn_psk])
+                                            conn_psk,
+                                            conn_dhchap])
                 if len(connections_list) > 0:
                     if args.format == "text":
                         table_format = "fancy_grid"
                     else:
                         table_format = "plain"
                     connections_out = tabulate(connections_list,
-                                      headers = ["Host NQN", "Address", "Connected", "QPairs Count", "Controller ID", "Secure", "PSK"],
+                                      headers = ["Host NQN", "Address", "Connected", "QPairs Count", "Controller ID", "Secure", "Uses\nPSK", "Uses\nDHCHAP"],
                                       tablefmt=table_format)
                     out_func(f"Connections for {args.subsystem}:\n{connections_out}")
                 else: