From 94e96bead00428affb1d24f07d410005c3e9db4a Mon Sep 17 00:00:00 2001 From: Gil Bregman Date: Tue, 8 Oct 2024 13:15:21 +0300 Subject: [PATCH] Implement inband authentication. Signed-off-by: Gil Bregman --- .env | 12 + .github/workflows/build-container.yml | 217 +------------ .gitmodules | 2 +- Makefile | 3 +- control/cli.py | 33 +- control/grpc.py | 301 ++++++++++++++++-- control/proto/gateway.proto | 4 + mk/demosecuredhchap.mk | 22 ++ mk/{demosecure.mk => demosecurepsk.mk} | 14 +- tests/ha/demo_test.sh | 420 +++++++++++++++++++++++++ tests/test_dhchap.py | 189 +++++++++++ tests/test_psk.py | 37 +-- 12 files changed, 987 insertions(+), 267 deletions(-) create mode 100644 mk/demosecuredhchap.mk rename mk/{demosecure.mk => demosecurepsk.mk} (74%) create mode 100755 tests/ha/demo_test.sh create mode 100644 tests/test_dhchap.py diff --git a/.env b/.env index aaca6de8..270679d0 100644 --- a/.env +++ b/.env @@ -86,3 +86,15 @@ BDEVPERF_CONTAINER_NAME="${COMPOSE_PROJECT_NAME}-bdevperf-1" DISC1="${COMPOSE_PROJECT_NAME}-discovery-1" GW1="${COMPOSE_PROJECT_NAME}-nvmeof-1" GW2="${COMPOSE_PROJECT_NAME}-nvmeof-2" + +# Keys for security tests +PSK_KEY1="NVMeTLSkey-1:01:YzrPElk4OYy1uUERriPwiiyEJE/+J5ckYpLB+5NHMsR2iBuT:" # generated using "nvme gen-tls-key" +PSK_KEY2="NVMeTLSkey-1:01:vUrPe33Auz/sgAAcYctjI0oOOEFM5lheeLy7U+yTsD/LHm9q:" +PSK_KEY3="NVMeTLSkey-1:01:IuIuyghntsi1iX5LdnlRp7MjON1QuYe4hELKYr5VwsL4AgHU:" + +DHCHAP_KEY1="DHHC-1:01:rPTE0Q73nd3hEqqEuQNaPL11G/aFXpOHtldWXz9vNCeef4WV:" # generated using "nvme gen-dhchap-key" +DHCHAP_KEY2="DHHC-1:01:x7ecfGgIdOEl+J5cJ9JcZHOS2By2Me6eDJUnrsT9MVrCWRYV:" +DHCHAP_KEY3="DHHC-1:01:eNNXGjidEHHStbUi2Gmpps0JcnofReFfy+NaulguGgt327hz:" +DHCHAP_KEY4="DHHC-1:01:c8D8fVPP/wcuxxRCd8mdQQFjOWtjcS2KmspzvkeOEoF6SUm6:" +DHCHAP_KEY5="DHHC-1:01:zNZ6nrs5JDIpqbH/ZP1VTAATxNf5i/rH44dci+vvjhsyI2ha:" +DHCHAP_KEY6="DHHC-1:01:Bu4tZd7X2oW7XxmVH5tGCdoS30pDX6bZvexHYoudeVlJW9yz:" diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index b43ed9bc..c4540b61 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -133,7 +133,7 @@ jobs: strategy: fail-fast: false matrix: - test: ["cli", "cli_change_lb", "state", "multi_gateway", "server", "grpc", "omap_lock", "old_omap", "log_files", "nsid", "psk"] + test: ["cli", "cli_change_lb", "state", "multi_gateway", "server", "grpc", "omap_lock", "old_omap", "log_files", "nsid", "psk", "dhchap"] runs-on: ubuntu-latest env: HUGEPAGES: 512 # for multi gateway test, approx 256 per gateway instance @@ -248,6 +248,10 @@ jobs: demo: needs: [build, build-ceph] + strategy: + fail-fast: false + matrix: + security_protocol: ["unsecured", "psk", "dhchap"] runs-on: ubuntu-latest env: HUGEPAGES: 512 @@ -297,155 +301,15 @@ jobs: - name: Test run: | - make demo OPTS=-T - - - name: List resources - run: | - # https://github.com/actions/toolkit/issues/766 - shopt -s expand_aliases - eval $(make alias) - cephnvmf subsystem list - subs=$(cephnvmf --output stdio --format json subsystem list | grep nqn | sed 's/"nqn": "//' | sed 's/",$//') - for sub in $subs - do - cephnvmf namespace list --subsystem $sub - cephnvmf listener list --subsystem $sub - cephnvmf host list --subsystem $sub - done - - - name: Run bdevperf - run: | - # see https://spdk.io/doc/nvmf_multipath_howto.html - shopt -s expand_aliases - eval $(make alias) - . .env - set -x - echo -n "ℹ️ Starting bdevperf container" - docker compose up -d bdevperf - sleep 10 - echo "ℹ️ bdevperf start up logs" - make logs SVC=bdevperf - eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' ) - - rpc="/usr/libexec/spdk/scripts/rpc.py" - echo "ℹ️ bdevperf bdev_nvme_set_options" - make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1" - echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: $NVMEOF_IO_PORT nqn: $NQN" - make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -f ipv4 -n $NQN -q ${NQN}host -l -1 -o 10" - echo "ℹ️ verify connection list" - conns=$(cephnvmf --output stdio --format json connection list --subsystem $NQN) - echo $conns | grep -q '"status": 0' - echo $conns | grep -q "\"nqn\": \"${NQN}host\"" - echo $conns | grep -q "\"trsvcid\": ${NVMEOF_IO_PORT}" - echo $conns | grep -q "\"traddr\": \"${NVMEOF_IP_ADDRESS}\"" - echo $conns | grep -q "\"adrfam\": \"ipv4\"" - echo $conns | grep -q "\"trtype\": \"TCP\"" - con_cnt=$(echo $conns | xargs -n 1 | grep traddr | wc -l) - if [ $con_cnt -ne 1 ]; then - echo "Number of connections ${con_cnt}, expected 1 list: ${conns}" - exit 1 - fi - echo $conns | grep -q "\"qpairs_count\": 1" - echo $conns | grep -q "\"connected\": true" - echo "ℹ️ bdevperf perform_tests" - eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' ) - timeout=$(expr $BDEVPERF_TEST_DURATION \* 2) - bdevperf="/usr/libexec/spdk/scripts/bdevperf.py" - make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests" - - - name: Check coredump existence - if: success() || failure() - id: check_coredumps - uses: andstor/file-existence-action@20b4d2e596410855db8f9ca21e96fbe18e12930b # v2, pinned to SHA for security reasons - with: - files: "/tmp/coredump/core.*" - - - name: Upload demo core dumps - if: steps.check_coredumps.outputs.files_exists == 'true' - uses: actions/upload-artifact@v4 - with: - name: core_demo - path: /tmp/coredump/core.* - - # For debugging purposes (provides an SSH connection to the runner) - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 - # with: - # limit-access-to-actor: true - - - name: Display logs - if: success() || failure() - run: make logs OPTS='' - - - name: Tear down - if: success() || failure() - run: | - make down - make clean - - demo-secure: - needs: [build, build-ceph] - runs-on: ubuntu-latest - env: - HUGEPAGES: 512 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup huge-pages - run: make setup HUGEPAGES=$HUGEPAGES - - - name: Download container images - uses: actions/download-artifact@v4 - with: - pattern: container_images* - merge-multiple: true - - - name: Load container images - run: | - docker load < nvmeof.tar - docker load < nvmeof-cli.tar - docker load < ceph.tar - docker load < bdevperf.tar - - - name: Start containers - timeout-minutes: 3 - run: | - make up - - - name: Wait for the Gateway to be listening - timeout-minutes: 3 - run: | - . .env - - echo using gateway $NVMEOF_IP_ADDRESS port $NVMEOF_GW_PORT - until nc -z $NVMEOF_IP_ADDRESS $NVMEOF_GW_PORT; do - echo -n . - sleep ${{ env.WAIT_INTERVAL_SECS }} - done - - - name: List containers - if: success() || failure() - run: make ps - - - name: List processes - if: success() || failure() - run: make top - - - name: Test - run: | - . .env - port2=`expr ${NVMEOF_IO_PORT} + 10` - make demosecure OPTS=-T HOSTNQN="${NQN}host" HOSTNQN2="${NQN}host2" NVMEOF_IO_PORT2=${port2} + ./tests/ha/demo_test.sh test_${{ matrix.security_protocol }} - name: List resources run: | # https://github.com/actions/toolkit/issues/766 shopt -s expand_aliases eval $(make alias) - cephnvmf get_subsystems cephnvmf subsystem list - subs=$(cephnvmf --output stdio --format json subsystem list | grep nqn | sed 's/"nqn": "//' | sed 's/",$//') + subs=$(cephnvmf --output stdio --format json subsystem list | jq -r '.subsystems[].nqn') for sub in $subs do cephnvmf namespace list --subsystem $sub @@ -455,70 +319,7 @@ jobs: - name: Run bdevperf run: | - # see https://spdk.io/doc/nvmf_multipath_howto.html - shopt -s expand_aliases - eval $(make alias) - . .env - set -x - echo -n "ℹ️ Starting bdevperf container" - docker compose up -d bdevperf - sleep 10 - echo "ℹ️ bdevperf start up logs" - make logs SVC=bdevperf - eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' ) - psk_path_prefix="/tmp/psk/" - psk_path="${psk_path_prefix}${NQN}" - mkdir -p ${psk_path} - echo -n "NVMeTLSkey-1:01:YzrPElk4OYy1uUERriPwiiyEJE/+J5ckYpLB+5NHMsR2iBuT:" > ${psk_path}/${NQN}host - chmod 600 ${psk_path}/${NQN}host - docker cp ${psk_path_prefix} ${BDEVPERF_CONTAINER_NAME}:${psk_path_prefix} - - rpc="/usr/libexec/spdk/scripts/rpc.py" - port2=`expr ${NVMEOF_IO_PORT} + 10` - echo "ℹ️ bdevperf bdev_nvme_set_options" - make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1" - echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: $NVMEOF_IO_PORT nqn: $NQN" - make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -f ipv4 -n $NQN -q ${NQN}host -l -1 -o 10 --psk ${psk_path}/${NQN}host" - echo "ℹ️ verify connection list" - conns=$(cephnvmf --output stdio --format json connection list --subsystem $NQN) - echo $conns | grep -q '"status": 0' - echo $conns | grep -q "\"nqn\": \"${NQN}host\"" - echo $conns | grep -q "\"trsvcid\": ${NVMEOF_IO_PORT}" - echo $conns | grep -q "\"traddr\": \"${NVMEOF_IP_ADDRESS}\"" - echo $conns | grep -q "\"adrfam\": \"ipv4\"" - echo $conns | grep -q "\"trtype\": \"TCP\"" - echo $conns | grep -q "\"qpairs_count\": 1" - echo $conns | grep -q "\"connected\": true" - echo $conns | grep -q "\"secure\": true" - echo $conns | grep -q -v "\"secure\": false" - echo $conns | grep -q "\"use_psk\": true" - echo $conns | grep -q "\"use_psk\": false" - con_cnt=$(echo $conns | xargs -n 2 | grep traddr | grep -v "n/a" | wc -l) - if [ $con_cnt -ne 1 ]; then - echo "Number of connections ${con_cnt}, expected 1, list: ${conns}" - exit 1 - fi - echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: ${port2} nqn: ${NQN}host2" - make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme1 -t tcp -a $NVMEOF_IP_ADDRESS -s ${port2} -f ipv4 -n $NQN -q "${NQN}host2" -l -1 -o 10" - echo "ℹ️ verify connection list again" - conns=$(cephnvmf --output stdio --format json connection list --subsystem $NQN) - con_cnt=$(echo $conns | xargs -n 2 | grep traddr | grep -v "n/a" | wc -l) - if [ $con_cnt -ne 2 ]; then - echo "Number of connections ${con_cnt}, expected 2, list: ${conns}" - exit 1 - fi - echo $conns | grep -q "\"nqn\": \"${NQN}host2\"" - echo $conns | grep -q "\"trsvcid\": ${port2}" - echo $conns | grep -q "\"secure\": true" - echo $conns | grep -q "\"secure\": false" - echo $conns | grep -q "\"use_psk\": true" - echo $conns | grep -q "\"use_psk\": false" - echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: ${port2} nqn: ${NQN}host2" - echo "ℹ️ bdevperf perform_tests" - eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' ) - timeout=$(expr $BDEVPERF_TEST_DURATION \* 2) - bdevperf="/usr/libexec/spdk/scripts/bdevperf.py" - make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests" + ./tests/ha/demo_test.sh bdevperf_${{ matrix.security_protocol }} - name: Check coredump existence if: success() || failure() @@ -886,7 +687,7 @@ jobs: push-images-to-ceph-registry: if: github.event_name == 'release' - needs: [pytest, demo, demo-secure, discovery, ha, atom, build-arm64] + needs: [pytest, demo, discovery, ha, atom, build-arm64] runs-on: ubuntu-latest steps: diff --git a/.gitmodules b/.gitmodules index f2611fc3..d39e8591 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "spdk"] path = spdk url = https://github.com/ceph/spdk.git - branch = ceph-nvmeof-v24.05 + branch = ceph-nvmeof-v24.05 diff --git a/Makefile b/Makefile index c3d90414..ca00c662 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,8 @@ endif # Includes include mk/containerized.mk include mk/demo.mk -include mk/demosecure.mk +include mk/demosecurepsk.mk +include mk/demosecuredhchap.mk include mk/misc.mk include mk/autohelp.mk diff --git a/control/cli.py b/control/cli.py index 217c4382..8c703c16 100644 --- a/control/cli.py +++ b/control/cli.py @@ -1056,14 +1056,28 @@ def host_add(self, args): if args.psk: if len(args.host_nqn) > 1: self.cli.parser.error(f"Can't have more than one host NQN when PSK keys are used") + if args.dhchap_key: + self.cli.parser.error(f"PSK and DH-HMAC-CHAP keys are mutually exclusive") + + if args.dhchap_key: + if len(args.host_nqn) > 1: + self.cli.parser.error(f"Can't have more than one host NQN when DH-HMAC-CHAP keys are used") + + if args.dhchap_ctrlr_key: + if not args.dhchap_key: + self.cli.parser.error(f"DH-HMAC-CHAP controller keys can not be used without DH-HMAC-CHAP keys") for i in range(len(args.host_nqn)): one_host_nqn = args.host_nqn[i] if one_host_nqn == "*" and args.psk: - self.cli.parser.error(f"PSK is only allowed for specific hosts") + self.cli.parser.error(f"PSK key is only allowed for specific hosts") + + if one_host_nqn == "*" and args.dhchap_key: + self.cli.parser.error(f"DH-HMAC-CHAP key is only allowed for specific hosts") - req = pb2.add_host_req(subsystem_nqn=args.subsystem, host_nqn=one_host_nqn, psk=args.psk) + req = pb2.add_host_req(subsystem_nqn=args.subsystem, host_nqn=one_host_nqn, + psk=args.psk, dhchap_key=args.dhchap_key, dhchap_ctrlr_key=args.dhchap_ctrlr_key) try: ret = self.stub.add_host(req) except Exception as ex: @@ -1173,14 +1187,15 @@ def host_list(self, args): hosts_list.append(["Any host", "n/a"]) for h in hosts_info.hosts: use_psk = "Yes" if h.use_psk else "No" - hosts_list.append([h.nqn, use_psk]) + use_dhchap = "Yes" if h.use_dhchap else "No" + hosts_list.append([h.nqn, use_psk, use_dhchap]) if len(hosts_list) > 0: if args.format == "text": table_format = "fancy_grid" else: table_format = "plain" hosts_out = tabulate(hosts_list, - headers = ["Host NQN", "Uses PSK"], + headers = ["Host NQN", "Uses PSK", "Uses DHCHAP"], tablefmt=table_format, stralign="center") out_func(f"Hosts allowed to access {args.subsystem}:\n{hosts_out}") else: @@ -1210,7 +1225,9 @@ def host_list(self, args): ] host_add_args = host_common_args + [ argument("--host-nqn", "-t", help="Host NQN list", nargs="+", required=True), - argument("--psk", help="Hosts PSK key list", required=False), + argument("--psk", help="Hosts PSK key", required=False), + argument("--dhchap-key", help="Host DH-HMAC-CHAP key", required=False), + argument("--dhchap-ctrlr-key", help="Host DH-HMAC-CHAP controller key", required=False), ] host_del_args = host_common_args + [ argument("--host-nqn", "-t", help="Host NQN list", nargs="+", required=True), @@ -1251,6 +1268,7 @@ def connection_list(self, args): for conn in connections_info.connections: conn_secure = "" conn_psk = "Yes" if conn.use_psk else "No" + conn_dhchap = "Yes" if conn.use_dhchap else "No" if conn.connected: conn_secure = "Yes" if conn.secure else "No" connections_list.append([conn.nqn, @@ -1259,14 +1277,15 @@ def connection_list(self, args): conn.qpairs_count if conn.connected else "", conn.controller_id if conn.connected else "", conn_secure, - conn_psk]) + conn_psk, + conn_dhchap]) if len(connections_list) > 0: if args.format == "text": table_format = "fancy_grid" else: table_format = "plain" connections_out = tabulate(connections_list, - headers = ["Host NQN", "Address", "Connected", "QPairs Count", "Controller ID", "Secure", "PSK"], + headers = ["Host NQN", "Address", "Connected", "QPairs Count", "Controller ID", "Secure", "Uses\nPSK", "Uses\nDHCHAP"], tablefmt=table_format) out_func(f"Connections for {args.subsystem}:\n{connections_out}") else: diff --git a/control/grpc.py b/control/grpc.py index 5a44bace..52e559ff 100644 --- a/control/grpc.py +++ b/control/grpc.py @@ -17,12 +17,17 @@ import contextlib import threading import time +import hashlib +import tempfile +from pathlib import Path from typing import Callable from collections import defaultdict import logging +import shutil import spdk.rpc.bdev as rpc_bdev import spdk.rpc.nvmf as rpc_nvmf +import spdk.rpc.keyring as rpc_keyring import spdk.rpc.log as rpc_log from spdk.rpc.client import JSONRPCException @@ -58,12 +63,16 @@ def group_id(self, request: monitor_pb2.group_id_req, context = None) -> Empty: return Empty() class SubsystemHostAuth: + MAX_PSK_KEY_NAME_LENGTH = 200 # taken from SPDK SPDK_TLS_PSK_MAX_LEN + def __init__(self): self.subsys_allow_any_hosts = defaultdict(dict) self.host_has_psk = defaultdict(dict) + self.host_has_dhchap = defaultdict(dict) def clean_subsystem(self, subsys): self.host_has_psk.pop(subsys, None) + self.host_has_dhchap.pop(subsys, None) self.subsys_allow_any_hosts.pop(subsys, None) def add_psk_host(self, subsys, host): @@ -83,6 +92,23 @@ def is_psk_host(self, subsys, host = None) -> bool: return True return False + def add_dhchap_host(self, subsys, host): + self.host_has_dhchap[subsys][host] = True + + def remove_dhchap_host(self, subsys, host): + if subsys in self.host_has_dhchap: + self.host_has_dhchap[subsys].pop(host, None) + if len(self.host_has_dhchap[subsys]) == 0: + self.host_has_dhchap.pop(subsys, None) # last host was removed from subsystem + + def is_dhchap_host(self, subsys, host = None) -> bool: + if subsys in self.host_has_dhchap: + if not host: + return len(self.host_has_dhchap[subsys]) != 0 + if host in self.host_has_dhchap[subsys]: + return True + return False + def allow_any_host(self, subsys): self.subsys_allow_any_hosts[subsys] = True @@ -158,6 +184,11 @@ class GatewayService(pb2_grpc.GatewayServicer): cluster_nonce: cluster context nonce map """ + PSK_PREFIX = "psk" + DHCHAP_PREFIX = "dhchap" + DHCHAP_CONTROLLER_PREFIX = "dhchap_ctrlr" + KEYS_DIR = "/var/tmp" + def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler, rpc_lock, omap_lock: OmapLock, group_id: int, spdk_rpc_client, spdk_rpc_subsystems_client, ceph_utils: CephUtils) -> None: """Constructor""" self.gw_logger_object = GatewayLogger(config) @@ -236,41 +267,164 @@ def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler, rp self.subsys_max_ns = {} self.host_info = SubsystemHostAuth() - def create_host_psk_file(self, subsysnqn : str, hostnqn : str, psk_value : str) -> str: + def get_directories_for_key_file(self, key_type : str, subsysnqn : str, create_dir : bool = False) -> []: + tmp_dirs = [] + dir_prefix = f"{key_type}_{subsysnqn}_" + + try: + for f in Path(self.KEYS_DIR).iterdir(): + if f.is_dir() and f.match(dir_prefix + "*"): + tmp_dirs.insert(0, str(f)) + except Exception: + self.logger.exception(f"Error listing files in {self.KEYS_DIR}") + return None + + if tmp_dirs: + return tmp_dirs + + if not create_dir: + return None + + tmp_dir_name = None + try: + tmp_dir_name = tempfile.mkdtemp(prefix=dir_prefix, dir=self.KEYS_DIR) + except Exception: + self.logger.exception("Error creating directory for key file") + return None + return [tmp_dir_name] + + def create_host_key_file(self, key_type : str, subsysnqn : str, hostnqn : str, key_value : str) -> str: assert subsysnqn, "Subsystem NQN can't be empty" assert hostnqn, "Host NQN can't be empty" - assert psk_value, "PSK value can't be empty" + assert key_type, "Key type can't be empty" + assert key_value, "Key value can't be empty" + + tmp_dir_names = self.get_directories_for_key_file(key_type, subsysnqn, create_dir = True) + if not tmp_dir_names: + return None - psk_dir = f"/tmp/psk/{subsysnqn}" - psk_file = f"{psk_dir}/{hostnqn}" + file = None + filepath = None + keyfile_prefix = f"{hostnqn}_" try: - os.makedirs(psk_dir, 0o755, True) + (file_fd, filepath) = tempfile.mkstemp(prefix=keyfile_prefix, dir=tmp_dir_names[0], text=True) except Exception: - self.logger.exception(f"Error creating directory {psk_dir}") + self.logger.exception("Error creating key file") + return None + if not filepath: + self.loger.error("Error creating key file") return None try: - with open(psk_file, 'wt') as f: - print(psk_value, end="", file=f) - os.chmod(psk_file, 0o600) + with open(file_fd, "wt") as f: + f.write(key_value) except Exception: - self.logger.exception(f"Error creating file {psk_file}") + self.logger.exception(f"Error creating file") + try: + os.remove(filepath) + except Exception: + pass return None - return psk_file + return filepath + + def create_host_psk_file(self, subsysnqn : str, hostnqn : str, key_value : str) -> str: + return self.create_host_key_file(self.PSK_PREFIX, subsysnqn, hostnqn, key_value) + + def create_host_dhchap_file(self, subsysnqn : str, hostnqn : str, key_value : str) -> str: + return self.create_host_key_file(self.DHCHAP_PREFIX, subsysnqn, hostnqn, key_value) + + def remove_host_key_file(self, key_type : str, subsysnqn : str, hostnqn : str) -> None: + assert key_type, "Key type can't be empty" + assert subsysnqn, "Subsystem NQN can't be empty" + + tmp_dir_names = self.get_directories_for_key_file(key_type, subsysnqn, create_dir = False) + if not tmp_dir_names: + return + + # If there is no host NQN remove all hosts in this subsystem + if not hostnqn: + for one_tmp_dir in tmp_dir_names: + try: + shutil.rmtree(one_tmp_dir, ignore_errors = True) + except Exception: + pass + return + + # We have a host NQN so only remove its files + for one_tmp_dir in tmp_dir_names: + for f in Path(one_tmp_dir).iterdir(): + if f.is_file() and f.match(f"{hostnqn}_*"): + try: + f.unlink() + except Exception: + self.logger.exception(f"Error deleting file {f.name}") + pass def remove_host_psk_file(self, subsysnqn : str, hostnqn : str) -> None: - psk_dir = f"/tmp/psk/{subsysnqn}" - psk_file = f"{psk_dir}/{hostnqn}" + self.remove_host_key_file(self.PSK_PREFIX, subsysnqn, hostnqn) + + def remove_host_dhchap_file(self, subsysnqn : str, hostnqn : str) -> None: + self.remove_host_key_file(self.DHCHAP_PREFIX, subsysnqn, hostnqn) + + def remove_all_host_key_files(self, subsysnqn : str, hostnqn : str) -> None: + self.remove_host_psk_file(subsysnqn, hostnqn) + self.remove_host_dhchap_file(subsysnqn, hostnqn) + + def remove_all_subsystem_key_files(self, subsysnqn : str) -> None: + self.remove_all_host_key_files(subsysnqn, None) + + @staticmethod + def construct_key_name_for_keyring(subsysnqn : str, hostnqn : str, prefix : str = None) -> str: + key_name = hashlib.sha256(subsysnqn.encode()).hexdigest() + "_" + hashlib.sha256(hostnqn.encode()).hexdigest() + if prefix: + key_name = prefix + "_" + key_name + return key_name + + def remove_key_from_keyring(self, key_type : str, subsysnqn : str, hostnqn : str) -> None: + assert self.rpc_lock.locked(), "RPC is unlocked when calling remove_key_from_keyring()" + key_name = GatewayService.construct_key_name_for_keyring(subsysnqn, hostnqn, key_type) try: - os.remove(psk_file) - except FileNotFoundError: - self.logger.exception(f"Error deleting file {psk_file}") + rpc_keyring.keyring_file_remove_key(self.spdk_rpc_client, key_name) + except Exception: pass + + def remove_psk_key_from_keyring(self, subsysnqn : str, hostnqn : str) -> None: + self.remove_key_from_keyring(self.PSK_PREFIX, subsysnqn, hostnqn) + + def remove_dhchap_key_from_keyring(self, subsysnqn : str, hostnqn : str) -> None: + self.remove_key_from_keyring(self.DHCHAP_PREFIX, subsysnqn, hostnqn) + + def remove_dhchap_controller_key_from_keyring(self, subsysnqn : str, hostnqn : str) -> None: + self.remove_key_from_keyring(self.DHCHAP_CONTROLLER_PREFIX, subsysnqn, hostnqn) + + def remove_all_host_keys_from_keyring(self, subsysnqn : str, hostnqn : str) -> None: + self.remove_psk_key_from_keyring(subsysnqn, hostnqn) + self.remove_dhchap_key_from_keyring(subsysnqn, hostnqn) + self.remove_dhchap_controller_key_from_keyring(subsysnqn, hostnqn) + + def remove_all_subsystem_keys_from_keyring(self, subsysnqn : str) -> None: + assert self.rpc_lock.locked(), "RPC is unlocked when calling remove_all_subsystem_keys_from_keyring()" try: - os.rmdir(psk_dir) - os.rmdir("/tmp/psk") + key_list = rpc_keyring.keyring_get_keys(self.spdk_rpc_client) except Exception: - self.logger.exception(f"Error deleting directory {psk_dir}") - pass + self.logger.exception("Can't list keyring keys") + return + for one_key in key_list: + key_path = None + key_name = None + try: + key_path = one_key["path"] + key_name = one_key["name"] + except Exception: + self.logger.exception(f"Can't get details for key {one_key}") + continue + if not key_name or not key_path: + continue + if (key_path.startswith(f"{self.KEYS_DIR}/{self.PSK_PREFIX}_{subsysnqn}_") or + key_path.startswith(f"{self.KEYS_DIR}/{self.DHCHAP_PREFIX}_{subsysnqn}_")): + try: + rpc_keyring.keyring_file_remove_key(self.spdk_rpc_client, key_name) + except Exception: + pass @staticmethod def is_valid_host_nqn(nqn): @@ -810,6 +964,8 @@ def delete_subsystem_safe(self, request, context): self.subsystem_listeners.pop(request.subsystem_nqn, None) self.host_info.clean_subsystem(request.subsystem_nqn) self.subsystem_nsid_bdev_and_uuid.remove_namespace(request.subsystem_nqn) + self.remove_all_subsystem_key_files(request.subsystem_nqn) + self.remove_all_subsystem_keys_from_keyring(request.subsystem_nqn) self.logger.debug(f"delete_subsystem {request.subsystem_nqn}: {ret}") except Exception as ex: self.logger.exception(delete_subsystem_error_prefix) @@ -1826,6 +1982,21 @@ def add_host_safe(self, request, context): self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) + if request.dhchap_key and request.host_nqn == "*": + errmsg=f"{host_failure_prefix}: DH-HMAC-CHAP key is only allowed for specific hosts" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) + + if request.dhchap_ctrlr_key and not request.dhchap_key: + errmsg=f"{host_failure_prefix}: DH-HMAC-CHAP controller key can only be used with a DH-HMAC-CHAP key" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) + + if request.psk and request.dhchap_key: + errmsg=f"{host_failure_prefix}: PSK and DH-HMAC-CHAP keys are mutually exclusive" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) + host_already_exist = self.matching_host_exists(context, request.subsystem_nqn, request.host_nqn) if host_already_exist: if request.host_nqn == "*": @@ -1838,12 +2009,44 @@ def add_host_safe(self, request, context): return pb2.req_status(status=errno.EEXIST, error_message=errmsg) psk_file = None + psk_key_name = None if request.psk: psk_file = self.create_host_psk_file(request.subsystem_nqn, request.host_nqn, request.psk) if not psk_file: errmsg=f"{host_failure_prefix}: Can't write PSK file" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.ENOENT, error_message=errmsg) + psk_key_name = GatewayService.construct_key_name_for_keyring(request.subsystem_nqn, request.host_nqn, GatewayService.PSK_PREFIX) + if len(psk_key_name) >= SubsystemHostAuth.MAX_PSK_KEY_NAME_LENGTH: + errmsg=f"{host_failure_prefix}: PSK key name {psk_key_name} is too long, max length is {SubsystemHostAuth.MAX_PSK_KEY_NAME_LENGTH}" + self.logger.error(f"{errmsg}") + return pb2.req_status(status=errno.E2BIG, error_message=errmsg) + + dhchap_file = None + dhchap_key_name = None + if request.dhchap_key: + dhchap_file = self.create_host_dhchap_file(request.subsystem_nqn, request.host_nqn, request.dhchap_key) + if not dhchap_file: + errmsg=f"{host_failure_prefix}: Can't write DH-HMAC-CHAP file" + self.logger.error(f"{errmsg}") + if psk_file: + self.remove_host_psk_file(request.subsystem_nqn, request.host_nqn) + return pb2.req_status(status=errno.ENOENT, error_message=errmsg) + dhchap_key_name = GatewayService.construct_key_name_for_keyring(request.subsystem_nqn, request.host_nqn, GatewayService.DHCHAP_PREFIX) + + dhchap_ctrlr_file = None + dhchap_ctrlr_key_name = None + if request.dhchap_ctrlr_key: + dhchap_ctrlr_file = self.create_host_dhchap_file(request.subsystem_nqn, request.host_nqn, request.dhchap_ctrlr_key) + if not dhchap_ctrlr_file: + errmsg=f"{host_failure_prefix}: Can't write DH-HMAC-CHAP controller file" + self.logger.error(f"{errmsg}") + if psk_file: + self.remove_host_psk_file(request.subsystem_nqn, request.host_nqn) + if dhchap_file: + self.remove_host_dhchap_file(request.subsystem_nqn, request.host_nqn) + return pb2.req_status(status=errno.ENOENT, error_message=errmsg) + dhchap_ctrlr_key_name = GatewayService.construct_key_name_for_keyring(request.subsystem_nqn, request.host_nqn, GatewayService.DHCHAP_CONTROLLER_PREFIX) omap_lock = self.omap_lock.get_omap_lock_to_use(context) with omap_lock: @@ -1859,24 +2062,56 @@ def add_host_safe(self, request, context): self.host_info.allow_any_host(request.subsystem_nqn) else: # Allow single host access to subsystem self.logger.info( - f"Received request to add host {request.host_nqn} to {request.subsystem_nqn}, psk: {request.psk}, context: {context}{peer_msg}") + f"Received request to add host {request.host_nqn} to {request.subsystem_nqn}, psk: {request.psk}, dhchap: {request.dhchap_key}, dhchap controller: {request.dhchap_ctrlr_key}, context: {context}{peer_msg}") + if psk_file: + try: + rpc_keyring.keyring_file_remove_key(self.spdk_rpc_client, psk_key_name) + except Exception: + pass + ret = rpc_keyring.keyring_file_add_key(self.spdk_rpc_client, psk_key_name, psk_file) + self.logger.debug(f"keyring_file_add_key {psk_key_name}: {ret}") + self.logger.info(f"Added PSK key {psk_key_name} to keyring") + if dhchap_file: + try: + rpc_keyring.keyring_file_remove_key(self.spdk_rpc_client, dhchap_key_name) + except Exception: + pass + ret = rpc_keyring.keyring_file_add_key(self.spdk_rpc_client, dhchap_key_name, dhchap_file) + self.logger.debug(f"keyring_file_add_key {dhchap_key_name}: {ret}") + self.logger.info(f"Added DH-HMAC-CHAP key {dhchap_key_name} to keyring") + if dhchap_ctrlr_file: + try: + rpc_keyring.keyring_file_remove_key(self.spdk_rpc_client, dhchap_ctrlr_key_name) + except Exception: + pass + ret = rpc_keyring.keyring_file_add_key(self.spdk_rpc_client, dhchap_ctrlr_key_name, dhchap_ctrlr_file) + self.logger.debug(f"keyring_file_add_key {dhchap_ctrlr_key_name}: {ret}") + self.logger.info(f"Added DH-HMAC-CHAP controller key {dhchap_ctrlr_key_name} to keyring") ret = rpc_nvmf.nvmf_subsystem_add_host( self.spdk_rpc_client, nqn=request.subsystem_nqn, host=request.host_nqn, - psk=psk_file, + psk=psk_key_name, + dhchap_key=dhchap_key_name, + dhchap_ctrlr_key=dhchap_ctrlr_key_name, ) self.logger.debug(f"add_host {request.host_nqn}: {ret}") if psk_file: self.host_info.add_psk_host(request.subsystem_nqn, request.host_nqn) self.remove_host_psk_file(request.subsystem_nqn, request.host_nqn) + try: + rpc_keyring.keyring_file_remove_key(self.spdk_rpc_client, psk_key_name) + except Exception: + pass + if dhchap_file: + self.host_info.add_dhchap_host(request.subsystem_nqn, request.host_nqn) except Exception as ex: if request.host_nqn == "*": self.logger.exception(all_host_failure_prefix) errmsg = f"{all_host_failure_prefix}:\n{ex}" else: - if psk_file: - self.remove_host_psk_file(request.subsystem_nqn, request.host_nqn) + self.remove_all_host_key_files(request.subsystem_nqn, request.host_nqn) + self.remove_all_host_keys_from_keyring(request.subsystem_nqn, request.host_nqn) self.logger.exception(host_failure_prefix) errmsg = f"{host_failure_prefix}:\n{ex}" resp = self.parse_json_exeption(ex) @@ -1895,6 +2130,8 @@ def add_host_safe(self, request, context): errmsg = all_host_failure_prefix else: errmsg = host_failure_prefix + self.remove_all_host_key_files(request.subsystem_nqn, request.host_nqn) + self.remove_all_host_keys_from_keyring(request.subsystem_nqn, request.host_nqn) self.logger.error(errmsg) return pb2.req_status(status=errno.EINVAL, error_message=errmsg) @@ -1908,6 +2145,8 @@ def add_host_safe(self, request, context): errmsg = f"Error persisting host {request.host_nqn} access addition" self.logger.exception(errmsg) errmsg = f"{errmsg}:\n{ex}" + self.remove_all_host_key_files(request.subsystem_nqn, request.host_nqn) + self.remove_all_host_keys_from_keyring(request.subsystem_nqn, request.host_nqn) return pb2.req_status(status=errno.EINVAL, error_message=errmsg) return pb2.req_status(status=0, error_message=os.strerror(0)) @@ -1986,6 +2225,9 @@ def remove_host_safe(self, request, context): ) self.logger.debug(f"remove_host {request.host_nqn}: {ret}") self.host_info.remove_psk_host(request.subsystem_nqn, request.host_nqn) + self.host_info.remove_dhchap_host(request.subsystem_nqn, request.host_nqn) + self.remove_all_host_key_files(request.subsystem_nqn, request.host_nqn) + self.remove_all_host_keys_from_keyring(request.subsystem_nqn, request.host_nqn) except Exception as ex: if request.host_nqn == "*": self.logger.exception(all_host_failure_prefix) @@ -2055,7 +2297,8 @@ def list_hosts_safe(self, request, context): for h in host_nqns: host_nqn = h["nqn"] psk = self.host_info.is_psk_host(request.subsystem, host_nqn) - one_host = pb2.host(nqn = host_nqn, use_psk = psk) + dhchap = self.host_info.is_dhchap_host(request.subsystem, host_nqn) + one_host = pb2.host(nqn = host_nqn, use_psk = psk, use_dhchap = dhchap) hosts.append(one_host) break except Exception: @@ -2155,6 +2398,7 @@ def list_connections_safe(self, request, context): found = False secure = False psk = False + dhchap = False for qp in qpair_ret: try: @@ -2189,6 +2433,7 @@ def list_connections_safe(self, request, context): continue psk = self.host_info.is_psk_host(request.subsystem, hostnqn) + dhchap = self.host_info.is_dhchap_host(request.subsystem, hostnqn) if request.subsystem in self.subsystem_listeners: if (adrfam, traddr, trsvcid, True) in self.subsystem_listeners[request.subsystem]: @@ -2201,7 +2446,7 @@ def list_connections_safe(self, request, context): one_conn = pb2.connection(nqn=hostnqn, connected=True, traddr=traddr, trsvcid=trsvcid, trtype=trtype, adrfam=adrfam, qpairs_count=conn["num_io_qpairs"], controller_id=conn["cntlid"], - secure=secure, use_psk=psk) + secure=secure, use_psk=psk, use_dhchap=dhchap) connections.append(one_conn) if hostnqn in host_nqns: host_nqns.remove(hostnqn) @@ -2211,9 +2456,11 @@ def list_connections_safe(self, request, context): for nqn in host_nqns: psk = False + dhchap = False psk = self.host_info.is_psk_host(request.subsystem, nqn) + dhchap = self.host_info.is_dhchap_host(request.subsystem, nqn) one_conn = pb2.connection(nqn=nqn, connected=False, traddr="", trsvcid=0, - qpairs_count=-1, controller_id=-1, use_psk=psk) + qpairs_count=-1, controller_id=-1, use_psk=psk, use_dhchap=dhchap) connections.append(one_conn) return pb2.connections_info(status = 0, error_message = os.strerror(0), diff --git a/control/proto/gateway.proto b/control/proto/gateway.proto index f57cd305..2eecf720 100644 --- a/control/proto/gateway.proto +++ b/control/proto/gateway.proto @@ -183,6 +183,8 @@ message add_host_req { string subsystem_nqn = 1; string host_nqn = 2; optional string psk = 3; + optional string dhchap_key = 4; + optional string dhchap_ctrlr_key = 5; } message remove_host_req { @@ -400,6 +402,7 @@ message listeners_info { message host { string nqn = 1; optional bool use_psk = 2; + optional bool use_dhchap = 3; } message hosts_info { @@ -421,6 +424,7 @@ message connection { int32 controller_id = 8; optional bool secure = 9; optional bool use_psk = 10; + optional bool use_dhchap = 11; } message connections_info { diff --git a/mk/demosecuredhchap.mk b/mk/demosecuredhchap.mk new file mode 100644 index 00000000..fafd5422 --- /dev/null +++ b/mk/demosecuredhchap.mk @@ -0,0 +1,22 @@ +## Demo secure DHCHAP: + +HOSTNQN=`cat /etc/nvme/hostnqn` +HOSTNQN2=`cat /etc/nvme/hostnqn | sed 's/......$$/ffffff/'` +HOSTNQN3=`cat /etc/nvme/hostnqn | sed 's/......$$/fffffe/'` +NVMEOF_IO_PORT2=`expr $(NVMEOF_IO_PORT) + 1` +NVMEOF_IO_PORT3=`expr $(NVMEOF_IO_PORT) + 2` +DHCHAPKEY1=$(DHCHAP_KEY1) +DHCHAPKEY2=$(DHCHAP_KEY2) +DHCHAPKEY3=$(DHCHAP_KEY3) +# demosecuredhchap +demosecuredhchap: + $(NVMEOF_CLI) subsystem add --subsystem $(NQN) --no-group-append + $(NVMEOF_CLI) namespace add --subsystem $(NQN) --rbd-pool $(RBD_POOL) --rbd-image $(RBD_IMAGE_NAME) --size $(RBD_IMAGE_SIZE) --rbd-create-image + $(NVMEOF_CLI) listener add --subsystem $(NQN) --host-name `docker ps -q -f name=$(NVMEOF_CONTAINER_NAME)` --traddr $(NVMEOF_IP_ADDRESS) --trsvcid $(NVMEOF_IO_PORT) + $(NVMEOF_CLI) listener add --subsystem $(NQN) --host-name `docker ps -q -f name=$(NVMEOF_CONTAINER_NAME)` --traddr $(NVMEOF_IP_ADDRESS) --trsvcid $(NVMEOF_IO_PORT2) + $(NVMEOF_CLI) listener add --subsystem $(NQN) --host-name `docker ps -q -f name=$(NVMEOF_CONTAINER_NAME)` --traddr $(NVMEOF_IP_ADDRESS) --trsvcid $(NVMEOF_IO_PORT3) + $(NVMEOF_CLI) host add --subsystem $(NQN) --host-nqn $(HOSTNQN) --dhchap-key ${DHCHAPKEY1} + $(NVMEOF_CLI) host add --subsystem $(NQN) --host-nqn $(HOSTNQN2) --dhchap-key ${DHCHAPKEY2} --dhchap-ctrlr-key ${DHCHAPKEY3} + $(NVMEOF_CLI) host add --subsystem $(NQN) --host-nqn $(HOSTNQN3) + +.PHONY: demosecuredhchap diff --git a/mk/demosecure.mk b/mk/demosecurepsk.mk similarity index 74% rename from mk/demosecure.mk rename to mk/demosecurepsk.mk index 9c01f052..ccb19046 100644 --- a/mk/demosecure.mk +++ b/mk/demosecurepsk.mk @@ -1,15 +1,19 @@ -## Demo secure: +## Demo secure PSK: HOSTNQN=`cat /etc/nvme/hostnqn` HOSTNQN2=`cat /etc/nvme/hostnqn | sed 's/......$$/ffffff/'` +HOSTNQN3=`cat /etc/nvme/hostnqn | sed 's/......$$/fffffe/'` NVMEOF_IO_PORT2=`expr $(NVMEOF_IO_PORT) + 1` -# demosecure -demosecure: +PSKKEY1=$(PSK_KEY1) +PSKKEY2=$(PSK_KEY2) +# demosecure-psk +demosecurepsk: $(NVMEOF_CLI) subsystem add --subsystem $(NQN) --no-group-append $(NVMEOF_CLI) namespace add --subsystem $(NQN) --rbd-pool $(RBD_POOL) --rbd-image $(RBD_IMAGE_NAME) --size $(RBD_IMAGE_SIZE) --rbd-create-image $(NVMEOF_CLI) listener add --subsystem $(NQN) --host-name `docker ps -q -f name=$(NVMEOF_CONTAINER_NAME)` --traddr $(NVMEOF_IP_ADDRESS) --trsvcid $(NVMEOF_IO_PORT) --secure $(NVMEOF_CLI) listener add --subsystem $(NQN) --host-name `docker ps -q -f name=$(NVMEOF_CONTAINER_NAME)` --traddr $(NVMEOF_IP_ADDRESS) --trsvcid $(NVMEOF_IO_PORT2) - $(NVMEOF_CLI) host add --subsystem $(NQN) --host-nqn "$(HOSTNQN)" --psk "NVMeTLSkey-1:01:YzrPElk4OYy1uUERriPwiiyEJE/+J5ckYpLB+5NHMsR2iBuT:" + $(NVMEOF_CLI) host add --subsystem $(NQN) --host-nqn "$(HOSTNQN)" --psk ${PSKKEY1} $(NVMEOF_CLI) host add --subsystem $(NQN) --host-nqn "$(HOSTNQN2)" + $(NVMEOF_CLI) host add --subsystem $(NQN) --host-nqn "$(HOSTNQN3)" --psk ${PSKKEY2} -.PHONY: demosecure +.PHONY: demosecurepsk diff --git a/tests/ha/demo_test.sh b/tests/ha/demo_test.sh new file mode 100755 index 00000000..2ce4fed5 --- /dev/null +++ b/tests/ha/demo_test.sh @@ -0,0 +1,420 @@ +#!/bin/bash + +function cephnvmf_func() +{ + /usr/bin/docker compose run --rm nvmeof-cli --server-address ${NVMEOF_IP_ADDRESS} --server-port ${NVMEOF_GW_PORT} $@ +} + +function demo_test_unsecured() +{ + make demo OPTS=-T + return $? +} + +function demo_test_psk() +{ + psk_path_prefix="/tmp/psk/" + psk_path="${psk_path_prefix}${NQN}" + rm -rf /tmp/temp-psk + mkdir -p /tmp/temp-psk/psk/${NQN} + echo -n ${PSK_KEY1} > /tmp/temp-psk/psk/${NQN}/${NQN}host + echo -n ${PSK_KEY3} > /tmp/temp-psk/psk/${NQN}/${NQN}host3 + chmod 0600 /tmp/temp-psk/psk/${NQN}/${NQN}host + chmod 0600 /tmp/temp-psk/psk/${NQN}/${NQN}host3 + make demosecurepsk OPTS=-T HOSTNQN="${NQN}host" HOSTNQN2="${NQN}host2" HOSTNQN3="${NQN}host3" NVMEOF_IO_PORT2=${port2} PSKKEY2=${PSK_KEY3} + + echo "ℹ️ verify PSK key files removal" + psk_key_list=`make -s exec SVC=nvmeof OPTS=-T CMD="/usr/local/bin/spdk_rpc -s /var/tmp/spdk.sock keyring_get_keys"` + [[ `echo $psk_key_list | jq -r '.[0].removed'` == "true" ]] + [[ `echo $psk_key_list | jq -r '.[1].removed'` == "true" ]] + [[ `echo $psk_key_list | jq -r '.[2].removed'` == "null" ]] + set +e + make -s exec SVC=nvmeof OPTS=-T CMD="ls -lR /var/tmp/psk_${NQN}_*" + if [[ $? -eq 0 ]]; then + echo "PSK key files should be deleted" + exit 1 + fi + set -e + return 0 +} + +function demo_test_dhchap() +{ + rm -rf /tmp/temp-dhchap + mkdir -p /tmp/temp-dhchap/dhchap/${NQN} + echo -n "${DHCHAP_KEY4}" > /tmp/temp-dhchap/dhchap/${NQN}/key1 + echo -n "${DHCHAP_KEY5}" > /tmp/temp-dhchap/dhchap/${NQN}/key2 + echo -n "${DHCHAP_KEY6}" > /tmp/temp-dhchap/dhchap/${NQN}/key3 + chmod 0600 /tmp/temp-dhchap/dhchap/${NQN}/key1 + chmod 0600 /tmp/temp-dhchap/dhchap/${NQN}/key2 + chmod 0600 /tmp/temp-dhchap/dhchap/${NQN}/key3 + + make demosecuredhchap OPTS=-T HOSTNQN="${NQN}host" HOSTNQN2="${NQN}host2" HOSTNQN3="${NQN}host3" NVMEOF_IO_PORT2=${port2} NVMEOF_IO_PORT3=${port3} DHCHAPKEY1="${DHCHAP_KEY4}" DHCHAPKEY2="${DHCHAP_KEY5}" DHCHAPKEY3="${DHCHAP_KEY6}" +} + +function demo_bdevperf_unsecured() +{ + echo -n "ℹ️ Starting bdevperf container" + docker compose up -d bdevperf + sleep 10 + echo "ℹ️ bdevperf start up logs" + make logs SVC=bdevperf + eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' ) + + echo "ℹ️ bdevperf bdev_nvme_set_options" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1" + echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: $NVMEOF_IO_PORT nqn: $NQN" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -f ipv4 -n $NQN -q ${NQN}host -l -1 -o 10" + + echo "ℹ️ verify connection list" + conns=$(cephnvmf_func --output stdio --format json connection list --subsystem $NQN) + [[ `echo $conns | jq -r '.status'` == "0" ]] + [[ `echo $conns | jq -r '.subsystem_nqn'` == "${NQN}" ]] + [[ `echo $conns | jq -r '.connections[0].nqn'` == "${NQN}host" ]] + [[ `echo $conns | jq -r '.connections[0].trsvcid'` == "${NVMEOF_IO_PORT}" ]] + [[ `echo $conns | jq -r '.connections[0].traddr'` == "${NVMEOF_IP_ADDRESS}" ]] + [[ `echo $conns | jq -r '.connections[0].adrfam'` == "ipv4" ]] + [[ `echo $conns | jq -r '.connections[0].trtype'` == "TCP" ]] + [[ `echo $conns | jq -r '.connections[0].connected'` == "true" ]] + [[ `echo $conns | jq -r '.connections[0].qpairs_count'` == "1" ]] + [[ `echo $conns | jq -r '.connections[1]'` == "null" ]] + + echo "ℹ️ bdevperf perform_tests" + eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' ) + timeout=$(expr $BDEVPERF_TEST_DURATION \* 2) + bdevperf="/usr/libexec/spdk/scripts/bdevperf.py" + make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests" + return $? +} + +function demo_bdevperf_psk() +{ + cephnvmf_func spdk_log_level set --level debug --print debug + echo -n "ℹ️ Starting bdevperf container" + docker compose up -d bdevperf + sleep 10 + echo "ℹ️ bdevperf start up logs" + make logs SVC=bdevperf + eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' ) + psk_path_prefix="/tmp/psk/" + psk_path="${psk_path_prefix}${NQN}" + docker cp /tmp/temp-psk/psk ${BDEVPERF_CONTAINER_NAME}:`dirname ${psk_path_prefix}` + make exec SVC=bdevperf OPTS=-T CMD="chown -R root:root ${psk_path_prefix}" + rm -rf /tmp/temp-psk + + echo "ℹ️ bdevperf bdev_nvme_set_options" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1" + + echo "ℹ️ bdevperf add PSK key name key1 to keyring" + make -s exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET keyring_file_add_key key1 ${psk_path}/${NQN}host" + echo "ℹ️ bdevperf add PSK key name key2 to keyring" + make -s exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET keyring_file_add_key key2 ${psk_path}/${NQN}host3" + echo "ℹ️ bdevperf list keyring" + make -s exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET keyring_get_keys" + + echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: $NVMEOF_IO_PORT nqn: $NQN using PSK" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -f ipv4 -n $NQN -q ${NQN}host -l -1 -o 10 --psk key1" + + echo "ℹ️ verify connection list" + conns=$(cephnvmf_func --output stdio --format json connection list --subsystem $NQN) + [[ `echo $conns | jq -r '.status'` == "0" ]] + [[ `echo $conns | jq -r '.subsystem_nqn'` == "${NQN}" ]] + + [[ `echo $conns | jq -r '.connections[0].nqn'` == "${NQN}host" ]] + [[ `echo $conns | jq -r '.connections[0].trsvcid'` == "${NVMEOF_IO_PORT}" ]] + [[ `echo $conns | jq -r '.connections[0].traddr'` == "${NVMEOF_IP_ADDRESS}" ]] + [[ `echo $conns | jq -r '.connections[0].adrfam'` == "ipv4" ]] + [[ `echo $conns | jq -r '.connections[0].trtype'` == "TCP" ]] + [[ `echo $conns | jq -r '.connections[0].qpairs_count'` == "1" ]] + [[ `echo $conns | jq -r '.connections[0].controller_id'` == "1" ]] + [[ `echo $conns | jq -r '.connections[0].connected'` == "true" ]] + [[ `echo $conns | jq -r '.connections[0].secure'` == "true" ]] + [[ `echo $conns | jq -r '.connections[0].use_psk'` == "true" ]] + [[ `echo $conns | jq -r '.connections[0].use_dhchap'` == "false" ]] + + [[ `echo $conns | jq -r '.connections[1].nqn'` == "${NQN}host3" ]] + [[ `echo $conns | jq -r '.connections[1].trsvcid'` == "0" ]] + [[ `echo $conns | jq -r '.connections[1].traddr'` == "" ]] + [[ `echo $conns | jq -r '.connections[1].adrfam'` == "ipv4" ]] + [[ `echo $conns | jq -r '.connections[1].trtype'` == "" ]] + [[ `echo $conns | jq -r '.connections[1].qpairs_count'` == "-1" ]] + [[ `echo $conns | jq -r '.connections[1].controller_id'` == "-1" ]] + [[ `echo $conns | jq -r '.connections[1].connected'` == "false" ]] + [[ `echo $conns | jq -r '.connections[1].use_psk'` == "true" ]] + [[ `echo $conns | jq -r '.connections[1].use_dhchap'` == "false" ]] + + [[ `echo $conns | jq -r '.connections[2].nqn'` == "${NQN}host2" ]] + [[ `echo $conns | jq -r '.connections[2].trsvcid'` == "0" ]] + [[ `echo $conns | jq -r '.connections[2].traddr'` == "" ]] + [[ `echo $conns | jq -r '.connections[2].adrfam'` == "ipv4" ]] + [[ `echo $conns | jq -r '.connections[2].trtype'` == "" ]] + [[ `echo $conns | jq -r '.connections[2].qpairs_count'` == "-1" ]] + [[ `echo $conns | jq -r '.connections[2].controller_id'` == "-1" ]] + [[ `echo $conns | jq -r '.connections[2].connected'` == "false" ]] + [[ `echo $conns | jq -r '.connections[2].use_psk'` == "false" ]] + [[ `echo $conns | jq -r '.connections[2].use_dhchap'` == "false" ]] + + [[ `echo $conns | jq -r '.connections[3]'` == "null" ]] + + echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: ${port2} nqn: ${NQN}host2 no PSK" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme1 -t tcp -a $NVMEOF_IP_ADDRESS -s ${port2} -f ipv4 -n $NQN -q "${NQN}host2" -l -1 -o 10" + + echo "ℹ️ verify connection list again" + conns=`cephnvmf_func --output stdio --format json connection list --subsystem $NQN` + + [[ `echo $conns | jq -r '.status'` == "0" ]] + [[ `echo $conns | jq -r '.subsystem_nqn'` == "${NQN}" ]] + + [[ `echo $conns | jq -r '.connections[0].nqn'` == "${NQN}host" ]] + [[ `echo $conns | jq -r '.connections[0].trsvcid'` == "${NVMEOF_IO_PORT}" ]] + [[ `echo $conns | jq -r '.connections[0].traddr'` == "${NVMEOF_IP_ADDRESS}" ]] + [[ `echo $conns | jq -r '.connections[0].adrfam'` == "ipv4" ]] + [[ `echo $conns | jq -r '.connections[0].trtype'` == "TCP" ]] + [[ `echo $conns | jq -r '.connections[0].qpairs_count'` == "1" ]] + [[ `echo $conns | jq -r '.connections[0].controller_id'` == "1" ]] + [[ `echo $conns | jq -r '.connections[0].connected'` == "true" ]] + [[ `echo $conns | jq -r '.connections[0].secure'` == "true" ]] + [[ `echo $conns | jq -r '.connections[0].use_psk'` == "true" ]] + [[ `echo $conns | jq -r '.connections[0].use_dhchap'` == "false" ]] + + [[ `echo $conns | jq -r '.connections[1].nqn'` == "${NQN}host2" ]] + [[ `echo $conns | jq -r '.connections[1].trsvcid'` == "${port2}" ]] + [[ `echo $conns | jq -r '.connections[1].traddr'` == "${NVMEOF_IP_ADDRESS}" ]] + [[ `echo $conns | jq -r '.connections[1].adrfam'` == "ipv4" ]] + [[ `echo $conns | jq -r '.connections[1].trtype'` == "TCP" ]] + [[ `echo $conns | jq -r '.connections[1].qpairs_count'` == "1" ]] + [[ `echo $conns | jq -r '.connections[1].controller_id'` == "2" ]] + [[ `echo $conns | jq -r '.connections[1].connected'` == "true" ]] + [[ `echo $conns | jq -r '.connections[1].secure'` == "false" ]] + [[ `echo $conns | jq -r '.connections[1].use_psk'` == "false" ]] + [[ `echo $conns | jq -r '.connections[1].use_dhchap'` == "false" ]] + + [[ `echo $conns | jq -r '.connections[2].nqn'` == "${NQN}host3" ]] + [[ `echo $conns | jq -r '.connections[2].trsvcid'` == "0" ]] + [[ `echo $conns | jq -r '.connections[2].traddr'` == "" ]] + [[ `echo $conns | jq -r '.connections[2].adrfam'` == "ipv4" ]] + [[ `echo $conns | jq -r '.connections[2].trtype'` == "" ]] + [[ `echo $conns | jq -r '.connections[2].qpairs_count'` == "-1" ]] + [[ `echo $conns | jq -r '.connections[2].controller_id'` == "-1" ]] + [[ `echo $conns | jq -r '.connections[2].connected'` == "false" ]] + [[ `echo $conns | jq -r '.connections[2].use_psk'` == "true" ]] + [[ `echo $conns | jq -r '.connections[2].use_dhchap'` == "false" ]] + + [[ `echo $conns | jq -r '.connections[3]'` == "null" ]] + + echo "ℹ️ get controllers list" + controllers=$(make -s exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_get_controllers") + + echo "ℹ️ bdevperf perform_tests" + eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' ) + timeout=$(expr $BDEVPERF_TEST_DURATION \* 2) + bdevperf="/usr/libexec/spdk/scripts/bdevperf.py" + make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests" + + echo "ℹ️ bdevperf detach controllers" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_detach_controller Nvme0" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_detach_controller Nvme1" + + echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: ${port2} nqn: ${NQN}host using PSK, unsecure listener" + set +e + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s ${port2} -f ipv4 -n $NQN -q ${NQN}host -l -1 -o 10 --psk key1" + if [[ $? -eq 0 ]]; then + echo "Using PSK keys on an unsecure listener should fail" + exit 1 + fi + echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: ${NVMEOF_IO_PORT} nqn: ${NQN}host using PSK, wrong key" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s ${NVMEOF_IO_PORT} -f ipv4 -n $NQN -q ${NQN}host -l -1 -o 10 --psk key2" + if [[ $? -eq 0 ]]; then + echo "Connecting using the wrong PSK key should fail" + exit 1 + fi + set -e + return 0 +} + +function demo_bdevperf_dhchap() +{ + echo -n "ℹ️ Starting bdevperf container" + docker compose up -d bdevperf + sleep 10 + echo "ℹ️ bdevperf start up logs" + make logs SVC=bdevperf + eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' ) + + echo "ℹ️ bdevperf bdev_nvme_set_options" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1" + + dhchap_path_prefix="/tmp/dhchap/" + dhchap_path="${dhchap_path_prefix}${NQN}" + docker cp /tmp/temp-dhchap/dhchap ${BDEVPERF_CONTAINER_NAME}:`dirname ${dhchap_path_prefix}` + make exec SVC=bdevperf OPTS=-T CMD="chown -R root:root ${dhchap_path_prefix}" + make exec SVC=bdevperf OPTS=-T CMD="chmod 0600 ${dhchap_path}/key1" + make exec SVC=bdevperf OPTS=-T CMD="chmod 0600 ${dhchap_path}/key2" + make exec SVC=bdevperf OPTS=-T CMD="chmod 0600 ${dhchap_path}/key3" + rm -rf /tmp/temp-dhchap + + echo "ℹ️ bdevperf add DHCHAP key name key1 to keyring" + make -s exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET keyring_file_add_key key1 ${dhchap_path}/key1" + echo "ℹ️ bdevperf add DHCHAP key name key2 to keyring" + make -s exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET keyring_file_add_key key2 ${dhchap_path}/key2" + echo "ℹ️ bdevperf add DHCHAP controller key name key3 to keyring" + make -s exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET keyring_file_add_key key3 ${dhchap_path}/key3" + + echo "ℹ️ bdevperf list keyring" + make -s exec SVC=bdevperf OPTS=-T CMD="$rpc -s $BDEVPERF_SOCKET keyring_get_keys" + + set +e + echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: ${NVMEOF_IO_PORT} nqn: ${NQN}host using DHCHAP, wrong key" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s ${NVMEOF_IO_PORT} -f ipv4 -n ${NQN} -q ${NQN}host -l -1 -o 10 --dhchap-key key3" + if [[ $? -eq 0 ]]; then + echo "Connecting using the wrong DHCAP key should fail" + exit 1 + fi + + echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: ${NVMEOF_IO_PORT} nqn: ${NQN}host using DHCHAP, no key" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s ${NVMEOF_IO_PORT} -f ipv4 -n ${NQN} -q ${NQN}host -l -1 -o 10" + if [[ $? -eq 0 ]]; then + echo "Connecting without a DHCAP key should fail" + exit 1 + fi + set -e + + echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: ${NVMEOF_IO_PORT} nqn: ${NQN}host using DHCHAP" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s ${NVMEOF_IO_PORT} -f ipv4 -n ${NQN} -q ${NQN}host -l -1 -o 10 --dhchap-key key1" + + set +e + echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: ${port2} nqn: ${NQN}host2 using DHCHAP controller, wrong key" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme1 -t tcp -a $NVMEOF_IP_ADDRESS -s ${port2} -f ipv4 -n ${NQN} -q ${NQN}host2 -l -1 -o 10 --dhchap-key key2 --dhchap-ctrlr-key key1" + if [[ $? -eq 0 ]]; then + echo "Connecting using the wrong DHCAP controller key should fail" + exit 1 + fi + set -e + + echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: ${port2} nqn: ${NQN}host2 using DHCHAP controller" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme1 -t tcp -a $NVMEOF_IP_ADDRESS -s ${port2} -f ipv4 -n ${NQN} -q ${NQN}host2 -l -1 -o 10 --dhchap-key key2 --dhchap-ctrlr-key key3" + + echo "ℹ️ bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: ${port3} nqn: ${NQN}host3 not using DHCHAP" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme2 -t tcp -a $NVMEOF_IP_ADDRESS -s ${port3} -f ipv4 -n ${NQN} -q ${NQN}host3 -l -1 -o 10" + + echo "ℹ️ get controllers list" + controllers=`make -s exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_get_controllers"` + + echo "ℹ️ verify connection list" + conns=`cephnvmf_func --output stdio --format json connection list --subsystem $NQN` + + [[ `echo $conns | jq -r '.status'` == "0" ]] + [[ `echo $conns | jq -r '.subsystem_nqn'` == "${NQN}" ]] + + [[ `echo $conns | jq -r '.connections[0].nqn'` == "${NQN}host" ]] + [[ `echo $conns | jq -r '.connections[0].trsvcid'` == "${NVMEOF_IO_PORT}" ]] + [[ `echo $conns | jq -r '.connections[0].traddr'` == "${NVMEOF_IP_ADDRESS}" ]] + [[ `echo $conns | jq -r '.connections[0].adrfam'` == "ipv4" ]] + [[ `echo $conns | jq -r '.connections[0].trtype'` == "TCP" ]] + [[ `echo $conns | jq -r '.connections[0].qpairs_count'` == "1" ]] + [[ `echo $conns | jq -r '.connections[0].controller_id'` == "3" ]] + [[ `echo $conns | jq -r '.connections[0].connected'` == "true" ]] + [[ `echo $conns | jq -r '.connections[0].secure'` == "false" ]] + [[ `echo $conns | jq -r '.connections[0].use_psk'` == "false" ]] + [[ `echo $conns | jq -r '.connections[0].use_dhchap'` == "true" ]] + + [[ `echo $conns | jq -r '.connections[1].nqn'` == "${NQN}host2" ]] + [[ `echo $conns | jq -r '.connections[1].trsvcid'` == "${port2}" ]] + [[ `echo $conns | jq -r '.connections[1].traddr'` == "${NVMEOF_IP_ADDRESS}" ]] + [[ `echo $conns | jq -r '.connections[1].adrfam'` == "ipv4" ]] + [[ `echo $conns | jq -r '.connections[1].trtype'` == "TCP" ]] + [[ `echo $conns | jq -r '.connections[1].qpairs_count'` == "1" ]] + [[ `echo $conns | jq -r '.connections[1].controller_id'` == "5" ]] + [[ `echo $conns | jq -r '.connections[1].connected'` == "true" ]] + [[ `echo $conns | jq -r '.connections[1].secure'` == "false" ]] + [[ `echo $conns | jq -r '.connections[1].use_psk'` == "false" ]] + [[ `echo $conns | jq -r '.connections[1].use_dhchap'` == "true" ]] + + [[ `echo $conns | jq -r '.connections[2].nqn'` == "${NQN}host3" ]] + [[ `echo $conns | jq -r '.connections[2].trsvcid'` == "${port3}" ]] + [[ `echo $conns | jq -r '.connections[2].traddr'` == "${NVMEOF_IP_ADDRESS}" ]] + [[ `echo $conns | jq -r '.connections[2].adrfam'` == "ipv4" ]] + [[ `echo $conns | jq -r '.connections[2].trtype'` == "TCP" ]] + [[ `echo $conns | jq -r '.connections[2].qpairs_count'` == "1" ]] + [[ `echo $conns | jq -r '.connections[2].controller_id'` == "6" ]] + [[ `echo $conns | jq -r '.connections[2].connected'` == "true" ]] + [[ `echo $conns | jq -r '.connections[2].secure'` == "false" ]] + [[ `echo $conns | jq -r '.connections[2].use_psk'` == "false" ]] + [[ `echo $conns | jq -r '.connections[2].use_dhchap'` == "false" ]] + + [[ `echo $conns | jq -r '.connections[3]'` == "null" ]] + + echo "ℹ️ bdevperf perform_tests" + eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' ) + timeout=$(expr $BDEVPERF_TEST_DURATION \* 2) + bdevperf="/usr/libexec/spdk/scripts/bdevperf.py" + make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests" + + echo "ℹ️ verify DHCHAP key files removal" + dhchap_key_list=`make -s exec SVC=nvmeof OPTS=-T CMD="/usr/local/bin/spdk_rpc -s /var/tmp/spdk.sock keyring_get_keys"` + path1=`echo ${dhchap_key_list} | jq -r '.[0].path'` + path2=`echo ${dhchap_key_list} | jq -r '.[1].path'` + path3=`echo ${dhchap_key_list} | jq -r '.[2].path'` + subsys_dir=`dirname ${path1}` + [[ `echo $dhchap_key_list | jq -r '.[0].removed'` == "false" ]] + [[ `echo $dhchap_key_list | jq -r '.[1].removed'` == "false" ]] + [[ `echo $dhchap_key_list | jq -r '.[2].removed'` == "false" ]] + [[ `echo $dhchap_key_list | jq -r '.[3].removed'` == "null" ]] + make exec SVC=nvmeof OPTS=-T CMD="test -f ${path1}" + make exec SVC=nvmeof OPTS=-T CMD="test -f ${path2}" + make exec SVC=nvmeof OPTS=-T CMD="test -f ${path3}" + make exec SVC=nvmeof OPTS=-T CMD="test -d ${subsys_dir}" + cephnvmf_func host del --subsystem $NQN --host-nqn ${NQN}host2 + make exec SVC=nvmeof OPTS=-T CMD="test -f ${path1}" + make exec SVC=nvmeof OPTS=-T CMD="test ! -f ${path2}" + make exec SVC=nvmeof OPTS=-T CMD="test ! -f ${path3}" + cephnvmf_func subsystem del --subsystem $NQN --force + make exec SVC=nvmeof OPTS=-T CMD="test ! -f ${path1}" + make exec SVC=nvmeof OPTS=-T CMD="test ! -d ${subsys_dir}" + dhchap_key_list=`make -s exec SVC=nvmeof OPTS=-T CMD="/usr/local/bin/spdk_rpc -s /var/tmp/spdk.sock keyring_get_keys"` + [[ `echo $dhchap_key_list | jq -r '.[0]'` == "null" ]] + + echo "ℹ️ bdevperf detach controllers" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_detach_controller Nvme0" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_detach_controller Nvme1" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_detach_controller Nvme2" + + echo "ℹ️ get controllers list again" + controllers=`make -s exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_get_controllers"` + [[ "${controllers}" == "[]" ]] + return 0 +} + +. .env + +set -e +set -x +rpc="/usr/libexec/spdk/scripts/rpc.py" +port2=`expr ${NVMEOF_IO_PORT} + 10` +port3=`expr ${NVMEOF_IO_PORT} + 20` +case "$1" in + test_unsecured) + demo_test_unsecured + ;; + test_psk) + demo_test_psk + ;; + test_dhchap) + demo_test_dhchap + ;; + bdevperf_unsecured) + demo_bdevperf_unsecured + ;; + bdevperf_psk) + demo_bdevperf_psk + ;; + bdevperf_dhchap) + demo_bdevperf_dhchap + ;; + *) + echo "Invalid argument $1" + exit 1 + ;; +esac + +exit $? diff --git a/tests/test_dhchap.py b/tests/test_dhchap.py new file mode 100644 index 00000000..0dd0138a --- /dev/null +++ b/tests/test_dhchap.py @@ -0,0 +1,189 @@ +import pytest +from control.server import GatewayServer +import socket +from control.cli import main as cli +from control.cli import main_test as cli_test +from control.cephutils import CephUtils +from control.utils import GatewayUtils +from control.config import GatewayConfig +import grpc +from control.proto import gateway_pb2 as pb2 +from control.proto import gateway_pb2_grpc as pb2_grpc +import os +import os.path + +image = "mytestdevimage" +pool = "rbd" +subsystem = "nqn.2016-06.io.spdk:cnode1" +hostnqn1 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7eb" +hostnqn2 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7ec" +hostnqn4 = "nqn.2014-08.org.nvmexpress:uuid:6488a49c-dfa3-11d4-ac31-b232c6c68a8a" +hostnqn5 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7ef" +hostnqn6 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7f0" +hostnqn7 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7f1" +hostnqn8 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7f2" +hostnqn9 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7f3" +hostnqn10 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7f4" +hostnqn11 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7f5" + +hostdhchap1 = "DHHC-1:00:MWPqcx1Ug1debg8fPIGpkqbQhLcYUt39k7UWirkblaKEH1kE:" +hostdhchap2 = "DHHC-1:00:ojmm6ISA2DBpZEldEJqJvA1/cAl1wDmeolS0fCIn2qZpK0b7gpx3qu76yHpjlOOXNyjqf0oFYCWxUqkXGN2xEBOVxoA=:" +hostdhchap4 = "DHHC-1:01:Ei7kUrD7iyrjzXDwIZ666sSPBswUk4wDjSQtodytVB5YiBru:" +hostdhchap5 = "DHHC-1:03:6EKZcEL86u4vzTE8sCETvskE3pLKE+qOorl9QxrRfLvfOEQ5GvqCzM41U8fFVAz1cs+T14PjSpd1J641rfeCC1x2VNg=:" +hostdhchap6 = "DHHC-1:02:ULMaRuJ40ui58nK4Qk5b0J89G3unbGb8mBUbt/XSrf18PBPvyL3sivZOINNh2o/fPpXbGg==:" + +host_name = socket.gethostname() +addr = "127.0.0.1" +config = "ceph-nvmeof.conf" + +@pytest.fixture(scope="module") +def gateway(config): + """Sets up and tears down Gateway""" + + addr = config.get("gateway", "addr") + port = config.getint("gateway", "port") + config.config["gateway-logs"]["log_level"] = "debug" + config.config["gateway"]["group"] = "" + ceph_utils = CephUtils(config) + + with GatewayServer(config) as gateway: + + # Start gateway + gateway.gw_logger_object.set_log_level("debug") + ceph_utils.execute_ceph_monitor_command("{" + f'"prefix":"nvme-gw create", "id": "{gateway.name}", "pool": "{pool}", "group": ""' + "}") + gateway.serve() + + # Bind the client and Gateway + channel = grpc.insecure_channel(f"{addr}:{port}") + yield gateway.gateway_rpc + + # Stop gateway + gateway.server.stop(grace=1) + gateway.gateway_rpc.gateway_state.delete_state() + +def test_setup(caplog, gateway): + gw = gateway + caplog.clear() + cli(["subsystem", "add", "--subsystem", subsystem]) + assert f"create_subsystem {subsystem}: True" in caplog.text + caplog.clear() + cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image, "--rbd-create-image", "--size", "16MB"]) + assert f"Adding namespace 1 to {subsystem}: Successful" in caplog.text + +def test_create_secure(caplog, gateway): + caplog.clear() + cli(["listener", "add", "--subsystem", subsystem, "--host-name", host_name, "-a", addr, "-s", "5001", "--secure"]) + assert f"Adding {subsystem} listener at {addr}:5001: Successful" in caplog.text + caplog.clear() + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn1, "--dhchap-key", hostdhchap1]) + assert f"Adding host {hostnqn1} to {subsystem}: Successful" in caplog.text + caplog.clear() + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn2, "--dhchap-key", hostdhchap2]) + assert f"Adding host {hostnqn2} to {subsystem}: Successful" in caplog.text + caplog.clear() + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn4, "--dhchap-key", hostdhchap4]) + assert f"Adding host {hostnqn4} to {subsystem}: Successful" in caplog.text + +def test_create_not_secure(caplog, gateway): + caplog.clear() + cli(["listener", "add", "--subsystem", subsystem, "--host-name", host_name, "-a", addr, "-s", "5002"]) + assert f"Adding {subsystem} listener at {addr}:5002: Successful" in caplog.text + caplog.clear() + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn6]) + assert f"Adding host {hostnqn6} to {subsystem}: Successful" in caplog.text + caplog.clear() + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn7]) + assert f"Adding host {hostnqn7} to {subsystem}: Successful" in caplog.text + +def test_create_secure_list(caplog, gateway): + caplog.clear() + rc = 0 + try: + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn8, hostnqn9, hostnqn10, "--dhchap-key", hostdhchap1]) + except SystemExit as sysex: + rc = int(str(sysex)) + pass + assert rc == 2 + assert f"error: Can't have more than one host NQN when DH-HMAC-CHAP keys are used" in caplog.text + +def test_create_secure_no_key(caplog, gateway): + caplog.clear() + rc = 0 + try: + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn5, "--dhchap-key"]) + except SystemExit as sysex: + rc = int(str(sysex)) + pass + assert rc == 2 + assert f"error: argument --dhchap-key: expected one argument" in caplog.text + +def test_dhchap_controller_key(caplog, gateway): + caplog.clear() + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn11, "--dhchap-key", hostdhchap5, "--dhchap-ctrlr-key", hostdhchap6]) + assert f"Adding host {hostnqn11} to {subsystem}: Successful" in caplog.text + +def test_list_dhchap_hosts(caplog, gateway): + caplog.clear() + hosts = cli_test(["host", "list", "--subsystem", subsystem]) + found = 0 + assert len(hosts.hosts) == 6 + for h in hosts.hosts: + if h.nqn == hostnqn1: + found += 1 + assert h.use_dhchap + elif h.nqn == hostnqn2: + found += 1 + assert h.use_dhchap + elif h.nqn == hostnqn4: + found += 1 + assert h.use_dhchap + elif h.nqn == hostnqn6: + found += 1 + assert not h.use_dhchap + elif h.nqn == hostnqn7: + found += 1 + assert not h.use_dhchap + elif h.nqn == hostnqn11: + found += 1 + assert h.use_dhchap + else: + assert False + assert found == 6 + +def test_allow_any_host_with_dhchap(caplog, gateway): + caplog.clear() + rc = 0 + try: + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", "*", "--dhchap-key", hostdhchap1]) + except SystemExit as sysex: + rc = int(str(sysex)) + pass + assert rc == 2 + assert f"error: DH-HMAC-CHAP key is only allowed for specific hosts" in caplog.text + +def test_dhchap_controller_with_no_dhchap_key(caplog, gateway): + caplog.clear() + rc = 0 + try: + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn10, "--dhchap-ctrlr-key", hostdhchap1]) + except SystemExit as sysex: + rc = int(str(sysex)) + pass + assert rc == 2 + assert f"error: DH-HMAC-CHAP controller keys can not be used without DH-HMAC-CHAP keys" in caplog.text + +def test_list_listeners(caplog, gateway): + caplog.clear() + listeners = cli_test(["listener", "list", "--subsystem", subsystem]) + assert len(listeners.listeners) == 2 + found = 0 + for l in listeners.listeners: + if l.trsvcid == 5001: + found += 1 + assert l.secure + elif l.trsvcid == 5002: + found += 1 + assert not l.secure + else: + assert False + assert found == 2 diff --git a/tests/test_psk.py b/tests/test_psk.py index 9ee2e05e..f6ff0996 100644 --- a/tests/test_psk.py +++ b/tests/test_psk.py @@ -15,7 +15,7 @@ image = "mytestdevimage" pool = "rbd" subsystem = "nqn.2016-06.io.spdk:cnode1" -hostnqn = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7eb" +hostnqn1 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7eb" hostnqn2 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7ec" hostnqn3 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7ee" hostnqn4 = "nqn.2014-08.org.nvmexpress:uuid:6488a49c-dfa3-11d4-ac31-b232c6c68a8a" @@ -26,15 +26,10 @@ hostnqn9 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7f3" hostnqn10 = "nqn.2014-08.org.nvmexpress:uuid:22207d09-d8af-4ed2-84ec-a6d80b0cf7f4" -hostpsk = "NVMeTLSkey-1:01:YzrPElk4OYy1uUERriPwiiyEJE/+J5ckYpLB+5NHMsR2iBuT:" +hostpsk1 = "NVMeTLSkey-1:01:YzrPElk4OYy1uUERriPwiiyEJE/+J5ckYpLB+5NHMsR2iBuT:" hostpsk2 = "NVMeTLSkey-1:02:FTFds4vH4utVcfrOforxbrWIgv+Qq4GQHgMdWwzDdDxE1bAqK2mOoyXxmbJxGeueEVVa/Q==:" hostpsk3 = "junk" hostpsk4 = "NVMeTLSkey-1:01:YzrPElk4OYy1uUERriPwiiyEJE/+J5ckYpLB+5NHMsR2iBuT:" -hostpsk5 = "NVMeTLSkey-1:01:sDSy/cehSZT7WBKuwfwvWzeYR5xV5BKBR3Q1ILO2StCDEfge:" -hostpsk6 = "NVMeTLSkey-1:01:e4zyO+j6DF5eZHanqYmHqFjEF2enzY8N/r1S4jijfabgsRUR:" -hostpsk7 = "NVMeTLSkey-1:02:IuO9SMATHiHFfiGkEqh/eKzAtx9zkDDs55PoAz1ndKhW0KZUHYfKtrsCgx0X+c30ygP8Ew==:" -hostpsk8 = "NVMeTLSkey-1:01:uJhA3uB/cMwNnz4PW8pf4LIPpt6wZ1ldt4n3OUERAc5iv38T:" -hostpsk9 = "NVMeTLSkey-1:01:YcM21gigb+0pFJXO+sOWj9Sz6gzIHxzBDTP8HYLEpqLBwznp:" host_name = socket.gethostname() addr = "127.0.0.1" @@ -65,11 +60,6 @@ def gateway(config): gateway.server.stop(grace=1) gateway.gateway_rpc.gateway_state.delete_state() -def write_file(filepath, content): - with open(filepath, "w") as f: - f.write(content) - os.chmod(filepath, 0o600) - def test_setup(caplog, gateway): gw = gateway caplog.clear() @@ -99,8 +89,8 @@ def test_create_secure(caplog, gateway): cli(["listener", "add", "--subsystem", subsystem, "--host-name", host_name, "-a", addr, "-s", "5001", "--secure"]) assert f"Adding {subsystem} listener at {addr}:5001: Successful" in caplog.text caplog.clear() - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn, "--psk", hostpsk]) - assert f"Adding host {hostnqn} to {subsystem}: Successful" in caplog.text + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn1, "--psk", hostpsk1]) + assert f"Adding host {hostnqn1} to {subsystem}: Successful" in caplog.text caplog.clear() cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn2, "--psk", hostpsk2]) assert f"Adding host {hostnqn2} to {subsystem}: Successful" in caplog.text @@ -123,7 +113,7 @@ def test_create_secure_list(caplog, gateway): caplog.clear() rc = 0 try: - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn8, hostnqn9, hostnqn10, "--psk", hostpsk]) + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn8, hostnqn9, hostnqn10, "--psk", hostpsk1]) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -152,7 +142,7 @@ def test_list_psk_hosts(caplog, gateway): found = 0 assert len(hosts.hosts) == 5 for h in hosts.hosts: - if h.nqn == hostnqn: + if h.nqn == hostnqn1: found += 1 assert h.use_psk elif h.nqn == hostnqn2: @@ -175,12 +165,23 @@ def test_allow_any_host_with_psk(caplog, gateway): caplog.clear() rc = 0 try: - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", "*", "--psk", hostpsk]) + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", "*", "--psk", hostpsk1]) + except SystemExit as sysex: + rc = int(str(sysex)) + pass + assert rc == 2 + assert f"error: PSK key is only allowed for specific hosts" in caplog.text + +def test_psk_with_dhchap(caplog, gateway): + caplog.clear() + rc = 0 + try: + cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn10, "--psk", hostpsk1, "--dhchap-key", "junk"]) except SystemExit as sysex: rc = int(str(sysex)) pass assert rc == 2 - assert f"error: PSK is only allowed for specific hosts" in caplog.text + assert f"error: PSK and DH-HMAC-CHAP keys are mutually exclusive" in caplog.text def test_list_listeners(caplog, gateway): caplog.clear()