Merge branch 'devel' into nvmeof-cli-spdk-logs

Signed-off-by: Gil Bregman <[email protected]>

.env

@@ -1,13 +1,15 @@
 # Globals
-VERSION="0.0.3"
-CEPH_VERSION="17.2.6"
+VERSION="0.0.5"
+CEPH_VERSION="18.2.0"
 SPDK_VERSION="23.01.1"
 CONTAINER_REGISTRY="quay.io/ceph"
 QUAY_SPDK="${CONTAINER_REGISTRY}/spdk"
 QUAY_CEPH="${CONTAINER_REGISTRY}/vstart-cluster"
 QUAY_NVMEOF="${CONTAINER_REGISTRY}/nvmeof"
 QUAY_NVMEOFCLI="${CONTAINER_REGISTRY}/nvmeof-cli"
 MAINTAINER="Ceph Developers <[email protected]>"
+COMPOSE_PROJECT_NAME="ceph-nvmeof"
+NVMEOF_CONTAINER_NAME="${COMPOSE_PROJECT_NAME}-nvmeof-1"
 
 # Performance
 NVMEOF_NOFILE=20480 # Max number of open files (depends on number of hosts connected)
@@ -18,18 +20,19 @@ HUGEPAGES_DIR="/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
 NVMEOF_VERSION="${VERSION}"
 NVMEOF_CONFIG="./ceph-nvmeof.conf"
 NVMEOF_SPDK_VERSION="${SPDK_VERSION}"
+NVMEOF_CEPH_VERSION="${CEPH_VERSION}"
 NVMEOF_NAME="ceph-nvmeof"
 NVMEOF_SUMMARY="Ceph NVMe over Fabrics Gateway"
 NVMEOF_DESCRIPTION="Service to provide block storage on top of Ceph for platforms (e.g.: VMWare) without native Ceph support (RBD), replacing existing approaches (iSCSI) with a newer and more versatile standard (NVMe-oF)."
 NVMEOF_URL="https://github.com/ceph/ceph-nvmeof"
 NVMEOF_TAGS="ceph,nvme-of,nvme-of gateway,rbd,block storage"
 NVMEOF_WANTS="ceph,rbd"
-NVMEOF_IP_ADDRESS="192.168.13.3"
+NVMEOF_IP_ADDRESS=192.168.13.3
+NVMEOF_IPV6_ADDRESS=2001:db8::3
 NVMEOF_IO_PORT=4420
 NVMEOF_GW_PORT=5500
 NVMEOF_DISC_PORT=8009
 NVMEOF_EXPOSE_SERVICES="${NVMEOF_IO_PORT}/tcp:nvme,${NVMEOF_GW_PORT}/tcp:grpc,${NVMEOF_DISC_PORT}/tcp:nvme-disc"
-NVMEOF_GIT_REPO="https://github.com/ceph/ceph-nvmeof.git"
 
 # NVMe-oF CLI
 MVMEOF_CLI_VERSION="${VERSION}"
@@ -47,6 +50,7 @@ SPDK_URL="https://spdk.io"
 SPDK_PKGDEP_ARGS="--rbd"
 SPDK_CONFIGURE_ARGS="--with-rbd --disable-tests --disable-unit-tests --disable-examples --enable-debug"
 SPDK_MAKEFLAGS=
+SPDK_DISABLE_VPCLMULQDQ=
 SPDK_CENTOS_BASE="https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/"
 SPDK_CENTOS_REPO_VER="9.0-21.el9"
 
@@ -55,10 +59,10 @@ CEPH_CLUSTER_VERSION="${CEPH_VERSION}"
 CEPH_VSTART_ARGS="--without-dashboard --memstore"
 
 # Demo settings
-RBD_POOL="rbd"
-RBD_IMAGE_NAME="demo_image"
-RBD_IMAGE_SIZE="10M"
-BDEV_NAME="demo_bdev"
+RBD_POOL=rbd
+RBD_IMAGE_NAME=demo_image
+RBD_IMAGE_SIZE=10M
+BDEV_NAME=demo_bdev
 NQN="nqn.2016-06.io.spdk:cnode1"
 SERIAL="SPDK00000000000001"
 

.github/workflows/build-container.yml

@@ -23,7 +23,7 @@ jobs:
           submodules: recursive
 
       - name: Build container images
-        run: make build
+        run: make build SPDK_DISABLE_VPCLMULQDQ="yes"
 
       - name: Save container images
         run: |
@@ -65,7 +65,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-          test: ["cli", "state", "multi_gateway", "server"]
+          test: ["cli", "state", "multi_gateway", "server", "grpc", "omap_lock"]
     runs-on: ubuntu-latest
     env:
       HUGEPAGES: 512 # for multi gateway test, approx 256 per gateway instance
@@ -87,6 +87,27 @@ jobs:
           docker load < nvmeof-devel.tar
           docker load < vstart-cluster.tar
 
+      - name: Clear space on disk
+        run: |
+          if [[ -d /usr/share/dotnet ]]; then
+            /usr/bin/du -sh /usr/share/dotnet
+            rm -rf /usr/share/dotnet
+          fi
+          if [[ -d /opt/ghc ]]; then
+            /usr/bin/du -sh /opt/ghc
+            rm -rf /opt/ghc
+          fi
+          if [[ -d /usr/local/share/boost ]]; then
+            /usr/bin/du -sh /usr/local/share/boost
+            rm -rf /usr/local/share/boost
+          fi
+          if [[ -n "$AGENT_TOOLSDIRECTORY" ]]; then
+            if [[ -d "$AGENT_TOOLSDIRECTORY" ]]; then
+              /usr/bin/du -sh "$AGENT_TOOLSDIRECTORY"
+              rm -rf "$AGENT_TOOLSDIRECTORY"
+            fi
+          fi
+
       - name: Start ceph cluster
         run: |
           make up SVC=ceph OPTS="--detach"
@@ -115,11 +136,14 @@ jobs:
           echo "💁 ls rbd:"
           make exec SVC=ceph OPTS="-T" CMD="rbd ls rbd"
 
+      - name: Run protoc
+        run: |
+          make protoc
+
       - name: Run ${{ matrix.test }} test
         run: |
           # Run tests code in current dir
           # Managing pytest’s output: https://docs.pytest.org/en/7.1.x/how-to/output.html
-          make protoc
           make run SVC="nvmeof-devel" OPTS="--volume=$(pwd)/tests:/src/tests --entrypoint=python3" CMD="-m pytest --show-capture=all -s --full-trace -vv -rA tests/test_${{ matrix.test }}.py"
 
       - name: Check coredump existence
@@ -197,14 +221,15 @@ jobs:
 
       - name: Test
         run: |
-          make demo OPTS=-T
+          make demo OPTS=-T NVMEOF_CONTAINER_NAME="ceph-nvmeof_nvmeof_1"
 
       - name: Get subsystems
         run: |
           # https://github.com/actions/toolkit/issues/766
           shopt -s expand_aliases
           eval $(make alias)
           nvmeof-cli get_subsystems
+          nvmeof-cli-ipv6 get_subsystems
 
       - name: Run bdevperf
         run: |
@@ -261,6 +286,10 @@ jobs:
 
   discovery:
     needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+          integration: ["container", "embedded"]
     runs-on: ubuntu-latest
     env:
       HUGEPAGES: 768 # 3 spdk instances
@@ -285,10 +314,12 @@ jobs:
           docker load < bdevperf.tar
 
       - name: Start discovery controller
+        if: matrix.integration == 'container'
         run: |
           docker-compose up --detach discovery
 
       - name: Wait for discovery controller to be listening
+        if: matrix.integration == 'container'
         timeout-minutes: 3
         run: |
           . .env
@@ -392,7 +423,14 @@ jobs:
           echo "ℹ️  bdevperf start up logs"
           make logs SVC=bdevperf
           eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' )
-          ip=$(container_ip $DISC1)
+
+          if [ "${{ matrix.integration }}" == "embedded" ]; then
+            ip=$(container_ip $GW1)
+            echo "ℹ️  Using discovery service in gateway $GW1 ip  $ip"
+          else
+            ip=$(container_ip $DISC1)
+            echo "ℹ️  Using standalone discovery container $DISC1 ip  $ip"
+          fi
           rpc="/usr/libexec/spdk/scripts/rpc.py"
           echo "ℹ️  bdevperf bdev_nvme_set_options"
           make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1"
@@ -412,14 +450,14 @@ jobs:
         id: check_coredumps
         uses: andstor/file-existence-action@20b4d2e596410855db8f9ca21e96fbe18e12930b # v2, pinned to SHA for security reasons
         with:
-            files: "/var/lib/systemd/coredump/*"
+            files: "/tmp/coredump/core.*"
 
       - name: Upload demo core dumps
         if: steps.check_coredumps.outputs.files_exists == 'true'
         uses: actions/upload-artifact@v1
         with:
           name: ceph_nvmeof_demo_cores-${{ github.run_number }}
-          path: /var/lib/systemd/coredump/*
+          path: /tmp/coredump/core.*
 
       - name: Display logs
         if: success() || failure()
@@ -437,6 +475,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
       - name: Download container images
         uses: actions/download-artifact@v3
         with:

Dockerfile

@@ -52,7 +52,19 @@ ARG NVMEOF_NAME \
     BUILD_DATE \
     NVMEOF_GIT_REPO \
     NVMEOF_GIT_BRANCH \
-    NVMEOF_GIT_COMMIT
+    NVMEOF_GIT_COMMIT \
+    NVMEOF_SPDK_VERSION \
+    NVMEOF_CEPH_VERSION \
+    NVMEOF_GIT_MODIFIED_FILES
+
+ENV NVMEOF_VERSION="${NVMEOF_VERSION}" \
+      NVMEOF_GIT_REPO="${NVMEOF_GIT_REPO}" \
+      NVMEOF_GIT_BRANCH="${NVMEOF_GIT_BRANCH}" \
+      NVMEOF_GIT_COMMIT="${NVMEOF_GIT_COMMIT}" \
+      BUILD_DATE="${BUILD_DATE}" \
+      NVMEOF_SPDK_VERSION="${NVMEOF_SPDK_VERSION}" \
+      NVMEOF_CEPH_VERSION="${NVMEOF_CEPH_VERSION}" \
+      NVMEOF_GIT_MODIFIED_FILES="${NVMEOF_GIT_MODIFIED_FILES}"
 
 # Generic labels
 LABEL name="$NVMEOF_NAME" \

Dockerfile.ceph

@@ -38,9 +38,6 @@ RUN rpm -vih https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.
 RUN rpm --import 'https://download.ceph.com/keys/release.asc'
 RUN cd /etc/yum.repos.d/ && curl -O https://copr.fedorainfracloud.org/coprs/ceph/el9/repo/epel-9/ceph-el9-epel-9.repo
 
-## WORKAROUND: remove when https://tracker.ceph.com/issues/61882 is fixed
-RUN rpm -vih "https://buildlogs.centos.org/centos/9-stream/storage/x86_64/ceph-quincy/Packages/t/thrift-0.14.0-7.el9s.x86_64.rpm"
-
 RUN \
     --mount=type=cache,target=/var/cache/microdnf \
     microdnf install -y \

Dockerfile.spdk

@@ -24,7 +24,8 @@ EOF
 
 ARG SPDK_PKGDEP_ARGS \
     SPDK_CONFIGURE_ARGS \
-    SPDK_MAKEFLAGS
+    SPDK_MAKEFLAGS \
+    SPDK_DISABLE_VPCLMULQDQ
 
 WORKDIR /src
 COPY . .
@@ -38,9 +39,19 @@ RUN \
     && dnf install -y \
         rpm-build \
         git-core \
+    && rpm -vih https://buildlogs.centos.org/centos/9-stream/storage/x86_64/ceph-reef/Packages/t/thrift-0.15.0-3.el9s.x86_64.rpm \
     && scripts/pkgdep.sh $SPDK_PKGDEP_ARGS \
     && dnf update -y
 
+# Disable RDSEED, see https://github.com/ceph/ceph-nvmeof/issues/259
+RUN \
+    sed -i "s/^\( \+'RDSEED'\)/#\1/" dpdk/config/x86/meson.build
+# If we run from github disable also VPCLMULQDQ as some of github's runners don't have it
+RUN \
+    if [[ -n "$SPDK_DISABLE_VPCLMULQDQ" ]]; then sed -i "s/^\( \+'VPCLMULQDQ'\)/#\1/" dpdk/config/x86/meson.build ; fi
+RUN \
+    cat dpdk/config/x86/meson.build
+
 RUN \
     --mount=type=cache,target=/var/cache/dnf \
     --mount=type=cache,target=/var/lib/dnf \
@@ -114,6 +125,7 @@ RUN \
     --mount=type=cache,target=/var/lib/dnf \
     rpm -vih $SPDK_CENTOS_BASE/centos-stream-repos-$SPDK_CENTOS_REPO_VER.noarch.rpm \
              $SPDK_CENTOS_BASE/centos-gpg-keys-$SPDK_CENTOS_REPO_VER.noarch.rpm \
+             https://buildlogs.centos.org/centos/9-stream/storage/x86_64/ceph-reef/Packages/t/thrift-0.15.0-3.el9s.x86_64.rpm \
     && dnf install -y /rpm/$(uname -m)/*.rpm \
     && dnf update -y
 

Makefile

@@ -24,12 +24,14 @@ setup: ## Configure huge-pages (requires sudo/root password)
 
 build pull logs: SVC ?= spdk bdevperf nvmeof nvmeof-devel nvmeof-cli discovery ceph
 
+build: export NVMEOF_GIT_REPO != git remote get-url origin
 build: export NVMEOF_GIT_BRANCH != git name-rev --name-only HEAD
 build: export NVMEOF_GIT_COMMIT != git rev-parse HEAD
 build: export SPDK_GIT_REPO != git -C spdk remote get-url origin
 build: export SPDK_GIT_BRANCH != git -C spdk name-rev --name-only HEAD
 build: export SPDK_GIT_COMMIT != git rev-parse HEAD:spdk
-build: export BUILD_DATE != date -u +"%Y-%m-%dT%H:%M:%SZ"
+build: export BUILD_DATE != date -u +"%Y-%m-%d %H:%M:%S %Z"
+build: export NVMEOF_GIT_MODIFIED_FILES != git status -s | grep -e "^ *M" | sed 's/^ *M //' | xargs
 
 up: ## Launch services
 up: SVC ?= ceph nvmeof ## Services

ceph-nvmeof.conf

@@ -17,7 +17,11 @@ state_update_notify = True
 state_update_interval_sec = 5
 #min_controller_id = 1
 #max_controller_id = 65519
-enable_discovery_controller = false
+enable_spdk_discovery_controller = False
+#omap_file_lock_duration = 60
+#omap_file_lock_retries = 15
+#omap_file_lock_retry_sleep_interval = 5
+#omap_file_update_reloads = 10
 
 [discovery]
 addr = 0.0.0.0
@@ -37,7 +41,8 @@ client_cert = ./client.crt
 
 [spdk]
 tgt_path = /usr/local/bin/nvmf_tgt
-rpc_socket = /var/tmp/spdk.sock
+#rpc_socket_dir = /var/tmp/
+#rpc_socket_name = spdk.sock
 #tgt_cmd_extra_args = --env-context="--no-huge -m1024" --iova-mode=va
 timeout = 60.0
 log_level = WARN

control/__main__.py

@@ -31,6 +31,7 @@
     args = parser.parse_args()
 
     config = GatewayConfig(args.config)
+    config.dump_config_file(logger)
     with GatewayServer(config) as gateway:
         gateway.serve()
         gateway.keep_alive()

control/cli.py

@@ -17,7 +17,7 @@
 
 from .proto import gateway_pb2_grpc as pb2_grpc
 from .proto import gateway_pb2 as pb2
-
+from .config import GatewayConfig
 
 def argument(*name_or_flags, **kwargs):
     """Helper function to format arguments for argparse command decorator."""
@@ -124,7 +124,9 @@ def stub(self):
 
     def connect(self, host, port, client_key, client_cert, server_cert):
         """Connects to server and sets stub."""
-        server = "{}:{}".format(host, port)
+        # We need to enclose IPv6 addresses in brackets before concatenating a colon and port number to it
+        host = GatewayConfig.escape_address_if_ipv6(host)
+        server = f"{host}:{port}"
 
         if client_key and client_cert:
             # Create credentials for mutual TLS and a secure channel
@@ -187,12 +189,16 @@ def delete_bdev(self, args):
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),
         argument("-s", "--serial", help="Serial number", required=False),
         argument("-m", "--max-namespaces", help="Maximum number of namespaces", type=int, default=0, required=False),
+        argument("-a", "--ana-reporting", help="Enable ANA reporting", action='store_true', required=False),
+        argument("-t", "--enable-ha", help="Enable automatic HA", action='store_true', required=False),
     ])
     def create_subsystem(self, args):
         """Creates a subsystem."""
         req = pb2.create_subsystem_req(subsystem_nqn=args.subnqn,
                                         serial_number=args.serial,
-                                        max_namespaces=args.max_namespaces)
+                                        max_namespaces=args.max_namespaces,
+                                        ana_reporting=args.ana_reporting,
+                                        enable_ha=args.enable_ha)
         ret = self.stub.create_subsystem(req)
         self.logger.info(f"Created subsystem {args.subnqn}: {ret.status}")
 
@@ -209,15 +215,20 @@ def delete_subsystem(self, args):
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),
         argument("-b", "--bdev", help="Bdev name", required=True),
         argument("-i", "--nsid", help="Namespace ID", type=int),
+        argument("-a", "--anagrpid", help="ANA group ID", type=int),
     ])
     def add_namespace(self, args):
         """Adds a namespace to a subsystem."""
+        if args.anagrpid == 0:
+	        args.anagrpid = 1
+
         req = pb2.add_namespace_req(subsystem_nqn=args.subnqn,
                                     bdev_name=args.bdev,
-                                    nsid=args.nsid)
+                                    nsid=args.nsid,
+                                    anagrpid=args.anagrpid)
         ret = self.stub.add_namespace(req)
         self.logger.info(
-            f"Added namespace {ret.nsid} to {args.subnqn}: {ret.status}")
+            f"Added namespace {ret.nsid} to {args.subnqn}, ANA group id {args.anagrpid} : {ret.status}")
 
     @cli.cmd([
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),