initial discovery

- container - CI test Signed-off-by: Alexander Indenbaum <[email protected]>
ceph · Sep 6, 2023 · deffe91 · deffe91
1 parent aa34589
commit deffe91
Show file tree

Hide file tree

Showing 6 changed files with 200 additions and 28 deletions.
diff --git a/.env b/.env
@@ -24,7 +24,6 @@ NVMEOF_DESCRIPTION="Service to provide block storage on top of Ceph for platform
 NVMEOF_URL="https://github.com/ceph/ceph-nvmeof"
 NVMEOF_TAGS="ceph,nvme-of,nvme-of gateway,rbd,block storage"
 NVMEOF_WANTS="ceph,rbd"
-NVMEOF_HOSTNAME="nvmeof"
 NVMEOF_IP_ADDRESS="192.168.13.3"
 NVMEOF_IO_PORT=4420
 NVMEOF_GW_PORT=5500
@@ -61,4 +60,9 @@ RBD_IMAGE_NAME="demo_image"
 RBD_IMAGE_SIZE="10M"
 BDEV_NAME="demo_bdev"
 NQN="nqn.2016-06.io.spdk:cnode1"
-NVMEOF_FIRST_GATEWAY_NAME="gw-1"
+SERIAL="SPDK00000000000001"
+
+# Container names in docker compose environent
+DISC1="ceph-nvmeof_discovery_1"
+GW1="ceph-nvmeof_nvmeof_1"
+GW2="ceph-nvmeof_nvmeof_2"
diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml
@@ -181,7 +181,7 @@ jobs:
         run: |
           . .env
 
-          echo using gateway $NVMEOF_IP_ADDRESS port $NVMEOF_GW_PORT timeout ${{ env.WAIT_TIMEOUT_MINS }}
+          echo using gateway $NVMEOF_IP_ADDRESS port $NVMEOF_GW_PORT
           until nc -z $NVMEOF_IP_ADDRESS $NVMEOF_GW_PORT; do
             echo -n .
             sleep ${{ env.WAIT_INTERVAL_SECS }}
@@ -243,21 +243,6 @@ jobs:
           path: |
             /tmp/coredump/core.*
 
-      #- name: Test mounting nvmeof device locally
-      #  run: |
-      #    . .env
-      #    sudo modprobe nvme-fabrics
-      #    sudo nvme list
-      #    sudo nvme discover -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT
-      #    sudo nvme connect -t tcp --traddr $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -n nqn.2016-06.io.spdk:cnode1
-      #    sudo nvme list
-      #    NVMEOF_DEVICE=$(sudo nvme list -o json | jq '.Devices[] | select(.ModelNumber=="SPDK bdev Controller").DevicePath')
-      #    sudo mkfs $NVMEOF_DEVICE
-      #    MOUNT_POINT=$(mktemp -d)
-      #    sudo mount $NVMEOF_DEVICE $MOUNT_POINT
-      #    cd $MOUNT_POINT
-      #    touch test
-
       # For debugging purposes (provides an SSH connection to the runner)
       #- name: Setup tmate session
       #  uses: mxschmitt/action-tmate@v3
@@ -274,18 +259,188 @@ jobs:
           make down
           make clean
 
-  push-to-registry:
-    if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags/v')
-    needs: [pytest, demo]
+  discovery:
+    needs: build
     runs-on: ubuntu-latest
+    env:
+      HUGEPAGES: 768 # 3 spdk instances
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: Setup huge-pages
+        run: make setup HUGEPAGES=$HUGEPAGES
+
       - name: Download container images
         uses: actions/download-artifact@v3
         with:
-          name: images
+          name: ceph_nvmeof_container_images-${{ github.run_number }}
+
+      - name: Load container images
+        run: |
+          docker load < nvmeof.tar
+          docker load < nvmeof-cli.tar
+          docker load < vstart-cluster.tar
+          docker load < bdevperf.tar
+
+      - name: Start discovery controller
+        run: |
+          docker-compose up --detach discovery
+
+      - name: Wait for discovery controller to be listening
+        timeout-minutes: 3
+        run: |
+          . .env
+          container_ip() {
+            docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"
+          }
+
+          ip=$(container_ip $DISC1)
+          echo using discovery controller $ip $NVMEOF_DISC_PORT
+          until nc -z $ip $NVMEOF_DISC_PORT; do
+            echo -n .
+            sleep ${{ env.WAIT_INTERVAL_SECS }}
+          done
+
+      - name: Start gateway with scale=2
+        run: |
+          docker-compose up --detach --scale nvmeof=2 nvmeof
+
+      - name: Wait for gateways to be listening
+        timeout-minutes: 3
+        run: |
+          . .env
+          container_ip() {
+            docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"
+          }
+
+          for gw in $GW1 $GW2; do
+            ip=$(container_ip $gw)
+            echo using gateway $ip $NVMEOF_GW_PORT
+            until nc -z $ip $NVMEOF_GW_PORT; do
+              echo -n .
+              sleep ${{ env.WAIT_INTERVAL_SECS }}
+            done
+            echo
+          done
+
+      - name: List containers
+        if: success() || failure()
+        run: |
+          docker-compose ps
+
+      - name: List processes
+        if: success() || failure()
+        run: |
+          docker-compose top
+
+      - name: Create RBD image
+        run: |
+          make rbd OPTS=-T
+
+      - name: Set up target
+        run: |
+          . .env
+
+          container_ip() {
+            docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"
+          }
+          # container id is the default hostname in docker environent
+          # i.e. default gateway name
+          container_id() {
+            docker ps -q -f name=$1
+          }
+          cli_gw() {
+            gw=$1
+            shift
+            docker-compose run --rm nvmeof-cli --server-address $gw --server-port $NVMEOF_GW_PORT "$@"
+          }
+
+          gw1=$(container_ip $GW1)
+          echo ℹ️ Using GW RPC $GW1 address $gw1 port $NVMEOF_GW_PORT
+          cli_gw $gw1 get_subsystems
+          cli_gw $gw1 create_bdev --pool $RBD_POOL --image $RBD_IMAGE_NAME --bdev $BDEV_NAME
+          cli_gw $gw1 create_subsystem --subnqn $NQN --serial $SERIAL
+          cli_gw $gw1 add_namespace --subnqn $NQN --bdev $BDEV_NAME
+          for gw in $GW1 $GW2; do
+            ip=$(container_ip $gw)
+            name=$(container_id $gw) # default hostname - container id
+            echo ℹ️ Create listener address $ip gateway $name
+            cli_gw $ip create_listener --subnqn $NQN --gateway-name $name --traddr $ip --trsvcid $NVMEOF_IO_PORT
+          done
+          cli_gw $gw1 add_host --subnqn $NQN --host "*"
+          for gw in $GW1 $GW2; do
+            ip=$(container_ip $gw)
+            echo ℹ️ Subsystems for name $gw ip $ip
+            cli_gw $ip get_subsystems
+          done
+
+      - name: Run bdevperf discovery
+        run: |
+          # See
+          # - https://github.com/spdk/spdk/blob/master/doc/jsonrpc.md
+          # - https://spdk.io/doc/nvmf_multipath_howto.html
+          . .env
+          container_ip() {
+            docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"
+          }
+
+          echo -n "ℹ️  Starting bdevperf container"
+          make up  SVC=bdevperf OPTS="--detach"
+          sleep 10
+          echo "ℹ️  bdevperf start up logs"
+          make logs SVC=bdevperf
+          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' )
+          ip=$(container_ip $DISC1)
+          rpc="/usr/libexec/spdk/scripts/rpc.py"
+          echo "ℹ️  bdevperf bdev_nvme_set_options"
+          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1"
+          echo "ℹ️  bdevperf start discovery ip: $ip port: $NVMEOF_DISC_PORT"
+          # -l -1 -o 10
+          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_start_discovery -b Nvme0 -t tcp -a $ip -s $NVMEOF_DISC_PORT -f ipv4 -w"
+          echo "ℹ️  bdevperf bdev_nvme_get_discovery_info"
+          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_get_discovery_info"
+          echo "ℹ️  bdevperf perform_tests"
+          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' )
+          timeout=$(expr $BDEVPERF_TEST_DURATION \* 2)
+          bdevperf="/usr/libexec/spdk/scripts/bdevperf.py"
+          make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests"
+
+      - name: Check coredump existence
+        if: success() || failure()
+        id: check_coredumps
+        uses: andstor/file-existence-action@20b4d2e596410855db8f9ca21e96fbe18e12930b # v2, pinned to SHA for security reasons
+        with:
+            files: "/var/lib/systemd/coredump/*"
+
+      - name: Upload demo core dumps
+        if: steps.check_coredumps.outputs.files_exists == 'true'
+        uses: actions/upload-artifact@v1
+        with:
+          name: ceph_nvmeof_demo_cores-${{ github.run_number }}
+          path: /var/lib/systemd/coredump/*
+
+      - name: Display logs
+        if: success() || failure()
+        run: make logs OPTS=''
+
+      - name: Tear down
+        if: success() || failure()
+        run: |
+          make down
+          make clean
+
+  push-to-registry:
+    if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags/v')
+    needs: [pytest, demo, discovery]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Download container images
+        uses: actions/download-artifact@v3
+        with:
+          name: ceph_nvmeof_container_images-${{ github.run_number }}
 
       - name: Load container images
         run: |
@@ -302,4 +457,3 @@ jobs:
       - name: Publish nvmeof containers when release/tag is created
         run: |
           make push
-
diff --git a/Makefile b/Makefile
@@ -22,7 +22,7 @@ setup: ## Configure huge-pages (requires sudo/root password)
 	@echo Actual Hugepages allocation: $$(cat $(HUGEPAGES_DIR))
 	@[ $$(cat $(HUGEPAGES_DIR)) -eq $(HUGEPAGES) ]
 
-build pull logs: SVC ?= spdk bdevperf nvmeof nvmeof-devel nvmeof-cli ceph
+build pull logs: SVC ?= spdk bdevperf nvmeof nvmeof-devel nvmeof-cli discovery ceph
 
 build: export NVMEOF_GIT_BRANCH != git name-rev --name-only HEAD
 build: export NVMEOF_GIT_COMMIT != git rev-parse HEAD
@@ -34,7 +34,7 @@ build: export BUILD_DATE != date -u +"%Y-%m-%dT%H:%M:%SZ"
 up: ## Launch services
 up: SVC ?= ceph nvmeof ## Services
 up: OPTS ?= --abort-on-container-exit --exit-code-from $(SVC) --remove-orphans
-up: override OPTS += --scale nvmeof=$(SCALE)
+#up: override OPTS += --scale nvmeof=$(SCALE)
 
 clean: $(CLEAN) setup  ## Clean-up environment
 clean: override HUGEPAGES = 0

diff --git a/ceph-nvmeof.conf b/ceph-nvmeof.conf
@@ -20,7 +20,6 @@ state_update_interval_sec = 5
 enable_discovery_controller = false
 
 [discovery]
-
 addr = 0.0.0.0
 port = 8009
 debug = 20

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -99,16 +99,15 @@ services:
         NVMEOF_GIT_COMMIT:
       labels:
         io.ceph.nvmeof:
-    hostname: nvmeof
     volumes:
       # sudo bash -c 'echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'
       # https://spdk.io/doc/containers.html
       # TODO: Pending of https://github.com/spdk/spdk/issues/2973
       - /dev/hugepages:/dev/hugepages
       - /dev/vfio/vfio:/dev/vfio/vfio
-      - ceph-conf:/etc/ceph:ro
       - $NVMEOF_CONFIG:/src/ceph-nvmeof.conf
       - /tmp/coredump:/tmp/coredump # core dump
+      - ceph-conf:/etc/ceph:ro
     cap_add:
       - SYS_ADMIN # huge-pages
       - CAP_SYS_NICE # RTE
@@ -127,9 +126,23 @@ services:
     extends:
       service: nvmeof-base
     image: $QUAY_NVMEOF:$NVMEOF_VERSION
+    ports:
+      - "$NVMEOF_IO_PORT" # I/O controllers
+      - "$NVMEOF_GW_PORT" # Gateway
+    depends_on:
+      ceph:
+        condition: service_healthy
+  discovery:
+    extends:
+      service: nvmeof-base
+    image: $QUAY_NVMEOF:$NVMEOF_VERSION
+    ports:
+      - "$NVMEOF_DISC_PORT" # Discovery
     depends_on:
       ceph:
         condition: service_healthy
+    entrypoint: >-
+      python3 -m control.discovery
   # Used to update lockfile (make update-lockfile)
   nvmeof-builder-base:
     extends:

diff --git a/mk/demo.mk b/mk/demo.mk
@@ -6,6 +6,8 @@ rbd: SVC = ceph
 rbd: CMD = bash -c "rbd -p $(RBD_POOL) info $(RBD_IMAGE_NAME) || rbd -p $(RBD_POOL) create $(RBD_IMAGE_NAME) --size $(RBD_IMAGE_SIZE)"
 
 # demo
+# the fist gateway in docker enviroment, hostname defaults to container id
+demo: export NVMEOF_HOSTNAME != docker ps -q -f name=ceph-nvmeof_nvmeof_1
 demo: rbd ## Expose RBD_IMAGE_NAME as NVMe-oF target
 	$(NVMEOF_CLI) create_bdev --pool $(RBD_POOL) --image $(RBD_IMAGE_NAME) --bdev $(BDEV_NAME)
 	$(NVMEOF_CLI) create_subsystem --subnqn $(NQN)