diff --git a/.env b/.env index 787c843f..a7dbc404 100644 --- a/.env +++ b/.env @@ -24,7 +24,6 @@ NVMEOF_DESCRIPTION="Service to provide block storage on top of Ceph for platform NVMEOF_URL="https://github.com/ceph/ceph-nvmeof" NVMEOF_TAGS="ceph,nvme-of,nvme-of gateway,rbd,block storage" NVMEOF_WANTS="ceph,rbd" -NVMEOF_HOSTNAME="nvmeof" NVMEOF_IP_ADDRESS="192.168.13.3" NVMEOF_IO_PORT=4420 NVMEOF_GW_PORT=5500 @@ -61,4 +60,9 @@ RBD_IMAGE_NAME="demo_image" RBD_IMAGE_SIZE="10M" BDEV_NAME="demo_bdev" NQN="nqn.2016-06.io.spdk:cnode1" -NVMEOF_FIRST_GATEWAY_NAME="gw-1" +SERIAL="SPDK00000000000001" + +# Container names in docker compose environent +DISC1="ceph-nvmeof_discovery_1" +GW1="ceph-nvmeof_nvmeof_1" +GW2="ceph-nvmeof_nvmeof_2" diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 5287c912..ebd58bed 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -181,7 +181,7 @@ jobs: run: | . .env - echo using gateway $NVMEOF_IP_ADDRESS port $NVMEOF_GW_PORT timeout ${{ env.WAIT_TIMEOUT_MINS }} + echo using gateway $NVMEOF_IP_ADDRESS port $NVMEOF_GW_PORT until nc -z $NVMEOF_IP_ADDRESS $NVMEOF_GW_PORT; do echo -n . sleep ${{ env.WAIT_INTERVAL_SECS }} @@ -243,21 +243,6 @@ jobs: path: | /tmp/coredump/core.* - #- name: Test mounting nvmeof device locally - # run: | - # . .env - # sudo modprobe nvme-fabrics - # sudo nvme list - # sudo nvme discover -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT - # sudo nvme connect -t tcp --traddr $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -n nqn.2016-06.io.spdk:cnode1 - # sudo nvme list - # NVMEOF_DEVICE=$(sudo nvme list -o json | jq '.Devices[] | select(.ModelNumber=="SPDK bdev Controller").DevicePath') - # sudo mkfs $NVMEOF_DEVICE - # MOUNT_POINT=$(mktemp -d) - # sudo mount $NVMEOF_DEVICE $MOUNT_POINT - # cd $MOUNT_POINT - # touch test - # For debugging purposes (provides an SSH connection to the runner) #- name: Setup tmate session # uses: mxschmitt/action-tmate@v3 @@ -274,18 +259,188 @@ jobs: make down make clean - push-to-registry: - if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags/v') - needs: [pytest, demo] + discovery: + needs: build runs-on: ubuntu-latest + env: + HUGEPAGES: 768 # 3 spdk instances + steps: - name: Checkout code uses: actions/checkout@v3 + - name: Setup huge-pages + run: make setup HUGEPAGES=$HUGEPAGES + - name: Download container images uses: actions/download-artifact@v3 with: - name: images + name: ceph_nvmeof_container_images-${{ github.run_number }} + + - name: Load container images + run: | + docker load < nvmeof.tar + docker load < nvmeof-cli.tar + docker load < vstart-cluster.tar + docker load < bdevperf.tar + + - name: Start discovery controller + run: | + docker-compose up --detach discovery + + - name: Wait for discovery controller to be listening + timeout-minutes: 3 + run: | + . .env + container_ip() { + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1" + } + + ip=$(container_ip $DISC1) + echo using discovery controller $ip $NVMEOF_DISC_PORT + until nc -z $ip $NVMEOF_DISC_PORT; do + echo -n . + sleep ${{ env.WAIT_INTERVAL_SECS }} + done + + - name: Start gateway with scale=2 + run: | + docker-compose up --detach --scale nvmeof=2 nvmeof + + - name: Wait for gateways to be listening + timeout-minutes: 3 + run: | + . .env + container_ip() { + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1" + } + + for gw in $GW1 $GW2; do + ip=$(container_ip $gw) + echo using gateway $ip $NVMEOF_GW_PORT + until nc -z $ip $NVMEOF_GW_PORT; do + echo -n . + sleep ${{ env.WAIT_INTERVAL_SECS }} + done + echo + done + + - name: List containers + if: success() || failure() + run: | + docker-compose ps + + - name: List processes + if: success() || failure() + run: | + docker-compose top + + - name: Create RBD image + run: | + make rbd OPTS=-T + + - name: Set up target + run: | + . .env + + container_ip() { + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1" + } + # container id is the default hostname in docker environent + # i.e. default gateway name + container_id() { + docker ps -q -f name=$1 + } + cli_gw() { + gw=$1 + shift + docker-compose run --rm nvmeof-cli --server-address $gw --server-port $NVMEOF_GW_PORT "$@" + } + + gw1=$(container_ip $GW1) + echo ℹ️ Using GW RPC $GW1 address $gw1 port $NVMEOF_GW_PORT + cli_gw $gw1 get_subsystems + cli_gw $gw1 create_bdev --pool $RBD_POOL --image $RBD_IMAGE_NAME --bdev $BDEV_NAME + cli_gw $gw1 create_subsystem --subnqn $NQN --serial $SERIAL + cli_gw $gw1 add_namespace --subnqn $NQN --bdev $BDEV_NAME + for gw in $GW1 $GW2; do + ip=$(container_ip $gw) + name=$(container_id $gw) # default hostname - container id + echo ℹ️ Create listener address $ip gateway $name + cli_gw $ip create_listener --subnqn $NQN --gateway-name $name --traddr $ip --trsvcid $NVMEOF_IO_PORT + done + cli_gw $gw1 add_host --subnqn $NQN --host "*" + for gw in $GW1 $GW2; do + ip=$(container_ip $gw) + echo ℹ️ Subsystems for name $gw ip $ip + cli_gw $ip get_subsystems + done + + - name: Run bdevperf discovery + run: | + # See + # - https://github.com/spdk/spdk/blob/master/doc/jsonrpc.md + # - https://spdk.io/doc/nvmf_multipath_howto.html + . .env + container_ip() { + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1" + } + + echo -n "ℹ️ Starting bdevperf container" + make up SVC=bdevperf OPTS="--detach" + sleep 10 + echo "ℹ️ bdevperf start up logs" + make logs SVC=bdevperf + eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' ) + ip=$(container_ip $DISC1) + rpc="/usr/libexec/spdk/scripts/rpc.py" + echo "ℹ️ bdevperf bdev_nvme_set_options" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1" + echo "ℹ️ bdevperf start discovery ip: $ip port: $NVMEOF_DISC_PORT" + # -l -1 -o 10 + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_start_discovery -b Nvme0 -t tcp -a $ip -s $NVMEOF_DISC_PORT -f ipv4 -w" + echo "ℹ️ bdevperf bdev_nvme_get_discovery_info" + make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_get_discovery_info" + echo "ℹ️ bdevperf perform_tests" + eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' ) + timeout=$(expr $BDEVPERF_TEST_DURATION \* 2) + bdevperf="/usr/libexec/spdk/scripts/bdevperf.py" + make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests" + + - name: Check coredump existence + if: success() || failure() + id: check_coredumps + uses: andstor/file-existence-action@20b4d2e596410855db8f9ca21e96fbe18e12930b # v2, pinned to SHA for security reasons + with: + files: "/var/lib/systemd/coredump/*" + + - name: Upload demo core dumps + if: steps.check_coredumps.outputs.files_exists == 'true' + uses: actions/upload-artifact@v1 + with: + name: ceph_nvmeof_demo_cores-${{ github.run_number }} + path: /var/lib/systemd/coredump/* + + - name: Display logs + if: success() || failure() + run: make logs OPTS='' + + - name: Tear down + if: success() || failure() + run: | + make down + make clean + + push-to-registry: + if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags/v') + needs: [pytest, demo, discovery] + runs-on: ubuntu-latest + + steps: + - name: Download container images + uses: actions/download-artifact@v3 + with: + name: ceph_nvmeof_container_images-${{ github.run_number }} - name: Load container images run: | @@ -302,4 +457,3 @@ jobs: - name: Publish nvmeof containers when release/tag is created run: | make push - diff --git a/Makefile b/Makefile index 1715f3c4..db01c41a 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ setup: ## Configure huge-pages (requires sudo/root password) @echo Actual Hugepages allocation: $$(cat $(HUGEPAGES_DIR)) @[ $$(cat $(HUGEPAGES_DIR)) -eq $(HUGEPAGES) ] -build pull logs: SVC ?= spdk bdevperf nvmeof nvmeof-devel nvmeof-cli ceph +build pull logs: SVC ?= spdk bdevperf nvmeof nvmeof-devel nvmeof-cli discovery ceph build: export NVMEOF_GIT_BRANCH != git name-rev --name-only HEAD build: export NVMEOF_GIT_COMMIT != git rev-parse HEAD @@ -34,7 +34,7 @@ build: export BUILD_DATE != date -u +"%Y-%m-%dT%H:%M:%SZ" up: ## Launch services up: SVC ?= ceph nvmeof ## Services up: OPTS ?= --abort-on-container-exit --exit-code-from $(SVC) --remove-orphans -up: override OPTS += --scale nvmeof=$(SCALE) +#up: override OPTS += --scale nvmeof=$(SCALE) clean: $(CLEAN) setup ## Clean-up environment clean: override HUGEPAGES = 0 diff --git a/ceph-nvmeof.conf b/ceph-nvmeof.conf index c895d6fc..f05cd065 100644 --- a/ceph-nvmeof.conf +++ b/ceph-nvmeof.conf @@ -20,7 +20,6 @@ state_update_interval_sec = 5 enable_discovery_controller = false [discovery] - addr = 0.0.0.0 port = 8009 debug = 20 diff --git a/docker-compose.yaml b/docker-compose.yaml index 7c518080..2a6828c2 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -99,16 +99,15 @@ services: NVMEOF_GIT_COMMIT: labels: io.ceph.nvmeof: - hostname: nvmeof volumes: # sudo bash -c 'echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages' # https://spdk.io/doc/containers.html # TODO: Pending of https://github.com/spdk/spdk/issues/2973 - /dev/hugepages:/dev/hugepages - /dev/vfio/vfio:/dev/vfio/vfio - - ceph-conf:/etc/ceph:ro - $NVMEOF_CONFIG:/src/ceph-nvmeof.conf - /tmp/coredump:/tmp/coredump # core dump + - ceph-conf:/etc/ceph:ro cap_add: - SYS_ADMIN # huge-pages - CAP_SYS_NICE # RTE @@ -127,9 +126,23 @@ services: extends: service: nvmeof-base image: $QUAY_NVMEOF:$NVMEOF_VERSION + ports: + - "$NVMEOF_IO_PORT" # I/O controllers + - "$NVMEOF_GW_PORT" # Gateway + depends_on: + ceph: + condition: service_healthy + discovery: + extends: + service: nvmeof-base + image: $QUAY_NVMEOF:$NVMEOF_VERSION + ports: + - "$NVMEOF_DISC_PORT" # Discovery depends_on: ceph: condition: service_healthy + entrypoint: >- + python3 -m control.discovery # Used to update lockfile (make update-lockfile) nvmeof-builder-base: extends: diff --git a/mk/demo.mk b/mk/demo.mk index cbeaf12a..4f2f9c80 100644 --- a/mk/demo.mk +++ b/mk/demo.mk @@ -6,6 +6,8 @@ rbd: SVC = ceph rbd: CMD = bash -c "rbd -p $(RBD_POOL) info $(RBD_IMAGE_NAME) || rbd -p $(RBD_POOL) create $(RBD_IMAGE_NAME) --size $(RBD_IMAGE_SIZE)" # demo +# the fist gateway in docker enviroment, hostname defaults to container id +demo: export NVMEOF_HOSTNAME != docker ps -q -f name=ceph-nvmeof_nvmeof_1 demo: rbd ## Expose RBD_IMAGE_NAME as NVMe-oF target $(NVMEOF_CLI) create_bdev --pool $(RBD_POOL) --image $(RBD_IMAGE_NAME) --bdev $(BDEV_NAME) $(NVMEOF_CLI) create_subsystem --subnqn $(NQN)