build-container.yml

name: "CI"
on:
  push:
    branches:
      - '*'
    tags:
      - 'v*'
  pull_request:
  workflow_dispatch:
  release:
    types:
      - created
env:
  WAIT_INTERVAL_SECS: 1
jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          # git submodule update --init --recursive
          submodules: recursive

      - name: Build container images
        run: make build SPDK_DISABLE_VPCLMULQDQ="yes" SPDK_DISABLE_AVX512="yes"

      - name: Save container images
        run: |
          . .env
          docker save $QUAY_NVMEOF:$NVMEOF_VERSION > nvmeof.tar
          docker save $QUAY_NVMEOFCLI:$NVMEOF_VERSION > nvmeof-cli.tar
          docker save $QUAY_CEPH:$CEPH_VERSION  > vstart-cluster.tar
          docker save bdevperf > bdevperf.tar
          docker save nvmeof-devel > nvmeof-devel.tar

      - name: Upload container images
        uses: actions/upload-artifact@v3
        with:
          name: ceph_nvmeof_container_images-${{ github.run_number }}
          path: |
            nvmeof.tar
            nvmeof-cli.tar
            nvmeof-devel.tar
            vstart-cluster.tar
            bdevperf.tar

      - name: Build stand-alone packages (RPMs and Python wheel)
        id: build-standalone-packages
        run: |
          export EXPORT_DIR=$(mktemp -d)
          make export-rpms
          make export-python
          echo "EXPORT_DIR=$EXPORT_DIR" >> "$GITHUB_ENV"

      - name: Upload stand-alone packages
        uses: actions/upload-artifact@v3
        with:
          name: ceph_nvmeof_standalone_packages-${{ github.run_number }}
          path: |
            ${{ env.EXPORT_DIR }}/**

  pytest:
    needs: build
    strategy:
      fail-fast: false
      matrix:
          test: ["cli", "state", "multi_gateway", "server", "grpc", "omap_lock"]
    runs-on: ubuntu-latest
    env:
      HUGEPAGES: 512 # for multi gateway test, approx 256 per gateway instance
    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Setup huge pages
        run: |
          make setup HUGEPAGES=$HUGEPAGES

      - name: Download container images
        uses: actions/download-artifact@v3
        with:
          name: ceph_nvmeof_container_images-${{ github.run_number }}

      - name: Load container images
        run: |
          docker load < nvmeof-devel.tar
          docker load < vstart-cluster.tar

      - name: Clear space on disk
        run: |
          if [[ -d /usr/share/dotnet ]]; then
            /usr/bin/du -sh /usr/share/dotnet
            rm -rf /usr/share/dotnet
          fi
          if [[ -d /opt/ghc ]]; then
            /usr/bin/du -sh /opt/ghc
            rm -rf /opt/ghc
          fi
          if [[ -d /usr/local/share/boost ]]; then
            /usr/bin/du -sh /usr/local/share/boost
            rm -rf /usr/local/share/boost
          fi
          if [[ -n "$AGENT_TOOLSDIRECTORY" ]]; then
            if [[ -d "$AGENT_TOOLSDIRECTORY" ]]; then
              /usr/bin/du -sh "$AGENT_TOOLSDIRECTORY"
              rm -rf "$AGENT_TOOLSDIRECTORY"
            fi
          fi

      - name: Start ceph cluster
        run: |
          make up SVC=ceph OPTS="--detach"

      - name: Wait for the ceph cluster container to become healthy
        timeout-minutes: 3
        run: |
          while true; do
            container_status=$(docker inspect --format='{{.State.Health.Status}}' ceph)
            if [[ $container_status == "healthy" ]]; then
              # success
              exit 0
            else
              # Wait for a specific time before checking again
              sleep ${{ env.WAIT_INTERVAL_SECS }}
              echo -n .
            fi
          done

      - name: Create RBD image
        run: |
          echo "💁 ceph list pools:"
          make exec SVC=ceph OPTS="-T" CMD="ceph osd lspools"
          echo "💁 rbd create:"
          make exec SVC=ceph OPTS="-T" CMD="rbd create rbd/mytestdevimage --size 16"
          echo "💁 ls rbd:"
          make exec SVC=ceph OPTS="-T" CMD="rbd ls rbd"

      - name: Run protoc
        run: |
          make protoc

      - name: Run ${{ matrix.test }} test
        run: |
          # Run tests code in current dir
          # Managing pytest’s output: https://docs.pytest.org/en/7.1.x/how-to/output.html
          make run SVC="nvmeof-devel" OPTS="--volume=$(pwd)/tests:/src/tests --entrypoint=python3" CMD="-m pytest --show-capture=all -s --full-trace -vv -rA tests/test_${{ matrix.test }}.py"

      - name: Check coredump existence
        if: success() || failure()
        id: check_coredumps
        uses: andstor/file-existence-action@20b4d2e596410855db8f9ca21e96fbe18e12930b # v2, pinned to SHA for security reasons
        with:
          files: "/tmp/coredump/core.*"

      - name: Upload ${{ matrix.test }} test core dumps
        if: steps.check_coredumps.outputs.files_exists == 'true'
        uses: actions/upload-artifact@v3
        with:
          name: ceph_nvmeof_pytest_${{ matrix.test }}_cores-${{ github.run_number }}
          path: |
            /tmp/coredump/core.*

      - name: Display logs
        if: success() || failure()
        run: |
          make logs OPTS=""

      - name: Tear down
        if: success() || failure()
        run: |
          make down
          make clean

  demo:
    needs: build
    runs-on: ubuntu-latest
    env:
      HUGEPAGES: 512
    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Setup huge-pages
        run: make setup HUGEPAGES=$HUGEPAGES

      - name: Download container images
        uses: actions/download-artifact@v3
        with:
          name: ceph_nvmeof_container_images-${{ github.run_number }}

      - name: Load container images
        run: |
          docker load < nvmeof.tar
          docker load < nvmeof-cli.tar
          docker load < vstart-cluster.tar
          docker load < bdevperf.tar

      - name: Start containers
        run: |
          make up OPTS=--detach

      - name: Wait for the Gateway to be listening
        timeout-minutes: 3
        run: |
          . .env

          echo using gateway $NVMEOF_IP_ADDRESS port $NVMEOF_GW_PORT
          until nc -z $NVMEOF_IP_ADDRESS $NVMEOF_GW_PORT; do
            echo -n .
            sleep ${{ env.WAIT_INTERVAL_SECS }}
          done

      - name: List containers
        if: success() || failure()
        run: make ps

      - name: List processes
        if: success() || failure()
        run: make top

      - name: Test
        run: |
          make demo OPTS=-T NVMEOF_CONTAINER_NAME="ceph-nvmeof_nvmeof_1"

      - name: Get subsystems
        run: |
          # https://github.com/actions/toolkit/issues/766
          shopt -s expand_aliases
          eval $(make alias)
          nvmeof-cli get_subsystems
          nvmeof-cli-ipv6 get_subsystems

      - name: Run bdevperf
        run: |
          # see https://spdk.io/doc/nvmf_multipath_howto.html
          . .env
          echo -n "ℹ️  Starting bdevperf container"
          make up  SVC=bdevperf OPTS="--detach"
          sleep 10
          echo "ℹ️  bdevperf start up logs"
          make logs SVC=bdevperf
          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' )

          rpc="/usr/libexec/spdk/scripts/rpc.py"
          echo "ℹ️  bdevperf bdev_nvme_set_options"
          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1"
          echo "ℹ️  bdevperf tcp connect ip: $NVMEOF_IP_ADDRESS port: $NVMEOF_IO_PORT nqn: $NQN"
          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_attach_controller -b Nvme0 -t tcp -a $NVMEOF_IP_ADDRESS -s $NVMEOF_IO_PORT -f ipv4 -n $NQN -l -1 -o 10"
          echo "ℹ️  bdevperf perform_tests"
          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' )
          timeout=$(expr $BDEVPERF_TEST_DURATION \* 2)
          bdevperf="/usr/libexec/spdk/scripts/bdevperf.py"
          make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests"

      - name: Check coredump existence
        if: success() || failure()
        id: check_coredumps
        uses: andstor/file-existence-action@20b4d2e596410855db8f9ca21e96fbe18e12930b # v2, pinned to SHA for security reasons
        with:
            files: "/tmp/coredump/core.*"

      - name: Upload demo core dumps
        if: steps.check_coredumps.outputs.files_exists == 'true'
        uses: actions/upload-artifact@v3
        with:
          name: ceph_nvmeof_demo_cores-${{ github.run_number }}
          path: |
            /tmp/coredump/core.*

      # For debugging purposes (provides an SSH connection to the runner)
      #- name: Setup tmate session
      #  uses: mxschmitt/action-tmate@v3
      #  with:
      #    limit-access-to-actor: true

      - name: Display logs
        if: success() || failure()
        run: make logs OPTS=''

      - name: Tear down
        if: success() || failure()
        run: |
          make down
          make clean

  discovery:
    needs: build
    strategy:
      fail-fast: false
      matrix:
          integration: ["container", "embedded"]
    runs-on: ubuntu-latest
    env:
      HUGEPAGES: 768 # 3 spdk instances

    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Setup huge-pages
        run: make setup HUGEPAGES=$HUGEPAGES

      - name: Download container images
        uses: actions/download-artifact@v3
        with:
          name: ceph_nvmeof_container_images-${{ github.run_number }}

      - name: Load container images
        run: |
          docker load < nvmeof.tar
          docker load < nvmeof-cli.tar
          docker load < vstart-cluster.tar
          docker load < bdevperf.tar

      - name: Start discovery controller
        if: matrix.integration == 'container'
        run: |
          docker-compose up --detach discovery

      - name: Wait for discovery controller to be listening
        if: matrix.integration == 'container'
        timeout-minutes: 3
        run: |
          . .env
          container_ip() {
            docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"
          }

          ip=$(container_ip $DISC1)
          echo using discovery controller $ip $NVMEOF_DISC_PORT
          until nc -z $ip $NVMEOF_DISC_PORT; do
            echo -n .
            sleep ${{ env.WAIT_INTERVAL_SECS }}
          done

      - name: Start gateway with scale=2
        run: |
          docker-compose up --detach --scale nvmeof=2 nvmeof

      - name: Wait for gateways to be listening
        timeout-minutes: 3
        run: |
          . .env
          container_ip() {
            docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"
          }

          for gw in $GW1 $GW2; do
            ip=$(container_ip $gw)
            echo using gateway $ip $NVMEOF_GW_PORT
            until nc -z $ip $NVMEOF_GW_PORT; do
              echo -n .
              sleep ${{ env.WAIT_INTERVAL_SECS }}
            done
            echo
          done

      - name: List containers
        if: success() || failure()
        run: |
          docker-compose ps

      - name: List processes
        if: success() || failure()
        run: |
          docker-compose top

      - name: Create RBD image
        run: |
          make rbd OPTS=-T

      - name: Set up target
        run: |
          . .env

          container_ip() {
            docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"
          }
          # container id is the default hostname in docker environent
          # i.e. default gateway name
          container_id() {
            docker ps -q -f name=$1
          }
          cli_gw() {
            gw=$1
            shift
            docker-compose run --rm nvmeof-cli --server-address $gw --server-port $NVMEOF_GW_PORT "$@"
          }

          gw1=$(container_ip $GW1)
          echo ℹ️ Using GW RPC $GW1 address $gw1 port $NVMEOF_GW_PORT
          cli_gw $gw1 get_subsystems
          cli_gw $gw1 create_bdev --pool $RBD_POOL --image $RBD_IMAGE_NAME --bdev $BDEV_NAME
          cli_gw $gw1 create_subsystem --subnqn $NQN --serial $SERIAL
          cli_gw $gw1 add_namespace --subnqn $NQN --bdev $BDEV_NAME
          for gw in $GW1 $GW2; do
            ip=$(container_ip $gw)
            name=$(container_id $gw) # default hostname - container id
            echo ℹ️ Create listener address $ip gateway $name
            cli_gw $ip create_listener --subnqn $NQN --gateway-name $name --traddr $ip --trsvcid $NVMEOF_IO_PORT
          done
          cli_gw $gw1 add_host --subnqn $NQN --host "*"
          for gw in $GW1 $GW2; do
            ip=$(container_ip $gw)
            echo ℹ️ Subsystems for name $gw ip $ip
            cli_gw $ip get_subsystems
          done

      - name: Run bdevperf discovery
        run: |
          # See
          # - https://github.com/spdk/spdk/blob/master/doc/jsonrpc.md
          # - https://spdk.io/doc/nvmf_multipath_howto.html
          . .env
          container_ip() {
            docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"
          }

          echo -n "ℹ️  Starting bdevperf container"
          make up  SVC=bdevperf OPTS="--detach"
          sleep 10
          echo "ℹ️  bdevperf start up logs"
          make logs SVC=bdevperf
          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_SOCKET | tr -d '\n\r' )

          if [ "${{ matrix.integration }}" == "embedded" ]; then
            ip=$(container_ip $GW1)
            echo "ℹ️  Using discovery service in gateway $GW1 ip  $ip"
          else
            ip=$(container_ip $DISC1)
            echo "ℹ️  Using standalone discovery container $DISC1 ip  $ip"
          fi
          rpc="/usr/libexec/spdk/scripts/rpc.py"
          echo "ℹ️  bdevperf bdev_nvme_set_options"
          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_set_options -r -1"
          echo "ℹ️  bdevperf start discovery ip: $ip port: $NVMEOF_DISC_PORT"
          # -l -1 -o 10
          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_start_discovery -b Nvme0 -t tcp -a $ip -s $NVMEOF_DISC_PORT -f ipv4 -w"
          echo "ℹ️  bdevperf bdev_nvme_get_discovery_info"
          make exec SVC=bdevperf OPTS=-T CMD="$rpc -v -s $BDEVPERF_SOCKET bdev_nvme_get_discovery_info"
          echo "ℹ️  bdevperf perform_tests"
          eval $(make run SVC=bdevperf OPTS="--entrypoint=env" | grep BDEVPERF_TEST_DURATION | tr -d '\n\r' )
          timeout=$(expr $BDEVPERF_TEST_DURATION \* 2)
          bdevperf="/usr/libexec/spdk/scripts/bdevperf.py"
          make exec SVC=bdevperf OPTS=-T CMD="$bdevperf -v -t $timeout -s $BDEVPERF_SOCKET perform_tests"

      - name: Check coredump existence
        if: success() || failure()
        id: check_coredumps
        uses: andstor/file-existence-action@20b4d2e596410855db8f9ca21e96fbe18e12930b # v2, pinned to SHA for security reasons
        with:
            files: "/tmp/coredump/core.*"

      - name: Upload demo core dumps
        if: steps.check_coredumps.outputs.files_exists == 'true'
        uses: actions/upload-artifact@v1
        with:
          name: ceph_nvmeof_demo_cores-${{ github.run_number }}
          path: /tmp/coredump/core.*

      - name: Display logs
        if: success() || failure()
        run: make logs OPTS=''

      - name: Tear down
        if: success() || failure()
        run: |
          make down
          make clean

  push-to-registry:
    if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags/v')
    needs: [pytest, demo, discovery]
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Download container images
        uses: actions/download-artifact@v3
        with:
          name: ceph_nvmeof_container_images-${{ github.run_number }}

      - name: Load container images
        run: |
          docker load < nvmeof.tar
          docker load < nvmeof-cli.tar

      - name: Login to quay.io
        uses: docker/login-action@v2
        with:
          registry: ${{ vars.CONTAINER_REGISTRY }}
          username: '${{ vars.CONTAINER_REGISTRY_USERNAME }}'
          password: '${{ secrets.CONTAINER_REGISTRY_PASSWORD }}'

      - name: Publish nvmeof containers when release/tag is created
        run: |
          make push