docker: support target-arch in SPDK

* Remove exising handling for AVX512*, VPCLMULQDQ and RDSEED instructions. * Replace it with spdk/configure --target-arc=<arch> * Tune SPDK's yum/dnf cache and settings. Signed-off-by: Ernesto Puerta <[email protected]>
ceph · Jan 19, 2024 · 737878f · 737878f
1 parent 5db106d
commit 737878f
Show file tree

Hide file tree

Showing 5 changed files with 112 additions and 74 deletions.
diff --git a/.env b/.env
@@ -48,16 +48,16 @@ SPDK_DESCRIPTION="The Storage Performance Development Kit (SPDK) provides a set
 SPDK_URL="https://spdk.io"
 
 SPDK_PKGDEP_ARGS="--rbd"
+# check spdk/configure --help
 SPDK_CONFIGURE_ARGS="--with-rbd --disable-tests --disable-unit-tests --disable-examples --enable-debug"
+SPDK_TARGET_ARCH="x86-64-v2"
 SPDK_MAKEFLAGS=
-SPDK_DISABLE_VPCLMULQDQ=
-SPDK_DISABLE_AVX512=
 SPDK_CENTOS_BASE="https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/"
 SPDK_CENTOS_REPO_VER="9.0-21.el9"
 
 # Ceph Cluster
 CEPH_CLUSTER_VERSION="${CEPH_VERSION}"
-# CEPH_CLUSTER_CEPH_REPO_BASEURL= # Uncomment for specifying a custom Ceph YUM repo
+#CEPH_CLUSTER_CEPH_REPO_BASEURL=
 CEPH_VSTART_ARGS="--memstore"
 CEPH_DEVEL_MGR_PATH=../ceph
 

diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml
@@ -1,11 +1,13 @@
 name: "CI"
 on:
   push:
-    branches:
-      - '*'
     tags:
       - 'v*'
   pull_request:
+    branches:
+      - devel
+  schedule:
+    - cron: '0 0 * * *'
   workflow_dispatch:
   release:
     types:
@@ -22,8 +24,26 @@ jobs:
           # git submodule update --init --recursive
           submodules: recursive
 
-      - name: Build container images
-        run: make build SPDK_DISABLE_VPCLMULQDQ="yes" SPDK_DISABLE_AVX512="yes"
+      - name: Build container images - ceph
+        run: make build SVC=ceph SPDK_TARGET_ARCH=x86-64-v2
+
+      - name: Build container images - spdk
+        run: make build SVC="spdk" SPDK_TARGET_ARCH=x86-64-v2
+
+      - name: Build container images - bdevperf
+        run: make build SVC="bdevperf" SPDK_TARGET_ARCH=x86-64-v2
+
+      - name: Build container images - nvmeof
+        run: make build SVC="nvmeof" SPDK_TARGET_ARCH=x86-64-v2
+
+      - name: Build container images - nvmeof-devel
+        run: make build SVC="nvmeof-devel" SPDK_TARGET_ARCH=x86-64-v2
+
+      - name: Build container images - nvmeof-cli
+        run: make build SVC="nvmeof-cli" SPDK_TARGET_ARCH=x86-64-v2
+
+      - name: Build container images - discovery
+        run: make build SVC="discovery" SPDK_TARGET_ARCH=x86-64-v2
 
       - name: Save container images
         run: |
@@ -45,21 +65,6 @@ jobs:
             vstart-cluster.tar
             bdevperf.tar
 
-      - name: Build stand-alone packages (RPMs and Python wheel)
-        id: build-standalone-packages
-        run: |
-          export EXPORT_DIR=$(mktemp -d)
-          make export-rpms
-          make export-python
-          echo "EXPORT_DIR=$EXPORT_DIR" >> "$GITHUB_ENV"
-
-      - name: Upload stand-alone packages
-        uses: actions/upload-artifact@v3
-        with:
-          name: ceph_nvmeof_standalone_packages-${{ github.run_number }}
-          path: |
-            ${{ env.EXPORT_DIR }}/**
-
   pytest:
     needs: build
     strategy:

diff --git a/Dockerfile.ceph b/Dockerfile.ceph
@@ -3,25 +3,9 @@
 FROM quay.io/centos/centos:stream9-minimal AS build
 
 ARG CEPH_CLUSTER_VERSION \
-    CEPH_CLUSTER_CEPH_REPO_BASEURL="https://download.ceph.com/rpm-$CEPH_CLUSTER_VERSION/el\$releasever" \
+    CEPH_CLUSTER_CEPH_REPO_BASEURL="https://download.ceph.com/rpm-${CEPH_CLUSTER_VERSION}/el\$releasever" \
     CEPH_CLUSTER_EPEL_REPO_URL="https://copr.fedorainfracloud.org/coprs/ceph/el9/repo/epel-9/ceph-el9-epel-9.repo"
 
-COPY <<EOF /etc/yum.repos.d/ceph.repo
-[Ceph]
-name=Ceph packages for \$basearch
-baseurl=${CEPH_CLUSTER_CEPH_REPO_BASEURL:?}/\$basearch
-enabled=1
-priority=2
-gpgcheck=0
-
-[Ceph-noarch]
-name=Ceph noarch packages
-baseurl=${CEPH_CLUSTER_CEPH_REPO_BASEURL:?}/noarch
-enabled=1
-priority=2
-gpgcheck=0
-EOF
-
 ARG MICRODNF_OPTS="\
     --enablerepo crb \
     --nobest \
@@ -58,9 +42,35 @@ ARG DEBUG_PACKAGES="\
     lsof \
     "
 
+RUN <<EOF
+    echo Log variables
+    echo ======================================================================
+    echo CEPH_CLUSTER_CEPH_REPO_BASEURL=$CEPH_CLUSTER_CEPH_REPO_BASEURL
+    echo CEPH_CLUSTER_VERSION=$CEPH_CLUSTER_VERSION
+    echo ======================================================================
+EOF
+
+COPY <<EOF /etc/yum.repos.d/ceph.repo
+[Ceph]
+name=Ceph packages for \$basearch
+baseurl=${CEPH_CLUSTER_CEPH_REPO_BASEURL}/\$basearch
+enabled=1
+priority=2
+gpgcheck=0
+
+[Ceph-noarch]
+name=Ceph noarch packages
+baseurl=${CEPH_CLUSTER_CEPH_REPO_BASEURL}/noarch
+enabled=1
+priority=2
+gpgcheck=0
+EOF
+
+RUN cat /etc/yum.repos.d/ceph.repo
+
 RUN rpm -vih https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
 RUN rpm --import 'https://download.ceph.com/keys/release.asc'
-RUN curl -O --output-dir /etc/yum.repos.d/ $CEPH_CLUSTER_EPEL_REPO_URL
+RUN curl -O --output-dir /etc/yum.repos.d/ ${CEPH_CLUSTER_EPEL_REPO_URL:?}
 
 RUN --mount=type=cache,target=/var/cache/microdnf \
     microdnf install -y $MICRODNF_OPTS \
@@ -106,7 +116,8 @@ RUN ln -sf $EC_PATH/* $CEPH_LIB && \
 
 USER ceph
 WORKDIR /ceph
-ADD --chown=ceph:ceph --chmod=755 https://raw.githubusercontent.com/ceph/ceph/v$CEPH_CLUSTER_VERSION/src/vstart.sh .
+ADD --chown=ceph:ceph --chmod=755 \
+    https://raw.githubusercontent.com/ceph/ceph/v${CEPH_CLUSTER_VERSION:?}/src/vstart.sh .
 
 COPY <<EOF ./CMakeCache.txt
 ceph_SOURCE_DIR:STATIC=/ceph

diff --git a/Dockerfile.spdk b/Dockerfile.spdk
@@ -1,14 +1,55 @@
 # syntax = docker/dockerfile:1.4
 # vim: syntax=dockerfile
+
+ARG DNF_OPTS="\
+  --enablerepo crb \
+  --nobest \
+  --nodocs \
+  --setopt=install_weak_deps=0 \
+  --setopt=keepcache=1 \
+  --setopt=cachedir=/var/cache/dnf \
+  "
+
 FROM quay.io/centos/centos:stream9 AS build
 
 ARG SPDK_CEPH_VERSION \
-    SPDK_VERSION
+    SPDK_VERSION \
+    DNF_OPTS \
+    SPDK_PKGDEP_ARGS \
+    SPDK_CONFIGURE_ARGS \
+    SPDK_TARGET_ARCH \
+    SPDK_MAKEFLAGS
+
+RUN <<EOF
+    echo Log variables
+    echo ======================================================================
+    echo SPDK_PKGDEP_ARGS=$SPDK_PKGDEP_ARGS
+    echo SPDK_CONFIGURE_ARGS=$SPDK_CONFIGURE_ARGS
+    echo SPDK_TARGET_ARCH=$SPDK_TARGET_ARCH
+    echo SPDK_MAKEFLAGS=$SPDK_MAKEFLAGS
+    echo ======================================================================
+EOF
+
+WORKDIR /src
+COPY . .
+
+COPY <<EOF /etc/yum.conf
+[main]
+gpgcheck=1
+installonly_limit=3
+clean_requirements_on_remove=True
+best=False
+skip_if_unavailable=False
+cachedir=/var/cache/dnf
+install_weak_deps=0
+keepcache=True
+tsflags=nodocs
+EOF
 
 COPY <<EOF /etc/yum.repos.d/ceph.repo
 [Ceph]
 name=Ceph packages for \$basearch
-baseurl=https://download.ceph.com/rpm-${SPDK_CEPH_VERSION?}/el\$releasever/\$basearch
+baseurl=https://download.ceph.com/rpm-${SPDK_CEPH_VERSION:?}/el\$releasever/\$basearch
 enabled=1
 priority=2
 gpgcheck=1
@@ -23,47 +64,27 @@ gpgcheck=1
 gpgkey=https://download.ceph.com/keys/release.asc
 EOF
 
-ARG SPDK_PKGDEP_ARGS \
-    SPDK_CONFIGURE_ARGS \
-    SPDK_MAKEFLAGS \
-    SPDK_DISABLE_VPCLMULQDQ \
-    SPDK_DISABLE_AVX512
-
-WORKDIR /src
-COPY . .
-
 RUN \
     --mount=type=cache,target=/var/cache/dnf \
     --mount=type=cache,target=/var/lib/dnf \
     --mount=type=cache,target=/root/.cache/pip \
-    dnf install -y 'dnf-command(config-manager)' \
+    dnf install $DNF_OPTS -y 'dnf-command(config-manager)' \
     && dnf config-manager --set-enabled crb \
-    && dnf install -y \
+    && dnf install $DNF_OPTS -y \
         rpm-build \
         git-core \
     && rpm -vih https://buildlogs.centos.org/centos/9-stream/storage/x86_64/ceph-reef/Packages/t/thrift-0.15.0-3.el9s.x86_64.rpm \
     && scripts/pkgdep.sh $SPDK_PKGDEP_ARGS \
-    && dnf update -y
-
-# Disable RDSEED, see https://github.com/ceph/ceph-nvmeof/issues/259
-RUN \
-    sed -i "s/^\( \+'RDSEED'\)/#\1/" dpdk/config/x86/meson.build
-# If we run from github disable also VPCLMULQDQ as some of github's runners don't have it
-RUN \
-    if [[ -n "$SPDK_DISABLE_VPCLMULQDQ" ]]; then sed -i "s/^\( \+'VPCLMULQDQ'\)/#\1/" dpdk/config/x86/meson.build ; fi
-RUN \
-    if [[ -n "$SPDK_DISABLE_AVX512" ]]; then sed -i "s/^\( \+'AVX512\)/#\1/" dpdk/config/x86/meson.build ; fi
-RUN \
-    cat dpdk/config/x86/meson.build
+    && dnf $DNF_OPTS update -y
 
 RUN \
     --mount=type=cache,target=/var/cache/dnf \
     --mount=type=cache,target=/var/lib/dnf \
     DEPS="no" \
-    SPDK_VERSION=${SPDK_VERSION?} \
+    SPDK_VERSION=${SPDK_VERSION:?} \
     RPM_RELEASE=0 \
     MAKEFLAGS=$SPDK_MAKEFLAGS \
-    rpmbuild/rpm.sh $SPDK_CONFIGURE_ARGS
+    rpmbuild/rpm.sh $SPDK_CONFIGURE_ARGS --target-arch="${SPDK_TARGET_ARCH:?}"
 
 # build bdevperf example, will not be a part of generated rpm
 RUN make -C ./examples/bdev/bdevperf
@@ -76,7 +97,8 @@ COPY --from=build /root/rpmbuild/rpm /rpm
 FROM registry.access.redhat.com/ubi9/ubi as spdk
 
 ARG SPDK_CEPH_VERSION \
-    SPDK_VERSION
+    SPDK_VERSION \
+    DNF_OPTS
 
 ARG SPDK_CENTOS_BASE="https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/Packages/"
 # This would become obsolete as the release rolls out new packages
@@ -121,6 +143,7 @@ LABEL io.spdk.version="$SPDK_VERSION" \
 LABEL org.centos.url="$SPDK_CENTOS_BASE" \
       org.centos.version="$SPDK_CENTOS_REPO_VER"
 
+COPY --from=build /etc/yum.conf /etc/yum.conf
 COPY --from=build /etc/yum.repos.d/ceph.repo /etc/yum.repos.d/ceph.repo
 
 RUN \
@@ -129,9 +152,9 @@ RUN \
     --mount=type=cache,target=/var/lib/dnf \
     rpm -vih $SPDK_CENTOS_BASE/centos-stream-repos-$SPDK_CENTOS_REPO_VER.noarch.rpm \
              $SPDK_CENTOS_BASE/centos-gpg-keys-$SPDK_CENTOS_REPO_VER.noarch.rpm \
-             https://buildlogs.centos.org/centos/9-stream/storage/x86_64/ceph-reef/Packages/t/thrift-0.15.0-3.el9s.x86_64.rpm \
-    && dnf install -y /rpm/$(uname -m)/*.rpm \
-    && dnf update -y
+    && rpm -vih https://buildlogs.centos.org/centos/9-stream/storage/x86_64/ceph-reef/Packages/t/thrift-0.15.0-3.el9s.x86_64.rpm \
+    && dnf $DNF_OPTS install -y /rpm/$(uname -m)/*.rpm \
+    && dnf $DNF_OPTS update -y
 
 ENTRYPOINT [ "/usr/local/bin/nvmf_tgt" ]
 CMD [ "-u", "-r", "/var/tmp/spdk.sock" ]

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -13,14 +13,13 @@ services:
         SPDK_CEPH_VERSION:
         SPDK_PKGDEP_ARGS:
         SPDK_CONFIGURE_ARGS:
+        SPDK_TARGET_ARCH:
         SPDK_MAKEFLAGS:
         SPDK_NAME:
         SPDK_SUMMARY:
         SPDK_DESCRIPTION:
         SPDK_URL:
         SPDK_MAINTAINER: $MAINTAINER
-        SPDK_DISABLE_VPCLMULQDQ:
-        SPDK_DISABLE_AVX512:
         BUILD_DATE:
         SPDK_GIT_REPO:
         SPDK_GIT_BRANCH: