daos-stack · brianjmurrell · Feb 2, 2023 · Feb 2, 2023 · Feb 2, 2023 · Feb 2, 2023
@@ -1,5 +1,5 @@
 #!/usr/bin/groovy
-/* Copyright (C) 2020 Intel Corporation
+/* Copyright (C) 2019-2023 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -38,7 +38,7 @@
  */
 // To use a test branch (i.e. PR) until it lands to master
 // I.e. for testing library changes
-//@Library(value="pipeline-lib@your_branch") _
-
-packageBuildingPipelineDAOSTest(['distros' : ['el8', 'leap15'], 'test-tag': 'mfu'])
+@Library(value="pipeline-lib@bmurrell/el9") _
 
+packageBuildingPipelineDAOSTest(['distros' : ['el8', 'el9', 'leap15', 'ubuntu20.04'],
+                                 'test-tag': 'mfu'])
@@ -0,0 +1,5 @@
+mpifileutils (0.11.1-4) focal; urgency=medium
+
+  * Initial packaging
+
+ -- Brian J. Murrell <[email protected]>  Mon, 30 Jan 2023 08:02:22 -0500
@@ -0,0 +1,69 @@
+Source: mpifileutils
+Maintainer: DAOS <[email protected]>
+Uploaders: Brian J. Murrell <[email protected]>
+Section: devel
+Priority: optional
+Build-Depends: debhelper-compat (= 12),
+               cmake,
+               chrpath,
+               libbz2-dev,
+               libssl-dev,
+               uuid-dev,
+               libattr1-dev,
+               libdtcmp-mpich-dev,
+               libcircle-mpich-dev,
+               libhdf5-vol-daos-mpich-dev,
+               libhdf5-mpich-dev,
+               libmpich-dev,
+               libdaos-dev
+Standards-Version: 4.4.1.0
+Vcs-Browser: https://github.com/hpc/mpifileutils
+Vcs-Git: https://github.com/hpc/mpifileutils.git
+Homepage: https://github.com/hpc/mpifileutils.git
+
+Package: mpifileutils
+Architecture: any
+Multi-Arch: foreign
+Depends: ${misc:Depends},
+         ${shlibs:Depends}
+Description: mpiFileUtils provides both a library called libmfu and a suite of MPI-based
+ tools to manage large datasets, which may vary from large directory trees to
+ large files. High-performance computing users often generate large datasets
+ with parallel applications that run with many processes (millions in some
+ cases). However those users are then stuck with single-process tools like cp
+ and rm to manage their datasets. This suite provides MPI-based tools to handle
+ typical jobs like copy, remove, and compare for such datasets, providing
+ speedups of up to 20-30x. It also provides a library that simplifies the
+ creation of new tools or can be used in applications.
+
+Package: libmfu4-mpich
+Architecture: linux-any
+Depends: ${misc:Depends}, ${shlibs:Depends}
+Description: mpiFileUtils provides both a library called libmfu and a suite of MPI-based
+ tools to manage large datasets, which may vary from large directory trees to
+ large files. High-performance computing users often generate large datasets
+ with parallel applications that run with many processes (millions in some
+ cases). However those users are then stuck with single-process tools like cp
+ and rm to manage their datasets. This suite provides MPI-based tools to handle
+ typical jobs like copy, remove, and compare for such datasets, providing
+ speedups of up to 20-30x. It also provides a library that simplifies the
+ creation of new tools or can be used in applications.
+ .
+ This package contains the shared libraries for use with MPICH2.
+
+Package: libmfu-mpich-dev
+Section: libdevel
+Architecture: linux-any
+Depends: libmfu4-mpich (= ${binary:Version}), ${misc:Depends}
+Description: mpiFileUtils provides both a library called libmfu and a suite of MPI-based
+ tools to manage large datasets, which may vary from large directory trees to
+ large files. High-performance computing users often generate large datasets
+ with parallel applications that run with many processes (millions in some
+ cases). However those users are then stuck with single-process tools like cp
+ and rm to manage their datasets. This suite provides MPI-based tools to handle
+ typical jobs like copy, remove, and compare for such datasets, providing
+ speedups of up to 20-30x. It also provides a library that simplifies the
+ creation of new tools or can be used in applications.
+ .
+ This package is needed to compile programs against libmfu4-mpich.
+ It contains the header files and links needed for compiling.
@@ -0,0 +1,48 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: mpifileutils
+Upstream-Contact: https://github.com/hpc/mpifileutils
+Source: https://github.com/hpc/mpifileutils
+Copyright: 2018, Lawrence Livermore National Security, LLC
+           2006-2007,2011-2015, Los Alamos National Security, LLC
+           2013-2015, UT-Battelle, LLC
+           2015, DataDirect Networks, Inc.
+
+Files: *
+Copyright: 2018, Lawrence Livermore National Security, LLC
+           2006-2007,2011-2015, Los Alamos National Security, LLC
+           2013-2015, UT-Battelle, LLC
+           2015, DataDirect Networks, Inc.
+License: BSD 3-Clause "New" or "Revised" License
+BSD 3-Clause License
+
+Copyright (c) 2018, Lawrence Livermore National Security, LLC
+Copyright (c) 2006-2007,2011-2015, Los Alamos National Security, LLC
+Copyright (2013-2015) UT-Battelle, LLC
+Copyright (c) 2015, DataDirect Networks, Inc.
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,2 @@
+usr/include/*
+usr/lib/x86_64-linux-gnu/lib*.so
@@ -0,0 +1 @@
+usr/lib/x86_64-linux-gnu/lib*.so.*
@@ -0,0 +1 @@
+usr/bin/*
@@ -0,0 +1,20 @@
+#! /usr/bin/make -f
+
+export DH_VERBOSE=1
+
+DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH)
+
+%:
+	dh $@
+
+override_dh_auto_configure:
+	dh_auto_configure -- -DENABLE_DAOS=ON							                          \
+		                 -DENABLE_LIBARCHIVE=OFF							                  \
+		                 -DENABLE_HDF5=ON                                                     \
+		                 -DHDF5_INCLUDE_DIRS=/usr/include/hdf5/mpich                          \
+		                 -DHDF5_LIBRARIES=/usr/lib/x86_64-linux-gnu/hdf5/mpich/libhdf5.so
+
+override_dh_auto_install-arch:
+	dh_auto_install
+	chrpath -d debian/tmp/usr/lib/*/*.so* debian/tmp/usr/bin/*
+	rm -f debian/tmp/usr/lib/*/*.la
@@ -0,0 +1 @@
+3.0 (quilt)
@@ -78,7 +78,7 @@ BuildRequires: libattr-devel
 
 
 %if (0%{?suse_version} >= 1500)
-BuildRequires: libfabric1 >= 1.12.0
+BuildRequires: libfabric1 >= 1.12.0, libfabric1 < 1.18
 %endif
 
 %if (0%{?suse_version} > 0)

@@ -34,18 +34,24 @@ RUN echo "$USER:$PASSWD" | chpasswd
 # add the user to the mock group so it can run mock
 RUN usermod -a -G mock $USER
 
+ARG CB0
+RUN dnf -y upgrade && \
+    dnf clean all
+
 # Monkey-patch rpmlint until a new release is made with
 # https://github.com/rpm-software-management/rpmlint/pull/795 in it
+# But make sure to patch after dnf upgrade so that an upgraded rpmlint
+# RPM doesn't wipe out our patch
 COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch .
-RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') &&                    \
-     patch -p1 &&                                                                            \
-     rm -f rpmlint/__pycache__/{cli,lint}.*.pyc) < rpmlint--ignore-unused-rpmlintrc.patch && \
+RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') &&                      \
+     if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then                               \
+         if ! patch -p1; then                                                                  \
+             exit 1;                                                                           \
+         fi;                                                                                   \
+         rm -f rpmlint/__pycache__/{cli,lint}.*.pyc;                                           \
+     fi) < rpmlint--ignore-unused-rpmlintrc.patch;                                             \
     rm -f  rpmlint--ignore-unused-rpmlintrc.patch
 
-ARG CB0
-RUN dnf -y upgrade && \
-    dnf clean all
-
 # show the release that was built
 ARG CACHEBUST
 RUN cat /etc/os-release
@@ -7,33 +7,60 @@
 FROM ubuntu:20.04
 LABEL org.opencontainers.image.authors="[email protected]"
 
-# use same UID as host and default value of 1000 if not specified
-ARG UID=1000
-ARG REPO_URL=""
-ARG REPO_UBUNTU_20_04=""
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    curl gpg
+
+ARG REPO_FILE_URL
+RUN if [ -n "$REPO_FILE_URL" ]; then                                 \
+        cd /etc/apt/sources.list.d &&                                \
+        curl -f -o daos_ci-ubuntu20.04-artifactory.list.tmp          \
+             "$REPO_FILE_URL"daos_ci-ubuntu20.04-artifactory.list && \
+        true > ../sources.list &&                                    \
+        mv daos_ci-ubuntu20.04-artifactory.list.tmp                  \
+           daos_ci-ubuntu20.04-artifactory.list;                     \
+    fi;                                                              \
+    cd -;                                                            \
+    curl -f -O "$REPO_FILE_URL"esad_repo.key;                        \
+    gpg --no-default-keyring --keyring ./temp-keyring.gpg            \
+        --import esad_repo.key;                                      \
+    mkdir -p /usr/local/share/keyrings/;                             \
+    gpg --no-default-keyring --keyring ./temp-keyring.gpg --export   \
+        --output /usr/local/share/keyrings/daos-stack-public.gpg;    \
+    rm ./temp-keyring.gpg;                                           \
+    url_prefix=https://downloads.linux.hpe.com/SDR/;                 \
+    for url in hpPublicKey2048.pub                                   \
+               hpPublicKey2048_key1.pub                              \
+               hpePublicKey2048_key1.pub; do                         \
+        curl -f -O "$url_prefix$url";                                \
+        gpg --no-default-keyring --keyring ./temp-keyring.gpg        \
+            --import "$(basename $url)";                             \
+    done;                                                            \
+    gpg --no-default-keyring --keyring ./temp-keyring.gpg --export   \
+        --output /usr/local/share/keyrings/hpe-sdr-public.gpg;       \
+    rm ./temp-keyring.gpg
 
 # Install basic tools
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
             autoconf bash ca-certificates curl debhelper dh-make        \
-            dpkg-dev dh-python doxygen gcc git git-buildpackage locales \
-            make patch pbuilder pkg-config python3-dev python3-distro   \
-            python3-distutils rpm scons wget cmake valgrind
+            dpkg-dev dh-python doxygen gcc git git-buildpackage         \
+            javahelper locales make patch pbuilder pkg-config           \
+            python3-dev python3-distro python3-distutils rpm scons wget \
+            cmake valgrind rpmdevtools
 
-# rpmdevtools
-RUN echo "deb [trusted=yes] ${REPO_URL}${REPO_UBUNTU_20_04} focal main" > /etc/apt/sources.list.d/daos-stack-ubuntu-stable-local.list
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-            rpmdevtools
+# use same UID as host and default value of 1000 if not specified
+ARG UID=1000
 
 # Add build user (to keep chrootbuild happy)
 ENV USER build
 RUN useradd -u $UID -ms /bin/bash $USER
 
 # need to run the build command as root, as it needs to chroot
-RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then                          \
-        echo "#includedir /etc/sudoers.d" >> /etc/sudoers;                              \
-    fi;                                                                                 \
-    echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \
-    echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build;        \
-    chmod 0440 /etc/sudoers.d/build;                                                    \
-    visudo -c;                                                                          \
+RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then                                     \
+        echo "#includedir /etc/sudoers.d" >> /etc/sudoers;                                         \
+    fi;                                                                                            \
+    echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build;            \
+    echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \
+    echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build;                   \
+    chmod 0440 /etc/sudoers.d/build;                                                               \
+    visudo -c;                                                                                     \
     sudo -l -U build
@@ -47,6 +47,18 @@ DISTRO_VERSION  ?= $(VERSION_ID)
 ORIG_TARGET_VER := 8
 SED_EXPR        := 1s/$(DIST)//p
 endif
+ifeq ($(patsubst %epel-9-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),)
+DIST            := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist})
+VERSION_ID      := 9
+DISTRO_ID       := el9
+DISTRO_BASE     := EL_9
+ifneq ($(DISTRO_VERSION_EL9),)
+override DISTRO_VERSION := $(DISTRO_VERSION_EL9)
+endif
+DISTRO_VERSION  ?= $(VERSION_ID)
+ORIG_TARGET_VER := 9
+SED_EXPR        := 1s/$(DIST)//p
+endif
 ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64)
 VERSION_ID      := 15.2
 DISTRO_ID       := sl15.2

@@ -39,6 +39,7 @@ PR_REPOS                 ?= $(shell git show -s --format=%B | sed -ne 's/^PR-rep
 LEAP_15_PR_REPOS         ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p')
 EL_7_PR_REPOS            ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p')
 EL_8_PR_REPOS            ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p')
+EL_9_PR_REPOS            ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el9: *\(.*\)/\1/p')
 UBUNTU_20_04_PR_REPOS    ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-ubuntu20: *\(.*\)/\1/p')
 REPO_FILES_PR            ?= $(shell git show -s --format=%B | sed -ne 's/^Repo-files-PR: *\(.*\)/\1/p')
 
@@ -66,7 +67,7 @@ DEB_BUILD        := $(DEB_TOP)/$(NAME)-$(VERSION)
 DEB_TARBASE      := $(DEB_TOP)/$(DEB_NAME)_$(VERSION)
 SOURCE           ?= $(eval SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\#/\\\#/g' -e 's/.*:  *//'))$(SOURCE)
 PATCHES          ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*:  *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES)
-OTHER_SOURCES    := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*:  *//' -e 's/.*\///' -e '/\.patch/d' -e p))$(OTHER_SOURCES)
+OTHER_SOURCES    := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e '/^Patch.*:/d' -e 's/Source.*:  *//' -e 's/.*\///' -e p))$(OTHER_SOURCES)
 SOURCES          := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES) $(OTHER_SOURCES))
 ifeq ($(ID_LIKE),debian)
 DEBS             := $(addsuffix _$(VERSION)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' $(TOPDIR)/debian/control))
@@ -84,7 +85,7 @@ define distro_map
 	case $(DISTRO_ID) in               \
 	    el7) distro="centos7"          \
 	    ;;                             \
-	    el8) distro="el8"              \
+	    el*) distro="$(DISTRO_ID)"     \
 	    ;;                             \
 	    sle12.3) distro="sles12.3"     \
 	    ;;                             \
@@ -271,6 +272,9 @@ $(DEB_TOP)/$(DEB_DSC): $(CALLING_MAKEFILE) $(DEB_BUILD).tar.$(SRC_EXT) \
 	cd $(DEB_BUILD); dpkg-buildpackage -S --no-sign --no-check-builddeps
 
 $(SRPM): $(SPEC) $(SOURCES)
+	if [ -f bz-1955184_find-requires ]; then \
+	    chmod 755 bz-1955184_find-requires;  \
+	fi
 	rpmbuild -bs $(COMMON_RPM_ARGS) $(RPM_BUILD_OPTIONS) $(SPEC)
 
 srpm: $(SRPM)
@@ -317,20 +321,6 @@ patch:
 	echo "PKG_GIT_COMMIT is not defined"
 endif
 
-# *_LOCAL_* repos are locally built packages.
-ifeq ($(LOCAL_REPOS),true)
-  ifneq ($(ARTIFACTORY_URL),)
-    ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),)
-      DISTRO_REPOS = disabled # any non-empty value here works and is not used beyond testing if the value is empty or not
-	  # convert to artifactory url
-      DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO := $(subst reposi,artifac,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO))
-      # $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists
-      # of values with spaces as environment variables
-      $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-daos,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO))
-      $(DISTRO_BASE)_LOCAL_REPOS += |[trusted=yes] $(ARTIFACTORY_URL)$(subst stack,stack-deps,$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO))
-    endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),)
-  endif # ifneq ($(ARTIFACTORY_URL),)
-endif # ifeq ($(LOCAL_REPOS),true)
 ifeq ($(ID_LIKE),debian)
 chrootbuild: $(DEB_TOP)/$(DEB_DSC)
 	$(call distro_map)                                      \
@@ -346,6 +336,8 @@ chrootbuild: $(DEB_TOP)/$(DEB_DSC)
 	DEB_TOP="$(DEB_TOP)"                                    \
 	DEB_DSC="$(DEB_DSC)"                                    \
 	DISTRO_ID_OPT="$(DISTRO_ID_OPT)"                        \
+	LOCAL_REPOS='$(LOCAL_REPOS)'                            \
+	ARTIFACTORY_URL="$(ARTIFACTORY_URL)"                    \
 	packaging/debian_chrootbuild
 else
 chrootbuild: $(SRPM) $(CALLING_MAKEFILE)
@@ -360,7 +352,7 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE)
 	REPO_FILE_URL="$(REPO_FILE_URL)"                        \
 	MOCK_OPTIONS="$(MOCK_OPTIONS)"                          \
 	RPM_BUILD_OPTIONS='$(RPM_BUILD_OPTIONS)'                \
-	DISTRO_REPOS='$(DISTRO_REPOS)'                          \
+	LOCAL_REPOS='$(LOCAL_REPOS)'                            \
 	ARTIFACTORY_URL="$(ARTIFACTORY_URL)"                    \
 	DISTRO_VERSION="$(DISTRO_VERSION)"                      \
 	TARGET="$<"                                             \
@@ -425,6 +417,7 @@ packaging_check:
 	          --exclude install                             \
 	          --exclude packaging                           \
 	          --exclude utils                               \
+	          --exclude .vscode                             \
 	          -bur $(PACKAGING_CHECK_DIR)/ packaging/; then \
 	    exit 1;                                             \
 	fi