ARROW-17246: [Packaging][deb][RPM] Don't use system jemalloc (#13739)

Because system jemalloc can't be used with dlopen(). If system jemalloc can't used with dlopen(), our shared libraried can't be loaded as bindings of script languages such as Ruby: + ruby -r gi -e 'p GI.load('\''Arrow'\'')' (null)-WARNING **: Failed to load shared library 'libarrow-glib.so.900' referenced by the typelib: /lib64/libjemalloc.so.2: cannot allocate memory in static TLS block This is caused because system jemalloc isn't built with --disable-initial-exec-tls. See also: * https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=951704 * jemalloc/jemalloc#1237 Authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Krisztián Szűcs <[email protected]>
apache · Jul 29, 2022 · 036fdf2 · 036fdf2
1 parent bb31c9a
commit 036fdf2
Show file tree

Hide file tree

Showing 17 changed files with 37 additions and 26 deletions.
diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh
@@ -158,6 +158,8 @@ echo "::endgroup::"
 
 
 echo "::group::Test Apache Arrow GLib"
+export G_DEBUG=fatal-warnings
+
 ${APT_INSTALL} libarrow-glib-dev=${package_version}
 ${APT_INSTALL} libarrow-glib-doc=${package_version}
 

diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh
@@ -46,12 +46,14 @@ repository_version="${distribution_version}"
 
 cmake_package=cmake
 cmake_command=cmake
+have_arrow_libs=no
 have_flight=yes
 have_gandiva=yes
 have_glib=yes
 have_parquet=yes
 have_python=yes
-have_arrow_libs=no
+have_ruby=yes
+ruby_devel_packages=(ruby-devel)
 install_command="dnf install -y --enablerepo=crb"
 uninstall_command="dnf remove -y"
 clean_command="dnf clean"
@@ -63,6 +65,7 @@ case "${distribution}-${distribution_version}" in
   almalinux-8)
     distribution_prefix="almalinux"
     have_arrow_libs=yes
+    ruby_devel_packages+=(redhat-rpm-config)
     install_command="dnf install -y --enablerepo=powertools"
     info_command="dnf info --enablerepo=powertools"
     ;;
@@ -73,12 +76,13 @@ case "${distribution}-${distribution_version}" in
     distribution_prefix="amazon-linux"
     cmake_package=cmake3
     cmake_command=cmake3
-    have_flight=no
-    have_gandiva=no
-    have_python=no
     if [ "$(arch)" != "aarch64" ]; then
       have_arrow_libs=yes
     fi
+    have_flight=no
+    have_gandiva=no
+    have_python=no
+    have_ruby=no
     install_command="yum install -y"
     uninstall_command="yum remove -y"
     clean_command="yum clean"
@@ -89,10 +93,11 @@ case "${distribution}-${distribution_version}" in
     distribution_prefix="centos"
     cmake_package=cmake3
     cmake_command=cmake3
+    have_arrow_libs=yes
     have_flight=no
     have_gandiva=no
     have_python=no
-    have_arrow_libs=yes
+    have_ruby=no
     install_command="yum install -y"
     uninstall_command="yum remove -y"
     clean_command="yum clean"
@@ -101,6 +106,7 @@ case "${distribution}-${distribution_version}" in
   centos-8)
     distribution_prefix="centos"
     repository_version+="-stream"
+    ruby_devel_packages+=(redhat-rpm-config)
     install_command="dnf install -y --enablerepo=powertools"
     info_command="dnf info --enablerepo=powertools"
     ;;
@@ -206,6 +212,8 @@ echo "::endgroup::"
 
 if [ "${have_glib}" = "yes" ]; then
   echo "::group::Test Apache Arrow GLib"
+  export G_DEBUG=fatal-warnings
+
   ${install_command} --enablerepo=epel arrow-glib-devel-${package_version}
   ${install_command} --enablerepo=epel arrow-glib-doc-${package_version}
 
@@ -215,18 +223,30 @@ if [ "${have_glib}" = "yes" ]; then
   valac --pkg arrow-glib --pkg posix build.vala
   ./build
   popd
+
+  if [ "${have_ruby}" = "yes" ]; then
+    ${install_command} "${ruby_devel_packages[@]}"
+    gem install gobject-introspection
+    ruby -r gi -e "p GI.load('Arrow')"
+  fi
   echo "::endgroup::"
 fi
 
 if [ "${have_flight}" = "yes" ]; then
   echo "::group::Test Apache Arrow Flight"
   ${install_command} --enablerepo=epel arrow-flight-glib-devel-${package_version}
   ${install_command} --enablerepo=epel arrow-flight-glib-doc-${package_version}
+  if [ "${have_ruby}" = "yes" ]; then
+    ruby -r gi -e "p GI.load('ArrowFlight')"
+  fi
   echo "::endgroup::"
 
   echo "::group::Test Apache Arrow Flight SQL"
   ${install_command} --enablerepo=epel arrow-flight-sql-glib-devel-${package_version}
   ${install_command} --enablerepo=epel arrow-flight-sql-glib-doc-${package_version}
+  if [ "${have_ruby}" = "yes" ]; then
+    ruby -r gi -e "p GI.load('ArrowFlightSQL')"
+  fi
   echo "::endgroup::"
 fi
 
@@ -240,6 +260,9 @@ echo "::group::Test Plasma"
 if [ "${have_glib}" = "yes" ]; then
   ${install_command} --enablerepo=epel plasma-glib-devel-${package_version}
   ${install_command} --enablerepo=epel plasma-glib-doc-${package_version}
+  if [ "${have_ruby}" = "yes" ]; then
+    ruby -r gi -e "p GI.load('Plasma')"
+  fi
 else
   ${install_command} --enablerepo=epel plasma-devel-${package_version}
 fi
@@ -250,6 +273,9 @@ if [ "${have_gandiva}" = "yes" ]; then
   if [ "${have_glib}" = "yes" ]; then
     ${install_command} --enablerepo=epel gandiva-glib-devel-${package_version}
     ${install_command} --enablerepo=epel gandiva-glib-doc-${package_version}
+    if [ "${have_ruby}" = "yes" ]; then
+      ruby -r gi -e "p GI.load('Gandiva')"
+    fi
   else
     ${install_command} --enablerepo=epel gandiva-devel-${package_version}
   fi
@@ -261,6 +287,9 @@ if [ "${have_parquet}" = "yes" ]; then
   if [ "${have_glib}" = "yes" ]; then
     ${install_command} --enablerepo=epel parquet-glib-devel-${package_version}
     ${install_command} --enablerepo=epel parquet-glib-doc-${package_version}
+    if [ "${have_ruby}" = "yes" ]; then
+      ruby -r gi -e "p GI.load('Parquet')"
+    fi
   else
     ${install_command} --enablerepo=epel parquet-devel-${package_version}
   fi

diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile
@@ -53,7 +53,6 @@ RUN \
     libgoogle-glog-dev \
     libgrpc++-dev \
     libgtest-dev \
-    libjemalloc-dev \
     liblz4-dev \
     libprotoc-dev \
     libprotobuf-dev \

diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile
@@ -53,7 +53,6 @@ RUN \
     libgoogle-glog-dev \
     libgrpc++-dev \
     libgtest-dev \
-    libjemalloc-dev \
     liblz4-dev \
     libprotoc-dev \
     libprotobuf-dev \

diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
@@ -48,7 +48,6 @@ RUN \
     libglib2.0-doc \
     libgoogle-glog-dev \
     libgtest-dev \
-    libjemalloc-dev \
     liblz4-dev \
     libre2-dev \
     libsnappy-dev \

diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
@@ -50,7 +50,6 @@ RUN \
     libgmock-dev \
     libgoogle-glog-dev \
     libgtest-dev \
-    libjemalloc-dev \
     liblz4-dev \
     libre2-dev \
     libsnappy-dev \

diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile
@@ -51,7 +51,6 @@ RUN \
     libgoogle-glog-dev \
     libgrpc++-dev \
     libgtest-dev \
-    libjemalloc-dev \
     liblz4-dev \
     libprotoc-dev \
     libprotobuf-dev \

diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in
@@ -18,7 +18,6 @@ Build-Depends:
   libgoogle-glog-dev,
 @USE_SYSTEM_GRPC@  libgrpc++-dev,
   libgtest-dev,
-  libjemalloc-dev,
   liblz4-dev,
 @USE_SYSTEM_GRPC@  libprotoc-dev,
 @USE_SYSTEM_GRPC@  libprotobuf-dev,
@@ -147,7 +146,6 @@ Depends:
   libbz2-dev,
 @USE_SYSTEM_C_ARES@  libc-ares-dev,
 @USE_SYSTEM_GRPC@  libgrpc++-dev,
-  libjemalloc-dev,
   liblz4-dev,
   libre2-dev,
   libsnappy-dev,

diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
@@ -4,7 +4,6 @@ usr/lib/*/cmake/arrow/ArrowOptions.cmake
 usr/lib/*/cmake/arrow/ArrowTargets*.cmake
 usr/lib/*/cmake/arrow/Find*Alt.cmake
 usr/lib/*/cmake/arrow/FindArrow.cmake
-usr/lib/*/cmake/arrow/Findjemalloc.cmake
 usr/lib/*/cmake/arrow/Find[BTuz]*.cmake
 usr/lib/*/cmake/arrow/arrow-config.cmake
 usr/lib/*/libarrow.a

diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules
@@ -62,8 +62,7 @@ override_dh_auto_configure:
 	  -DCUDAToolkit_ROOT=/usr				\
 	  -DPARQUET_REQUIRE_ENCRYPTION=ON			\
 	  -DPythonInterp_FIND_VERSION=ON			\
-	  -DPythonInterp_FIND_VERSION_MAJOR=3			\
-	  -Djemalloc_SOURCE=SYSTEM
+	  -DPythonInterp_FIND_VERSION_MAJOR=3
 
 override_dh_auto_build:
 	dh_auto_build				\

diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
@@ -40,7 +40,6 @@ RUN \
     glog-devel \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     json-devel \
     libarchive \
     libzstd-devel \

diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile
@@ -39,7 +39,6 @@ RUN \
     git \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     json-devel \
     libarchive \
     libzstd-devel \

diff --git a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile
@@ -37,7 +37,6 @@ RUN \
     glog-devel \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     json-devel \
     lz4-devel \
     make \

diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
@@ -105,7 +105,6 @@ BuildRequires:	git
 %if %{use_glog}
 BuildRequires:	glog-devel
 %endif
-BuildRequires:	jemalloc-devel
 %if %{use_gcs}
 BuildRequires:	json-devel
 %endif
@@ -194,7 +193,6 @@ cd cpp
   -DPythonInterp_FIND_VERSION=ON \
   -DPythonInterp_FIND_VERSION_MAJOR=3 \
 %endif
-  -Djemalloc_SOURCE=SYSTEM \
   -G"Unix Makefiles"
 %arrow_cmake_build
 cd -
@@ -243,7 +241,6 @@ Requires:	gflags
 %if %{use_glog}
 Requires:	glog
 %endif
-Requires:	jemalloc
 %if %{have_zstd}
 Requires:	libzstd
 %endif
@@ -275,7 +272,6 @@ Requires:	bzip2-devel
 %if %{use_flight}
 Requires:	c-ares-devel
 %endif
-Requires:	jemalloc-devel
 %if %{use_gcs}
 Requires:	json-devel
 %endif
@@ -323,7 +319,6 @@ Libraries and header files for Apache Arrow C++.
 %if %{use_flight}
 %{_libdir}/cmake/arrow/Findc-aresAlt.cmake
 %endif
-%{_libdir}/cmake/arrow/Findjemalloc.cmake
 %{_libdir}/cmake/arrow/Findlz4Alt.cmake
 %if %{have_re2}
 %{_libdir}/cmake/arrow/Findre2Alt.cmake

diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile
@@ -39,7 +39,6 @@ RUN \
     glog-devel \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     json-devel \
     libzstd-devel \
     lz4-devel \

diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile
@@ -40,7 +40,6 @@ RUN \
     glog-devel \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     json-devel \
     libarchive \
     libzstd-devel \

diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile
@@ -39,7 +39,6 @@ RUN \
     git \
     gobject-introspection-devel \
     gtk-doc \
-    jemalloc-devel \
     json-devel \
     libarchive \
     libzstd-devel \