From 036fdf2d03c3a6986109f053d94fc237d2c2f82d Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 30 Jul 2022 01:17:14 +0900 Subject: [PATCH] ARROW-17246: [Packaging][deb][RPM] Don't use system jemalloc (#13739) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Because system jemalloc can't be used with dlopen(). If system jemalloc can't used with dlopen(), our shared libraried can't be loaded as bindings of script languages such as Ruby: + ruby -r gi -e 'p GI.load('\''Arrow'\'')' (null)-WARNING **: Failed to load shared library 'libarrow-glib.so.900' referenced by the typelib: /lib64/libjemalloc.so.2: cannot allocate memory in static TLS block This is caused because system jemalloc isn't built with --disable-initial-exec-tls. See also: * https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=951704 * https://github.com/jemalloc/jemalloc/issues/1237 Authored-by: Sutou Kouhei Signed-off-by: Krisztián Szűcs --- dev/release/verify-apt.sh | 2 + dev/release/verify-yum.sh | 39 ++++++++++++++++--- .../apt/debian-bookworm/Dockerfile | 1 - .../apt/debian-bullseye/Dockerfile | 1 - .../apache-arrow/apt/ubuntu-bionic/Dockerfile | 1 - .../apache-arrow/apt/ubuntu-focal/Dockerfile | 1 - .../apache-arrow/apt/ubuntu-jammy/Dockerfile | 1 - .../apache-arrow/debian/control.in | 2 - .../apache-arrow/debian/libarrow-dev.install | 1 - .../linux-packages/apache-arrow/debian/rules | 3 +- .../apache-arrow/yum/almalinux-8/Dockerfile | 1 - .../apache-arrow/yum/almalinux-9/Dockerfile | 1 - .../yum/amazon-linux-2/Dockerfile | 1 - .../apache-arrow/yum/arrow.spec.in | 5 --- .../apache-arrow/yum/centos-7/Dockerfile | 1 - .../yum/centos-8-stream/Dockerfile | 1 - .../yum/centos-9-stream/Dockerfile | 1 - 17 files changed, 37 insertions(+), 26 deletions(-) diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh index 846d9c7ec3b8a..121abdbc6eb2b 100755 --- a/dev/release/verify-apt.sh +++ b/dev/release/verify-apt.sh @@ -158,6 +158,8 @@ echo "::endgroup::" echo "::group::Test Apache Arrow GLib" +export G_DEBUG=fatal-warnings + ${APT_INSTALL} libarrow-glib-dev=${package_version} ${APT_INSTALL} libarrow-glib-doc=${package_version} diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh index 889dddd9d1b9e..93bbcfe136607 100755 --- a/dev/release/verify-yum.sh +++ b/dev/release/verify-yum.sh @@ -46,12 +46,14 @@ repository_version="${distribution_version}" cmake_package=cmake cmake_command=cmake +have_arrow_libs=no have_flight=yes have_gandiva=yes have_glib=yes have_parquet=yes have_python=yes -have_arrow_libs=no +have_ruby=yes +ruby_devel_packages=(ruby-devel) install_command="dnf install -y --enablerepo=crb" uninstall_command="dnf remove -y" clean_command="dnf clean" @@ -63,6 +65,7 @@ case "${distribution}-${distribution_version}" in almalinux-8) distribution_prefix="almalinux" have_arrow_libs=yes + ruby_devel_packages+=(redhat-rpm-config) install_command="dnf install -y --enablerepo=powertools" info_command="dnf info --enablerepo=powertools" ;; @@ -73,12 +76,13 @@ case "${distribution}-${distribution_version}" in distribution_prefix="amazon-linux" cmake_package=cmake3 cmake_command=cmake3 - have_flight=no - have_gandiva=no - have_python=no if [ "$(arch)" != "aarch64" ]; then have_arrow_libs=yes fi + have_flight=no + have_gandiva=no + have_python=no + have_ruby=no install_command="yum install -y" uninstall_command="yum remove -y" clean_command="yum clean" @@ -89,10 +93,11 @@ case "${distribution}-${distribution_version}" in distribution_prefix="centos" cmake_package=cmake3 cmake_command=cmake3 + have_arrow_libs=yes have_flight=no have_gandiva=no have_python=no - have_arrow_libs=yes + have_ruby=no install_command="yum install -y" uninstall_command="yum remove -y" clean_command="yum clean" @@ -101,6 +106,7 @@ case "${distribution}-${distribution_version}" in centos-8) distribution_prefix="centos" repository_version+="-stream" + ruby_devel_packages+=(redhat-rpm-config) install_command="dnf install -y --enablerepo=powertools" info_command="dnf info --enablerepo=powertools" ;; @@ -206,6 +212,8 @@ echo "::endgroup::" if [ "${have_glib}" = "yes" ]; then echo "::group::Test Apache Arrow GLib" + export G_DEBUG=fatal-warnings + ${install_command} --enablerepo=epel arrow-glib-devel-${package_version} ${install_command} --enablerepo=epel arrow-glib-doc-${package_version} @@ -215,6 +223,12 @@ if [ "${have_glib}" = "yes" ]; then valac --pkg arrow-glib --pkg posix build.vala ./build popd + + if [ "${have_ruby}" = "yes" ]; then + ${install_command} "${ruby_devel_packages[@]}" + gem install gobject-introspection + ruby -r gi -e "p GI.load('Arrow')" + fi echo "::endgroup::" fi @@ -222,11 +236,17 @@ if [ "${have_flight}" = "yes" ]; then echo "::group::Test Apache Arrow Flight" ${install_command} --enablerepo=epel arrow-flight-glib-devel-${package_version} ${install_command} --enablerepo=epel arrow-flight-glib-doc-${package_version} + if [ "${have_ruby}" = "yes" ]; then + ruby -r gi -e "p GI.load('ArrowFlight')" + fi echo "::endgroup::" echo "::group::Test Apache Arrow Flight SQL" ${install_command} --enablerepo=epel arrow-flight-sql-glib-devel-${package_version} ${install_command} --enablerepo=epel arrow-flight-sql-glib-doc-${package_version} + if [ "${have_ruby}" = "yes" ]; then + ruby -r gi -e "p GI.load('ArrowFlightSQL')" + fi echo "::endgroup::" fi @@ -240,6 +260,9 @@ echo "::group::Test Plasma" if [ "${have_glib}" = "yes" ]; then ${install_command} --enablerepo=epel plasma-glib-devel-${package_version} ${install_command} --enablerepo=epel plasma-glib-doc-${package_version} + if [ "${have_ruby}" = "yes" ]; then + ruby -r gi -e "p GI.load('Plasma')" + fi else ${install_command} --enablerepo=epel plasma-devel-${package_version} fi @@ -250,6 +273,9 @@ if [ "${have_gandiva}" = "yes" ]; then if [ "${have_glib}" = "yes" ]; then ${install_command} --enablerepo=epel gandiva-glib-devel-${package_version} ${install_command} --enablerepo=epel gandiva-glib-doc-${package_version} + if [ "${have_ruby}" = "yes" ]; then + ruby -r gi -e "p GI.load('Gandiva')" + fi else ${install_command} --enablerepo=epel gandiva-devel-${package_version} fi @@ -261,6 +287,9 @@ if [ "${have_parquet}" = "yes" ]; then if [ "${have_glib}" = "yes" ]; then ${install_command} --enablerepo=epel parquet-glib-devel-${package_version} ${install_command} --enablerepo=epel parquet-glib-doc-${package_version} + if [ "${have_ruby}" = "yes" ]; then + ruby -r gi -e "p GI.load('Parquet')" + fi else ${install_command} --enablerepo=epel parquet-devel-${package_version} fi diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile index 42a7327614c14..152ac08a7cad3 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile @@ -53,7 +53,6 @@ RUN \ libgoogle-glog-dev \ libgrpc++-dev \ libgtest-dev \ - libjemalloc-dev \ liblz4-dev \ libprotoc-dev \ libprotobuf-dev \ diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile index 1562f1d662c75..440a07f09c29c 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile @@ -53,7 +53,6 @@ RUN \ libgoogle-glog-dev \ libgrpc++-dev \ libgtest-dev \ - libjemalloc-dev \ liblz4-dev \ libprotoc-dev \ libprotobuf-dev \ diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile index 64ebb3b72dd8a..b897fa73a33de 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile @@ -48,7 +48,6 @@ RUN \ libglib2.0-doc \ libgoogle-glog-dev \ libgtest-dev \ - libjemalloc-dev \ liblz4-dev \ libre2-dev \ libsnappy-dev \ diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile index 9efc5602687b3..5abee7c2e0928 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile @@ -50,7 +50,6 @@ RUN \ libgmock-dev \ libgoogle-glog-dev \ libgtest-dev \ - libjemalloc-dev \ liblz4-dev \ libre2-dev \ libsnappy-dev \ diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile index e6f657533f8e2..7e95c37642a5c 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile @@ -51,7 +51,6 @@ RUN \ libgoogle-glog-dev \ libgrpc++-dev \ libgtest-dev \ - libjemalloc-dev \ liblz4-dev \ libprotoc-dev \ libprotobuf-dev \ diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in index 8de71bda75590..e0a63788280b5 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/control.in +++ b/dev/tasks/linux-packages/apache-arrow/debian/control.in @@ -18,7 +18,6 @@ Build-Depends: libgoogle-glog-dev, @USE_SYSTEM_GRPC@ libgrpc++-dev, libgtest-dev, - libjemalloc-dev, liblz4-dev, @USE_SYSTEM_GRPC@ libprotoc-dev, @USE_SYSTEM_GRPC@ libprotobuf-dev, @@ -147,7 +146,6 @@ Depends: libbz2-dev, @USE_SYSTEM_C_ARES@ libc-ares-dev, @USE_SYSTEM_GRPC@ libgrpc++-dev, - libjemalloc-dev, liblz4-dev, libre2-dev, libsnappy-dev, diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install index b55c76d69289b..40a834ced42f1 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install +++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install @@ -4,7 +4,6 @@ usr/lib/*/cmake/arrow/ArrowOptions.cmake usr/lib/*/cmake/arrow/ArrowTargets*.cmake usr/lib/*/cmake/arrow/Find*Alt.cmake usr/lib/*/cmake/arrow/FindArrow.cmake -usr/lib/*/cmake/arrow/Findjemalloc.cmake usr/lib/*/cmake/arrow/Find[BTuz]*.cmake usr/lib/*/cmake/arrow/arrow-config.cmake usr/lib/*/libarrow.a diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules index 9178111ebaf0c..2f3b694d6806c 100755 --- a/dev/tasks/linux-packages/apache-arrow/debian/rules +++ b/dev/tasks/linux-packages/apache-arrow/debian/rules @@ -62,8 +62,7 @@ override_dh_auto_configure: -DCUDAToolkit_ROOT=/usr \ -DPARQUET_REQUIRE_ENCRYPTION=ON \ -DPythonInterp_FIND_VERSION=ON \ - -DPythonInterp_FIND_VERSION_MAJOR=3 \ - -Djemalloc_SOURCE=SYSTEM + -DPythonInterp_FIND_VERSION_MAJOR=3 override_dh_auto_build: dh_auto_build \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile index 0a37bab4b12cd..a75fd022bdae0 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile @@ -40,7 +40,6 @@ RUN \ glog-devel \ gobject-introspection-devel \ gtk-doc \ - jemalloc-devel \ json-devel \ libarchive \ libzstd-devel \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile index 8561779f658ad..df63f7a1a7e97 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-9/Dockerfile @@ -39,7 +39,6 @@ RUN \ git \ gobject-introspection-devel \ gtk-doc \ - jemalloc-devel \ json-devel \ libarchive \ libzstd-devel \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile index fa1ded66bcea0..68d604d64ed03 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile @@ -37,7 +37,6 @@ RUN \ glog-devel \ gobject-introspection-devel \ gtk-doc \ - jemalloc-devel \ json-devel \ lz4-devel \ make \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index 78f974f97111e..098342bacd616 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -105,7 +105,6 @@ BuildRequires: git %if %{use_glog} BuildRequires: glog-devel %endif -BuildRequires: jemalloc-devel %if %{use_gcs} BuildRequires: json-devel %endif @@ -194,7 +193,6 @@ cd cpp -DPythonInterp_FIND_VERSION=ON \ -DPythonInterp_FIND_VERSION_MAJOR=3 \ %endif - -Djemalloc_SOURCE=SYSTEM \ -G"Unix Makefiles" %arrow_cmake_build cd - @@ -243,7 +241,6 @@ Requires: gflags %if %{use_glog} Requires: glog %endif -Requires: jemalloc %if %{have_zstd} Requires: libzstd %endif @@ -275,7 +272,6 @@ Requires: bzip2-devel %if %{use_flight} Requires: c-ares-devel %endif -Requires: jemalloc-devel %if %{use_gcs} Requires: json-devel %endif @@ -323,7 +319,6 @@ Libraries and header files for Apache Arrow C++. %if %{use_flight} %{_libdir}/cmake/arrow/Findc-aresAlt.cmake %endif -%{_libdir}/cmake/arrow/Findjemalloc.cmake %{_libdir}/cmake/arrow/Findlz4Alt.cmake %if %{have_re2} %{_libdir}/cmake/arrow/Findre2Alt.cmake diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile index 93d4a7812b11a..9c93e2f2407f3 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -39,7 +39,6 @@ RUN \ glog-devel \ gobject-introspection-devel \ gtk-doc \ - jemalloc-devel \ json-devel \ libzstd-devel \ lz4-devel \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile index ca9e44d409d48..b29cc4565bd6b 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile @@ -40,7 +40,6 @@ RUN \ glog-devel \ gobject-introspection-devel \ gtk-doc \ - jemalloc-devel \ json-devel \ libarchive \ libzstd-devel \ diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile index 2bc036432cd1f..38e6ae4531ee8 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-9-stream/Dockerfile @@ -39,7 +39,6 @@ RUN \ git \ gobject-introspection-devel \ gtk-doc \ - jemalloc-devel \ json-devel \ libarchive \ libzstd-devel \