ARROW-14390: [Packaging][Ubuntu] Add support for Ubuntu 21.10

We need a workaround for aws-sdk-cpp with g++ 11 because Ubuntu 21.10 ships g++ 11. See also: aws/aws-sdk-cpp#1750 Closes #11532 from kou/packaging-ubuntu-21.10 Authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
apache · Oct 26, 2021 · 1bac505 · 1bac505
1 parent 11e359c
commit 1bac505
Show file tree

Hide file tree

Showing 10 changed files with 181 additions and 11 deletions.
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -3937,11 +3937,19 @@ macro(build_awssdk)
                       DEPENDS aws_checksums_ep)
   add_dependencies(AWS::aws-c-event-stream aws_c_event_stream_ep)
 
+  set(AWSSDK_PATCH_COMMAND)
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER
+                                              "10")
+    # Workaround for https://github.com/aws/aws-sdk-cpp/issues/1750
+    set(AWSSDK_PATCH_COMMAND "sed" "-i.bak" "-e" "s/\"-Werror\"//g"
+                             "<SOURCE_DIR>/cmake/compiler_settings.cmake")
+  endif()
   externalproject_add(awssdk_ep
                       ${EP_LOG_OPTIONS}
                       URL ${AWSSDK_SOURCE_URL}
                       URL_HASH "SHA256=${ARROW_AWSSDK_BUILD_SHA256_CHECKSUM}"
                       CMAKE_ARGS ${AWSSDK_CMAKE_ARGS}
+                      PATCH_COMMAND ${AWSSDK_PATCH_COMMAND}
                       BUILD_BYPRODUCTS ${AWS_CPP_SDK_COGNITO_IDENTITY_STATIC_LIBRARY}
                                        ${AWS_CPP_SDK_CORE_STATIC_LIBRARY}
                                        ${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY}

diff --git a/dev/release/binary-task.rb b/dev/release/binary-task.rb
@@ -1010,8 +1010,8 @@ def available_apt_targets
       ["debian", "bookworm", "main"],
       ["ubuntu", "bionic", "main"],
       ["ubuntu", "focal", "main"],
-      ["ubuntu", "groovy", "main"],
       ["ubuntu", "hirsute", "main"],
+      ["ubuntu", "impish", "main"],
     ]
   end
 

diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
@@ -154,16 +154,21 @@ test_apt() {
                 "ubuntu:focal" \
                 "arm64v8/ubuntu:focal" \
                 "ubuntu:hirsute" \
-                "arm64v8/ubuntu:hirsute"; do \
+                "arm64v8/ubuntu:hirsute" \
+                "ubuntu:impish" \
+                "arm64v8/ubuntu:impish"; do \
     case "${target}" in
-      arm64v8/debian:bullseye|arm64v8/debian:bookworm|arm64v8/ubuntu:hirsute)
-        # qemu-user-static in Ubuntu 20.04 has a crash bug:
-        #   https://bugs.launchpad.net/qemu/+bug/1749393
-        continue
-        ;;
       arm64v8/*)
         if [ "$(arch)" = "aarch64" -o -e /usr/bin/qemu-aarch64-static ]; then
-          : # OK
+          case "${target}" in
+          arm64v8/debian:buster|arm64v8/ubuntu:bionic|arm64v8/ubuntu:focal)
+            ;; # OK
+          *)
+            # qemu-user-static in Ubuntu 20.04 has a crash bug:
+            #   https://bugs.launchpad.net/qemu/+bug/1749393
+            continue
+            ;;
+          esac
         else
           continue
         fi

diff --git a/dev/tasks/linux-packages/Rakefile b/dev/tasks/linux-packages/Rakefile
@@ -167,6 +167,8 @@ class LocalBinaryTask < BinaryTask
       # "ubuntu-focal-arm64",
       "ubuntu-hirsute",
       # "ubuntu-hirsute-arm64",
+      "ubuntu-impish",
+      # "ubuntu-impish-arm64",
     ]
   end
 
@@ -211,6 +213,8 @@ class LocalBinaryTask < BinaryTask
     # Disable aarch64 targets by default for now
     # because they require some setups on host.
     [
+      "almalinux-8",
+      # "almalinux-8-aarch64",
       "amazon-linux-2",
       # "amazon-linux-2-aarch64",
       "centos-7",

diff --git a/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-impish/Dockerfile b/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-impish/Dockerfile
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM ubuntu:impish
+
+RUN \
+  echo "debconf debconf/frontend select Noninteractive" | \
+    debconf-set-selections
+
+RUN \
+  echo 'APT::Install-Recommends "false";' > \
+    /etc/apt/apt.conf.d/disable-install-recommends
+
+ARG DEBUG
+
+RUN \
+  quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \
+  apt update ${quiet} && \
+  apt install -y -V ${quiet} \
+    build-essential \
+    debhelper \
+    devscripts \
+    fakeroot \
+    gnupg \
+    lsb-release && \
+  apt clean && \
+  rm -rf /var/lib/apt/lists/*
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish-arm64/from b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish-arm64/from
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+arm64v8/ubuntu:impish
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish/Dockerfile
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG FROM=ubuntu:impish
+FROM ${FROM}
+
+RUN \
+  echo "debconf debconf/frontend select Noninteractive" | \
+    debconf-set-selections
+
+RUN \
+  echo 'APT::Install-Recommends "false";' > \
+    /etc/apt/apt.conf.d/disable-install-recommends
+
+ARG DEBUG
+RUN \
+  quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \
+  apt update ${quiet} && \
+  apt install -y -V ${quiet} \
+    build-essential \
+    ccache \
+    clang \
+    cmake \
+    debhelper \
+    devscripts \
+    git \
+    gtk-doc-tools \
+    libboost-filesystem-dev \
+    libboost-system-dev \
+    libbrotli-dev \
+    libbz2-dev \
+    libc-ares-dev \
+    libcurl4-openssl-dev \
+    libgirepository1.0-dev \
+    libglib2.0-doc \
+    libgmock-dev \
+    libgoogle-glog-dev \
+    libgrpc++-dev \
+    libgtest-dev \
+    liblz4-dev \
+    libprotoc-dev \
+    libprotobuf-dev \
+    libre2-dev \
+    libsnappy-dev \
+    libssl-dev \
+    libthrift-dev \
+    libutf8proc-dev \
+    libzstd-dev \
+    llvm-dev \
+    lsb-release \
+    ninja-build \
+    pkg-config \
+    protobuf-compiler-grpc \
+    python3-dev \
+    python3-numpy \
+    python3-pip \
+    python3-setuptools \
+    rapidjson-dev \
+    tzdata \
+    zlib1g-dev && \
+  if apt list | grep -q '^libcuda1'; then \
+    apt install -y -V ${quiet} nvidia-cuda-toolkit; \
+  else \
+    :; \
+  fi && \
+  apt clean && \
+  python3 -m pip install --no-use-pep517 meson && \
+  ln -s /usr/local/bin/meson /usr/bin/ && \
+  rm -rf /var/lib/apt/lists/*
diff --git a/dev/tasks/linux-packages/github.linux.amd64.yml b/dev/tasks/linux-packages/github.linux.amd64.yml
@@ -54,6 +54,10 @@ jobs:
           ARROW_VERSION: {{ arrow.version }}
           REPO: {{ '${{ secrets.REPO }}' }}
           YUM_TARGETS: {{ target }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: packages
+          path: packages/*/{{ task_namespace }}/repositories/
       - name: Docker Push
         continue-on-error: true
         shell: bash

diff --git a/dev/tasks/linux-packages/package-task.rb b/dev/tasks/linux-packages/package-task.rb
@@ -100,16 +100,20 @@ def download(url, output_path)
     unless File.exist?(absolute_output_path)
       mkdir_p(File.dirname(absolute_output_path))
       rake_output_message "Downloading... #{url}"
-      URI(url).open do |downloaded_file|
+      open_url(url) do |downloaded_file|
         File.open(absolute_output_path, "wb") do |output_file|
-          output_file.print(downloaded_file.read)
+          IO.copy_stream(downloaded_file, output_file)
         end
       end
     end
 
     absolute_output_path
   end
 
+  def open_url(url, &block)
+    URI(url).open(&block)
+  end
+
   def substitute_content(content)
     content.gsub(/@(.+?)@/) do |matched|
       yield($1, matched)
@@ -270,6 +274,8 @@ def apt_targets_default
       # "ubuntu-focal-arm64",
       "ubuntu-hirsute",
       # "ubuntu-hirsute-arm64",
+      "ubuntu-impish",
+      # "ubuntu-impish-arm64",
     ]
   end
 

diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -446,7 +446,8 @@ tasks:
                   "debian-bookworm",
                   "ubuntu-bionic",
                   "ubuntu-focal",
-                  "ubuntu-hirsute"] %}
+                  "ubuntu-hirsute",
+                  "ubuntu-impish"] %}
   {% for architecture in ["amd64", "arm64"] %}
   {{ target }}-{{ architecture }}:
     {% if architecture == "amd64" %}