ARROW-15873: [CI] Migrate from Ubuntu 21.04 to 22.04

Ubuntu 21.04 reached EOL at 2022-01. Ubuntu 22.04 will be released in 2022-04. Closes apache#12781 from kou/ci-ubuntu-22.04 Authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
Bit-Quill · Apr 9, 2022 · e453ffe · e453ffe
1 parent 961ec77
commit e453ffe
Show file tree

Hide file tree

Showing 10 changed files with 227 additions and 103 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -28,6 +28,7 @@
 !ci/**
 !c_glib/Gemfile
 !dev/archery/setup.py
+!dev/release/setup-*.sh
 !docs/requirements*.txt
 !python/requirements*.txt
 !python/manylinux1/**

diff --git a/.env b/.env
@@ -61,7 +61,8 @@ GO=1.16
 HDFS=3.2.1
 JDK=8
 KARTOTHEK=latest
-LLVM=12
+# LLVM 12 and GCC 11 reports -Wmismatched-new-delete.
+LLVM=13
 MAVEN=3.5.4
 NODE=16
 NUMPY=latest

diff --git a/ci/docker/ubuntu-18.04-verify-rc.dockerfile b/ci/docker/ubuntu-18.04-verify-rc.dockerfile
@@ -19,42 +19,8 @@ ARG arch=amd64
 FROM ${arch}/ubuntu:18.04
 
 ENV DEBIAN_FRONTEND=noninteractive
-
-ARG llvm=12
-RUN apt-get update -y -q && \
-    apt-get install -y -q --no-install-recommends \
-       apt-transport-https \
-       ca-certificates \
-       gnupg \
-       wget && \
-    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-    echo "deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-${llvm} main" > \
-       /etc/apt/sources.list.d/llvm.list && \
-    apt-get update -y -q && \
-    apt-get install -y -q --no-install-recommends \
-        build-essential \
-        clang \
-        cmake \
-        curl \
-        git \
-        libcurl4-openssl-dev \
-        libgirepository1.0-dev \
-        libglib2.0-dev \
-        libsqlite3-dev \
-        libssl-dev \
-        llvm-${llvm}-dev \
-        maven \
-        ninja-build \
-        openjdk-11-jdk \
-        pkg-config \
-        python3-pip \
-        python3.8-dev \
-        python3.8-venv \
-        ruby-dev \
-        wget \
-        tzdata && \
+COPY dev/release/setup-ubuntu.sh /
+RUN /setup-ubuntu.sh && \
+    rm /setup-ubuntu.sh && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists*
-
-RUN python3.8 -m pip install -U pip && \
-    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
diff --git a/ci/docker/ubuntu-20.04-verify-rc.dockerfile b/ci/docker/ubuntu-20.04-verify-rc.dockerfile
@@ -19,28 +19,8 @@ ARG arch=amd64
 FROM ${arch}/ubuntu:20.04
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update -y -q && \
-    apt-get install -y -q --no-install-recommends \
-        build-essential \
-        clang \
-        cmake \
-        curl \
-        git \
-        libcurl4-openssl-dev \
-        libgirepository1.0-dev \
-        libglib2.0-dev \
-        libsqlite3-dev \
-        libssl-dev \
-        llvm-dev \
-        maven \
-        ninja-build \
-        nlohmann-json3-dev \
-        openjdk-11-jdk \
-        pkg-config \
-        python3-dev \
-        python3-pip \
-        python3-venv \
-        ruby-dev \
-        wget && \
+COPY dev/release/setup-ubuntu.sh /
+RUN /setup-ubuntu.sh && \
+    rm /setup-ubuntu.sh && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists*
diff --git a/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile b/ci/docker/ubuntu-22.04-cpp-minimal.dockerfile
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base=amd64/ubuntu:22.04
+FROM ${base}
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN echo "debconf debconf/frontend select Noninteractive" | \
+        debconf-set-selections
+
+RUN apt-get update -y -q && \
+    apt-get install -y -q \
+        build-essential \
+        ccache \
+        cmake \
+        git \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        python3-pip \
+        wget && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists*
+
+# Installs LLVM toolchain, for Gandiva and testing other compilers
+#
+# Note that this is installed before the base packages to improve iteration
+# while debugging package list with docker build.
+ARG llvm
+RUN latest_system_llvm=14 && \
+    if [ ${llvm} -gt ${latest_system_llvm} ]; then \
+      apt-get update -y -q && \
+      apt-get install -y -q --no-install-recommends \
+          apt-transport-https \
+          ca-certificates \
+          gnupg \
+          lsb-release \
+          wget && \
+      wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+      code_name=$(lsb_release --codename --short) && \
+      if [ ${llvm} -gt 10 ]; then \
+        echo "deb https://apt.llvm.org/${code_name}/ llvm-toolchain-${code_name}-${llvm} main" > \
+           /etc/apt/sources.list.d/llvm.list; \
+      fi; \
+    fi && \
+    apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends \
+        clang-${llvm} \
+        llvm-${llvm}-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists*
+
+COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
+
+COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_gcs_testbench.sh default
+
+ENV ARROW_BUILD_TESTS=ON \
+    ARROW_DATASET=ON \
+    ARROW_FLIGHT=ON \
+    ARROW_GANDIVA=ON \
+    ARROW_GCS=ON \
+    ARROW_HDFS=ON \
+    ARROW_HOME=/usr/local \
+    ARROW_INSTALL_NAME_RPATH=OFF \
+    ARROW_NO_DEPRECATED_API=ON \
+    ARROW_ORC=ON \
+    ARROW_PARQUET=ON \
+    ARROW_PLASMA=ON \
+    ARROW_S3=ON \
+    ARROW_USE_CCACHE=ON \
+    ARROW_WITH_BROTLI=ON \
+    ARROW_WITH_BZ2=ON \
+    ARROW_WITH_LZ4=ON \
+    ARROW_WITH_OPENTELEMETRY=OFF \
+    ARROW_WITH_SNAPPY=ON \
+    ARROW_WITH_ZLIB=ON \
+    ARROW_WITH_ZSTD=ON \
+    CMAKE_GENERATOR="Unix Makefiles" \
+    PARQUET_BUILD_EXAMPLES=ON \
+    PARQUET_BUILD_EXECUTABLES=ON \
+    PATH=/usr/lib/ccache/:$PATH \
+    PYTHON=python3
diff --git a/ci/docker/ubuntu-21.04-cpp.dockerfile → ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-21.04-cpp.dockerfile → ci/docker/ubuntu-22.04-cpp.dockerfile
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-ARG base=amd64/ubuntu:21.04
+ARG base=amd64/ubuntu:22.04
 FROM ${base}
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -29,7 +29,7 @@ RUN echo "debconf debconf/frontend select Noninteractive" | \
 # while debugging package list with docker build.
 ARG clang_tools
 ARG llvm
-RUN latest_system_llvm=12 && \
+RUN latest_system_llvm=14 && \
     if [ ${llvm} -gt ${latest_system_llvm} -o \
          ${clang_tools} -gt ${latest_system_llvm} ]; then \
       apt-get update -y -q && \
@@ -96,6 +96,7 @@ RUN apt-get update -y -q && \
         pkg-config \
         protobuf-compiler \
         protobuf-compiler-grpc \
+        python3-dev \
         python3-pip \
         rapidjson-dev \
         rsync \
@@ -104,6 +105,38 @@ RUN apt-get update -y -q && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists*
 
+ARG gcc_version=""
+RUN if [ "${gcc_version}" = "" ]; then \
+      apt-get update -y -q && \
+      apt-get install -y -q --no-install-recommends \
+          g++ \
+          gcc; \
+    else \
+      if [ "${gcc_version}" -gt "11" ]; then \
+          apt-get update -y -q && \
+          apt-get install -y -q --no-install-recommends software-properties-common && \
+          add-apt-repository ppa:ubuntu-toolchain-r/volatile; \
+      fi; \
+      apt-get update -y -q && \
+      apt-get install -y -q --no-install-recommends \
+          g++-${gcc_version} \
+          gcc-${gcc_version} && \
+      update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \
+      update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \
+      update-alternatives --install \
+        /usr/bin/$(uname --machine)-linux-gnu-gcc \
+        $(uname --machine)-linux-gnu-gcc \
+        /usr/bin/$(uname --machine)-linux-gnu-gcc-${gcc_version} 100 && \
+      update-alternatives --install \
+        /usr/bin/$(uname --machine)-linux-gnu-g++ \
+        $(uname --machine)-linux-gnu-g++ \
+        /usr/bin/$(uname --machine)-linux-gnu-g++-${gcc_version} 100 && \
+      update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100 && \
+      update-alternatives --set cc /usr/bin/gcc && \
+      update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100 && \
+      update-alternatives --set c++ /usr/bin/g++; \
+    fi
+
 COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
 
@@ -149,27 +182,3 @@ ENV ARROW_BUILD_TESTS=ON \
     Protobuf_SOURCE=BUNDLED \
     PATH=/usr/lib/ccache/:$PATH \
     PYTHON=python3
-
-ARG gcc_version=""
-RUN if [ "${gcc_version}" = "" ]; then \
-      apt-get update -y -q && \
-      apt-get install -y -q --no-install-recommends \
-          g++ \
-          gcc; \
-    else \
-      if [ "${gcc_version}" -gt "10" ]; then \
-          apt-get update -y -q && \
-          apt-get install -y -q --no-install-recommends software-properties-common && \
-          add-apt-repository ppa:ubuntu-toolchain-r/volatile; \
-      fi; \
-      apt-get update -y -q && \
-      apt-get install -y -q --no-install-recommends \
-          g++-${gcc_version} \
-          gcc-${gcc_version} && \
-      update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \
-      update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \
-      update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100 && \
-      update-alternatives --set cc /usr/bin/gcc && \
-      update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100 && \
-      update-alternatives --set c++ /usr/bin/g++; \
-    fi
diff --git a/ci/docker/ubuntu-22.04-verify-rc.dockerfile b/ci/docker/ubuntu-22.04-verify-rc.dockerfile
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG arch=amd64
+FROM ${arch}/ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+COPY dev/release/setup-ubuntu.sh /
+RUN /setup-ubuntu.sh && \
+    rm /setup-ubuntu.sh && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists*
diff --git a/dev/release/setup-ubuntu.sh b/dev/release/setup-ubuntu.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -16,10 +18,40 @@
 # under the License.
 
 # A script to install dependencies required for release
-# verification on Ubuntu 20.04
+# verification on Ubuntu.
+
+set -exu
+
+codename=$(. /etc/os-release && echo ${UBUNTU_CODENAME})
 
-apt-get update
-apt-get -y install \
+case ${codename} in
+  bionic)
+    llvm=12
+    nlohmann_json=
+    python=3.8
+    apt-get update -y -q
+    apt-get install -y -q --no-install-recommends \
+      apt-transport-https \
+      ca-certificates \
+      gnupg \
+      wget
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    echo "deb https://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${llvm} main" > \
+      /etc/apt/sources.list.d/llvm.list
+    apt-get update -y -q
+    apt-get install -y -q --no-install-recommends \
+      llvm-${llvm}-dev
+    ;;
+  *)
+    nlohmann_json=3
+    python=3
+    apt-get update -y -q
+    apt-get install -y -q --no-install-recommends \
+      llvm-dev
+    ;;
+esac
+
+apt-get install -y -q --no-install-recommends \
   build-essential \
   clang \
   cmake \
@@ -30,12 +62,22 @@ apt-get -y install \
   libglib2.0-dev \
   libsqlite3-dev \
   libssl-dev \
-  llvm-dev \
   maven \
   ninja-build \
+  nlohmann-json${nlohmann_json}-dev \
   openjdk-11-jdk \
   pkg-config \
   python3-pip \
-  python3-venv \
+  python${python}-dev \
+  python${python}-venv \
   ruby-dev \
-  wget
+  wget \
+  tzdata
+
+case ${codename} in
+  bionic)
+    python${python} -m pip install -U pip
+    update-alternatives \
+      --install /usr/bin/python3 python3 /usr/bin/python${python} 1
+    ;;
+esac