From 0b9819ca75434be5a8ee0fbaecb31b5f6b7f057a Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Thu, 13 Apr 2023 11:35:50 -0700 Subject: [PATCH] Upgrade Protobuf C++ to 22.3 --- COMPILING.md | 4 +- buildscripts/grpc-java-artifacts/Dockerfile | 14 ++--- .../Dockerfile.multiarch.base | 1 + .../Dockerfile.ubuntu2004.base | 1 + buildscripts/kokoro/linux_artifacts.sh | 2 +- buildscripts/make_dependencies.bat | 11 +++- buildscripts/make_dependencies.sh | 54 ++++++++++++------- buildscripts/toolchain.cmake | 9 ++++ compiler/build.gradle | 2 +- 9 files changed, 67 insertions(+), 31 deletions(-) create mode 100644 buildscripts/toolchain.cmake diff --git a/COMPILING.md b/COMPILING.md index 3c5ad537e07..8596db5fb61 100644 --- a/COMPILING.md +++ b/COMPILING.md @@ -44,11 +44,11 @@ This section is only necessary if you are making changes to the code generation. Most users only need to use `skipCodegen=true` as discussed above. ### Build Protobuf -The codegen plugin is C++ code and requires protobuf 21.7 or later. +The codegen plugin is C++ code and requires protobuf 22.3 or later. For Linux, Mac and MinGW: ``` -$ PROTOBUF_VERSION=21.7 +$ PROTOBUF_VERSION=22.3 $ curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-all-$PROTOBUF_VERSION.tar.gz $ tar xzf protobuf-all-$PROTOBUF_VERSION.tar.gz $ cd protobuf-$PROTOBUF_VERSION diff --git a/buildscripts/grpc-java-artifacts/Dockerfile b/buildscripts/grpc-java-artifacts/Dockerfile index b75b54318a6..8356a5bcc44 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile +++ b/buildscripts/grpc-java-artifacts/Dockerfile @@ -1,10 +1,8 @@ FROM centos:7.9.2009 -RUN yum install -y \ - autoconf \ - automake \ - gcc-c++ \ - gcc-c++.i686 \ +RUN yum install -y centos-release-scl && \ + yum install -y \ + devtoolset-7-gcc-c++ \ glibc-devel \ glibc-devel.i686 \ java-1.8.0-openjdk-devel \ @@ -19,7 +17,9 @@ RUN yum install -y \ && \ yum clean all -# Install Maven +RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \ + tar xz -C /var/local RUN curl -Ls http://dlcdn.apache.org/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | \ tar xz -C /var/local -ENV PATH /var/local/apache-maven-3.3.9/bin:$PATH + +ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.3.9/bin:$PATH diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base index 8f7cfae2f52..f9d4fb4fcc3 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base @@ -4,6 +4,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ + cmake \ build-essential \ ca-certificates \ curl \ diff --git a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base index 2d11d76c373..126a417b58f 100644 --- a/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base +++ b/buildscripts/grpc-java-artifacts/Dockerfile.ubuntu2004.base @@ -4,6 +4,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ + cmake \ build-essential \ ca-certificates \ curl \ diff --git a/buildscripts/kokoro/linux_artifacts.sh b/buildscripts/kokoro/linux_artifacts.sh index c80d3436fb7..6a5a0a58d5a 100755 --- a/buildscripts/kokoro/linux_artifacts.sh +++ b/buildscripts/kokoro/linux_artifacts.sh @@ -11,7 +11,7 @@ readonly GRPC_JAVA_DIR="$(cd "$(dirname "$0")"/../.. && pwd)" trap spongify_logs EXIT "$GRPC_JAVA_DIR"/buildscripts/build_docker.sh -"$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh grpc-java-artifacts-x86 /grpc-java/buildscripts/build_artifacts_in_docker.sh +"$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh grpc-java-artifacts-x86 scl enable devtoolset-7 /grpc-java/buildscripts/build_artifacts_in_docker.sh # grpc-android, grpc-cronet and grpc-binder require the Android SDK, so build outside of Docker and # use --include-build for its grpc-core dependency diff --git a/buildscripts/make_dependencies.bat b/buildscripts/make_dependencies.bat index 2bbfd394d46..68ab1dac501 100644 --- a/buildscripts/make_dependencies.bat +++ b/buildscripts/make_dependencies.bat @@ -1,5 +1,7 @@ set PROTOBUF_VER=21.7 -set CMAKE_NAME=cmake-3.3.2-win32-x86 +@rem set PROTOBUF_VER=22.3 +set ABSL_VERSION=20230125.2 +set CMAKE_NAME=cmake-3.26.3-windows-x86_64 if not exist "protobuf-%PROTOBUF_VER%\build\Release\" ( call :installProto || exit /b 1 @@ -20,9 +22,14 @@ if not exist "%CMAKE_NAME%" ( set PATH=%PATH%;%cd%\%CMAKE_NAME%\bin :hasCmake @rem GitHub requires TLSv1.2, and for whatever reason our powershell doesn't have it enabled -powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/archive/v%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1 +powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1 powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('protobuf.zip', '.') }" || exit /b 1 del protobuf.zip +powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/abseil/abseil-cpp/archive/refs/tags/%ABSL_VERSION%.zip -OutFile absl.zip }" || exit /b 1 +powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('absl.zip', '.') }" || exit /b 1 +del absl.zip +@rem rmdir protobuf-%PROTOBUF_VER%\third_party\abseil-cpp +@rem rename abseil-cpp-%ABSL_VERSION% protobuf-%PROTOBUF_VER%\third_party\abseil-cpp mkdir protobuf-%PROTOBUF_VER%\build pushd protobuf-%PROTOBUF_VER%\build diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 3d02a72f4eb..114e72682e9 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -3,13 +3,16 @@ # Build protoc set -evux -o pipefail -PROTOBUF_VERSION=21.7 +PROTOBUF_VERSION=22.3 +ABSL_VERSION=20230125.2 # ARCH is x86_64 bit unless otherwise specified. ARCH="${ARCH:-x86_64}" DOWNLOAD_DIR=/tmp/source INSTALL_DIR="/tmp/protobuf-cache/$PROTOBUF_VERSION/$(uname -s)-$ARCH" +BUILDSCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" mkdir -p $DOWNLOAD_DIR +cd "$DOWNLOAD_DIR" # Start with a sane default NUM_CPU=4 @@ -26,27 +29,42 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then echo "Not building protobuf. Already built" # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date else - if [[ ! -d "$DOWNLOAD_DIR"/protobuf-"${PROTOBUF_VERSION}" ]]; then - curl -Ls https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz | tar xz -C $DOWNLOAD_DIR + if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then + curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz + curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz + rmdir "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp" + mv "abseil-cpp-$ABSL_VERSION" "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp" fi - pushd $DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION} + # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first + rm -rf "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" + mkdir "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" + pushd "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build" # install here so we don't need sudo if [[ "$ARCH" == x86* ]]; then - ./configure CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} --disable-shared \ - --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == aarch* ]]; then - ./configure --disable-shared --host=aarch64-linux-gnu --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == ppc* ]]; then - ./configure --disable-shared --host=powerpc64le-linux-gnu --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == s390* ]]; then - ./configure --disable-shared --host=s390x-linux-gnu --prefix="$INSTALL_DIR" - elif [[ "$ARCH" == loongarch* ]]; then - ./configure --disable-shared --host=loongarch64-unknown-linux-gnu --prefix="$INSTALL_DIR" + CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} cmake .. \ + -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" + else + if [[ "$ARCH" == aarch_64 ]]; then + GCC_ARCH=aarch64-linux-gnu + elif [[ "$ARCH" == ppcle_64 ]]; then + GCC_ARCH=powerpc64le-linux-gnu + elif [[ "$ARCH" == s390_64 ]]; then + GCC_ARCH=s390x-linux-gnu + elif [[ "$ARCH" == loongarch_64 ]]; then + GCC_ARCH=loongarch64-unknown-linux-gnu + else + echo "Unknown architecture: $ARCH" + exit 1 + fi + cmake .. \ + -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -Dcrosscompile_ARCH="$GCC_ARCH" \ + -DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake fi - # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first - make clean - make V=0 -j$NUM_CPU - make install + cmake --build . -j "$NUM_CPU" + cmake --install . + [ -d "$INSTALL_DIR/lib64" ] && mv "$INSTALL_DIR/lib64" "$INSTALL_DIR/lib" popd fi diff --git a/buildscripts/toolchain.cmake b/buildscripts/toolchain.cmake new file mode 100644 index 00000000000..b71515cebda --- /dev/null +++ b/buildscripts/toolchain.cmake @@ -0,0 +1,9 @@ +set(CMAKE_SYSTEM_NAME Linux) + +set(CMAKE_C_COMPILER "${crosscompile_ARCH}-gcc") +set(CMAKE_CXX_COMPILER "${crosscompile_ARCH}-g++") +set(CMAKE_FIND_ROOT_PATH "/usr/${crosscompile_ARCH}/") + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) diff --git a/compiler/build.gradle b/compiler/build.gradle index ef4c2ad7790..c348206da6b 100644 --- a/compiler/build.gradle +++ b/compiler/build.gradle @@ -99,7 +99,7 @@ model { all { if (toolChain in Gcc || toolChain in Clang) { cppCompiler.define("GRPC_VERSION", version) - cppCompiler.args "--std=c++0x" + cppCompiler.args "--std=c++14" addEnvArgs("CXXFLAGS", cppCompiler.args) addEnvArgs("CPPFLAGS", cppCompiler.args) if (osdetector.os == "osx") {