Skip to content

Commit

Permalink
Upgrade Protobuf C++ to 22.3
Browse files Browse the repository at this point in the history
  • Loading branch information
ejona86 committed Apr 20, 2023
1 parent 8aa2547 commit 0b9819c
Show file tree
Hide file tree
Showing 9 changed files with 67 additions and 31 deletions.
4 changes: 2 additions & 2 deletions COMPILING.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,11 @@ This section is only necessary if you are making changes to the code
generation. Most users only need to use `skipCodegen=true` as discussed above.

### Build Protobuf
The codegen plugin is C++ code and requires protobuf 21.7 or later.
The codegen plugin is C++ code and requires protobuf 22.3 or later.

For Linux, Mac and MinGW:
```
$ PROTOBUF_VERSION=21.7
$ PROTOBUF_VERSION=22.3
$ curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-all-$PROTOBUF_VERSION.tar.gz
$ tar xzf protobuf-all-$PROTOBUF_VERSION.tar.gz
$ cd protobuf-$PROTOBUF_VERSION
Expand Down
14 changes: 7 additions & 7 deletions buildscripts/grpc-java-artifacts/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
FROM centos:7.9.2009

RUN yum install -y \
autoconf \
automake \
gcc-c++ \
gcc-c++.i686 \
RUN yum install -y centos-release-scl && \
yum install -y \
devtoolset-7-gcc-c++ \
glibc-devel \
glibc-devel.i686 \
java-1.8.0-openjdk-devel \
Expand All @@ -19,7 +17,9 @@ RUN yum install -y \
&& \
yum clean all

# Install Maven
RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.26.3/cmake-3.26.3-linux-x86_64.tar.gz | \
tar xz -C /var/local
RUN curl -Ls http://dlcdn.apache.org/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | \
tar xz -C /var/local
ENV PATH /var/local/apache-maven-3.3.9/bin:$PATH

ENV PATH /var/local/cmake-3.26.3-linux-x86_64/bin:/var/local/apache-maven-3.3.9/bin:$PATH
1 change: 1 addition & 0 deletions buildscripts/grpc-java-artifacts/Dockerfile.multiarch.base
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get upgrade -y && \
apt-get install -y --no-install-recommends \
cmake \
build-essential \
ca-certificates \
curl \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
apt-get upgrade -y && \
apt-get install -y --no-install-recommends \
cmake \
build-essential \
ca-certificates \
curl \
Expand Down
2 changes: 1 addition & 1 deletion buildscripts/kokoro/linux_artifacts.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ readonly GRPC_JAVA_DIR="$(cd "$(dirname "$0")"/../.. && pwd)"
trap spongify_logs EXIT

"$GRPC_JAVA_DIR"/buildscripts/build_docker.sh
"$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh grpc-java-artifacts-x86 /grpc-java/buildscripts/build_artifacts_in_docker.sh
"$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh grpc-java-artifacts-x86 scl enable devtoolset-7 /grpc-java/buildscripts/build_artifacts_in_docker.sh

# grpc-android, grpc-cronet and grpc-binder require the Android SDK, so build outside of Docker and
# use --include-build for its grpc-core dependency
Expand Down
11 changes: 9 additions & 2 deletions buildscripts/make_dependencies.bat
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
set PROTOBUF_VER=21.7
set CMAKE_NAME=cmake-3.3.2-win32-x86
@rem set PROTOBUF_VER=22.3
set ABSL_VERSION=20230125.2
set CMAKE_NAME=cmake-3.26.3-windows-x86_64

if not exist "protobuf-%PROTOBUF_VER%\build\Release\" (
call :installProto || exit /b 1
Expand All @@ -20,9 +22,14 @@ if not exist "%CMAKE_NAME%" (
set PATH=%PATH%;%cd%\%CMAKE_NAME%\bin
:hasCmake
@rem GitHub requires TLSv1.2, and for whatever reason our powershell doesn't have it enabled
powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/archive/v%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1
powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/google/protobuf/releases/download/v%PROTOBUF_VER%/protobuf-%PROTOBUF_VER%.zip -OutFile protobuf.zip }" || exit /b 1
powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('protobuf.zip', '.') }" || exit /b 1
del protobuf.zip
powershell -command "$ErrorActionPreference = 'stop'; & { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ; iwr https://github.com/abseil/abseil-cpp/archive/refs/tags/%ABSL_VERSION%.zip -OutFile absl.zip }" || exit /b 1
powershell -command "$ErrorActionPreference = 'stop'; & { Add-Type -AssemblyName System.IO.Compression.FileSystem; [System.IO.Compression.ZipFile]::ExtractToDirectory('absl.zip', '.') }" || exit /b 1
del absl.zip
@rem rmdir protobuf-%PROTOBUF_VER%\third_party\abseil-cpp
@rem rename abseil-cpp-%ABSL_VERSION% protobuf-%PROTOBUF_VER%\third_party\abseil-cpp
mkdir protobuf-%PROTOBUF_VER%\build
pushd protobuf-%PROTOBUF_VER%\build

Expand Down
54 changes: 36 additions & 18 deletions buildscripts/make_dependencies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,16 @@
# Build protoc
set -evux -o pipefail

PROTOBUF_VERSION=21.7
PROTOBUF_VERSION=22.3
ABSL_VERSION=20230125.2

# ARCH is x86_64 bit unless otherwise specified.
ARCH="${ARCH:-x86_64}"
DOWNLOAD_DIR=/tmp/source
INSTALL_DIR="/tmp/protobuf-cache/$PROTOBUF_VERSION/$(uname -s)-$ARCH"
BUILDSCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
mkdir -p $DOWNLOAD_DIR
cd "$DOWNLOAD_DIR"

# Start with a sane default
NUM_CPU=4
Expand All @@ -26,27 +29,42 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then
echo "Not building protobuf. Already built"
# TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date
else
if [[ ! -d "$DOWNLOAD_DIR"/protobuf-"${PROTOBUF_VERSION}" ]]; then
curl -Ls https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz | tar xz -C $DOWNLOAD_DIR
if [[ ! -d "protobuf-${PROTOBUF_VERSION}" ]]; then
curl -Ls "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz" | tar xz
curl -Ls "https://github.com/abseil/abseil-cpp/archive/refs/tags/${ABSL_VERSION}.tar.gz" | tar xz
rmdir "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp"
mv "abseil-cpp-$ABSL_VERSION" "protobuf-$PROTOBUF_VERSION/third_party/abseil-cpp"
fi
pushd $DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}
# the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first
rm -rf "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build"
mkdir "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build"
pushd "$DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}/build"
# install here so we don't need sudo
if [[ "$ARCH" == x86* ]]; then
./configure CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} --disable-shared \
--prefix="$INSTALL_DIR"
elif [[ "$ARCH" == aarch* ]]; then
./configure --disable-shared --host=aarch64-linux-gnu --prefix="$INSTALL_DIR"
elif [[ "$ARCH" == ppc* ]]; then
./configure --disable-shared --host=powerpc64le-linux-gnu --prefix="$INSTALL_DIR"
elif [[ "$ARCH" == s390* ]]; then
./configure --disable-shared --host=s390x-linux-gnu --prefix="$INSTALL_DIR"
elif [[ "$ARCH" == loongarch* ]]; then
./configure --disable-shared --host=loongarch64-unknown-linux-gnu --prefix="$INSTALL_DIR"
CFLAGS=-m${ARCH#*_} CXXFLAGS=-m${ARCH#*_} cmake .. \
-DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \
-DCMAKE_INSTALL_PREFIX="$INSTALL_DIR"
else
if [[ "$ARCH" == aarch_64 ]]; then
GCC_ARCH=aarch64-linux-gnu
elif [[ "$ARCH" == ppcle_64 ]]; then
GCC_ARCH=powerpc64le-linux-gnu
elif [[ "$ARCH" == s390_64 ]]; then
GCC_ARCH=s390x-linux-gnu
elif [[ "$ARCH" == loongarch_64 ]]; then
GCC_ARCH=loongarch64-unknown-linux-gnu
else
echo "Unknown architecture: $ARCH"
exit 1
fi
cmake .. \
-DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF \
-DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" -Dcrosscompile_ARCH="$GCC_ARCH" \
-DCMAKE_TOOLCHAIN_FILE=$BUILDSCRIPTS_DIR/toolchain.cmake
fi
# the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first
make clean
make V=0 -j$NUM_CPU
make install
cmake --build . -j "$NUM_CPU"
cmake --install .
[ -d "$INSTALL_DIR/lib64" ] && mv "$INSTALL_DIR/lib64" "$INSTALL_DIR/lib"
popd
fi

Expand Down
9 changes: 9 additions & 0 deletions buildscripts/toolchain.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
set(CMAKE_SYSTEM_NAME Linux)

set(CMAKE_C_COMPILER "${crosscompile_ARCH}-gcc")
set(CMAKE_CXX_COMPILER "${crosscompile_ARCH}-g++")
set(CMAKE_FIND_ROOT_PATH "/usr/${crosscompile_ARCH}/")

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
2 changes: 1 addition & 1 deletion compiler/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ model {
all {
if (toolChain in Gcc || toolChain in Clang) {
cppCompiler.define("GRPC_VERSION", version)
cppCompiler.args "--std=c++0x"
cppCompiler.args "--std=c++14"
addEnvArgs("CXXFLAGS", cppCompiler.args)
addEnvArgs("CPPFLAGS", cppCompiler.args)
if (osdetector.os == "osx") {
Expand Down

0 comments on commit 0b9819c

Please sign in to comment.