Simplify protoc install in protobuf generation scripts (#11785)

python · Apr 21, 2024 · 1017c52 · 1017c52
1 parent 4872c30
commit 1017c52
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 43 deletions.
diff --git a/scripts/generate_proto_stubs.sh b/scripts/generate_proto_stubs.sh
@@ -15,38 +15,25 @@ set -ex -o pipefail
 PROTOBUF_VERSION=26.1
 MYPY_PROTOBUF_VERSION=3.6.0
 
-if uname -a | grep Darwin; then
-  # brew install coreutils wget
-  PLAT=osx
-else
-  # sudo apt install -y unzip
-  PLAT=linux
-fi
+# brew install coreutils wget
+# sudo apt install -y unzip
 REPO_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")"/..)"
 TMP_DIR="$(mktemp -d)"
 PYTHON_PROTOBUF_FILENAME="protobuf-$PROTOBUF_VERSION.zip"
-PROTOC_FILENAME="protoc-$PROTOBUF_VERSION-$PLAT-x86_64.zip"
-PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/$PROTOC_FILENAME"
 PYTHON_PROTOBUF_URL="https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/$PYTHON_PROTOBUF_FILENAME"
+PYTHON_PROTOBUF_DIR="protobuf-$PROTOBUF_VERSION"
 
 cd "$TMP_DIR"
 echo "Working in $TMP_DIR"
 
-# Install protoc
-wget "$PROTOC_URL"
-mkdir protoc_install
-unzip "$PROTOC_FILENAME" -d protoc_install
-protoc_install/bin/protoc --version
-
 # Fetch protoc-python (which contains all the .proto files)
 wget "$PYTHON_PROTOBUF_URL"
 unzip "$PYTHON_PROTOBUF_FILENAME"
-PYTHON_PROTOBUF_DIR="protobuf-$PROTOBUF_VERSION"
 
 # Prepare virtualenv
 python3 -m venv .venv
 source .venv/bin/activate
-python3 -m pip install pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
+python3 -m pip install grpcio-tools pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
 
 # Remove existing pyi
 find "$REPO_ROOT/stubs/protobuf/" -name '*_pb2.pyi' -delete
@@ -62,8 +49,9 @@ PROTO_FILES=$(grep '"//:.*_proto"' $PYTHON_PROTOBUF_DIR/python/dist/BUILD.bazel
 )
 
 # And regenerate!
-# shellcheck disable=SC2086
-protoc_install/bin/protoc \
+PROTOC_VERSION=$(python3 -m grpc_tools.protoc --version)
+echo $PROTOC_VERSION
+python3 -m grpc_tools.protoc \
   --proto_path="$PYTHON_PROTOBUF_DIR/src" \
   --mypy_out="relax_strict_optional_primitives:$REPO_ROOT/stubs/protobuf" \
   $PROTO_FILES
@@ -76,7 +64,11 @@ rm -rf "$TMP_DIR"
 cd "$REPO_ROOT"
 
 sed --in-place="" \
-  "s/extra_description = .*$/extra_description = \"Generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) on [protobuf v$PROTOBUF_VERSION](https:\/\/github.com\/protocolbuffers\/protobuf\/releases\/tag\/v$PROTOBUF_VERSION) (python protobuf==$PYTHON_PROTOBUF_VERSION)\"/" \
+  "s/extra_description = .*$/extra_description = \"\
+Partially generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) \
+and $PROTOC_VERSION \
+on [protobuf v$PROTOBUF_VERSION](https:\/\/github.com\/protocolbuffers\/protobuf\/releases\/tag\/v$PROTOBUF_VERSION) \
+(python protobuf==$PYTHON_PROTOBUF_VERSION).\"/" \
   stubs/protobuf/METADATA.toml
 
 # use `|| true` so the script still continues even if a pre-commit hook

diff --git a/scripts/sync_tensorflow_protobuf_stubs.sh b/scripts/sync_tensorflow_protobuf_stubs.sh
@@ -5,51 +5,39 @@
 
 set -euxo pipefail
 
-# Need protoc >= 3.15 for explicit optional
-PROTOBUF_VERSION=25.3 # 4.25.3
 # Whenever you update TENSORFLOW_VERSION here, version should be updated
 # in stubs/tensorflow/METADATA.toml and vice-versa.
 TENSORFLOW_VERSION=2.12.1
 MYPY_PROTOBUF_VERSION=3.6.0
 
-if uname -a | grep Darwin; then
-    # brew install coreutils wget
-    PLAT=osx
-else
-    PLAT=linux
-fi
+# brew install coreutils wget
+# sudo apt install -y unzip
 REPO_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")"/..)"
 TMP_DIR="$(mktemp -d)"
 TENSORFLOW_FILENAME="v$TENSORFLOW_VERSION.zip"
-PROTOC_FILENAME="protoc-$PROTOBUF_VERSION-$PLAT-x86_64.zip"
-PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/$PROTOC_FILENAME"
 TENSORFLOW_URL="https://github.com/tensorflow/tensorflow/archive/refs/tags/$TENSORFLOW_FILENAME"
+TENSORFLOW_DIR="tensorflow-$TENSORFLOW_VERSION"
 
 cd "$TMP_DIR"
 echo "Working in $TMP_DIR"
 
-# Install protoc
-wget "$PROTOC_URL"
-mkdir protoc_install
-unzip "$PROTOC_FILENAME" -d protoc_install
-protoc_install/bin/protoc --version
-
 # Fetch tensorflow (which contains all the .proto files)
 wget "$TENSORFLOW_URL"
 unzip "$TENSORFLOW_FILENAME"
-TENSORFLOW_DIR="tensorflow-$TENSORFLOW_VERSION"
 
 # Prepare virtualenv
 python3 -m venv .venv
 source .venv/bin/activate
-python3 -m pip install pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
+python3 -m pip install grpcio-tools pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
 
 # Remove existing pyi
 find "$REPO_ROOT/stubs/tensorflow/" -name "*_pb2.pyi" -delete
 
 # Folders here cover the more commonly used protobufs externally and
 # their dependencies. Tensorflow has more protobufs and can be added if requested.
-protoc_install/bin/protoc \
+PROTOC_VERSION=$(python3 -m grpc_tools.protoc --version)
+echo $PROTOC_VERSION
+python3 -m grpc_tools.protoc \
   --proto_path="$TENSORFLOW_DIR" \
   --mypy_out "relax_strict_optional_primitives:$REPO_ROOT/stubs/tensorflow" \
   $TENSORFLOW_DIR/tensorflow/compiler/xla/*.proto \
@@ -89,8 +77,11 @@ rm \
   stubs/tensorflow/tensorflow/core/util/example_proto_fast_parsing_test_pb2.pyi \
 
 sed --in-place="" \
-    "s/extra_description = .*$/extra_description = \"Partially generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) on tensorflow==$TENSORFLOW_VERSION\"/" \
-    stubs/tensorflow/METADATA.toml
+  "s/extra_description = .*$/extra_description = \"\
+Partially generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) \
+and $PROTOC_VERSION \
+on tensorflow==$TENSORFLOW_VERSION.\"/" \
+  stubs/tensorflow/METADATA.toml
 
 # use `|| true` so the script still continues even if a pre-commit hook
 # applies autofixes (which will result in a nonzero exit code)

diff --git a/stubs/protobuf/METADATA.toml b/stubs/protobuf/METADATA.toml
@@ -2,7 +2,7 @@
 # in scripts/generate_proto_stubs.sh and vice-versa.
 version = "5.26.*"
 upstream_repository = "https://github.com/protocolbuffers/protobuf"
-extra_description = "Generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) on [protobuf v26.1](https://github.com/protocolbuffers/protobuf/releases/tag/v26.1) (python protobuf==5.26.1)"
+extra_description = "Generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 25.1 on [protobuf v26.1](https://github.com/protocolbuffers/protobuf/releases/tag/v26.1) (python protobuf==5.26.1)"
 partial_stub = true
 
 [tool.stubtest]

diff --git a/stubs/tensorflow/METADATA.toml b/stubs/tensorflow/METADATA.toml
@@ -2,7 +2,7 @@ version = "2.15.*"
 upstream_repository = "https://github.com/tensorflow/tensorflow"
 # requires a version of numpy with a `py.typed` file
 requires = ["numpy>=1.20", "types-protobuf", "types-requests"]
-extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) on tensorflow==2.12.1"
+extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 25.1 on tensorflow==2.12.1."
 partial_stub = true
 
 [tool.stubtest]