From 1017c525f84b5e78a75ad909c6a4ef2b5c0610e6 Mon Sep 17 00:00:00 2001 From: Avasam Date: Sun, 21 Apr 2024 13:33:31 -0400 Subject: [PATCH] Simplify protoc install in protobuf generation scripts (#11785) --- scripts/generate_proto_stubs.sh | 32 +++++++++------------- scripts/sync_tensorflow_protobuf_stubs.sh | 33 +++++++++-------------- stubs/protobuf/METADATA.toml | 2 +- stubs/tensorflow/METADATA.toml | 2 +- 4 files changed, 26 insertions(+), 43 deletions(-) diff --git a/scripts/generate_proto_stubs.sh b/scripts/generate_proto_stubs.sh index 1c062939e884..1d238a0bcfbc 100755 --- a/scripts/generate_proto_stubs.sh +++ b/scripts/generate_proto_stubs.sh @@ -15,38 +15,25 @@ set -ex -o pipefail PROTOBUF_VERSION=26.1 MYPY_PROTOBUF_VERSION=3.6.0 -if uname -a | grep Darwin; then - # brew install coreutils wget - PLAT=osx -else - # sudo apt install -y unzip - PLAT=linux -fi +# brew install coreutils wget +# sudo apt install -y unzip REPO_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")"/..)" TMP_DIR="$(mktemp -d)" PYTHON_PROTOBUF_FILENAME="protobuf-$PROTOBUF_VERSION.zip" -PROTOC_FILENAME="protoc-$PROTOBUF_VERSION-$PLAT-x86_64.zip" -PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/$PROTOC_FILENAME" PYTHON_PROTOBUF_URL="https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/$PYTHON_PROTOBUF_FILENAME" +PYTHON_PROTOBUF_DIR="protobuf-$PROTOBUF_VERSION" cd "$TMP_DIR" echo "Working in $TMP_DIR" -# Install protoc -wget "$PROTOC_URL" -mkdir protoc_install -unzip "$PROTOC_FILENAME" -d protoc_install -protoc_install/bin/protoc --version - # Fetch protoc-python (which contains all the .proto files) wget "$PYTHON_PROTOBUF_URL" unzip "$PYTHON_PROTOBUF_FILENAME" -PYTHON_PROTOBUF_DIR="protobuf-$PROTOBUF_VERSION" # Prepare virtualenv python3 -m venv .venv source .venv/bin/activate -python3 -m pip install pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION" +python3 -m pip install grpcio-tools pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION" # Remove existing pyi find "$REPO_ROOT/stubs/protobuf/" -name '*_pb2.pyi' -delete @@ -62,8 +49,9 @@ PROTO_FILES=$(grep '"//:.*_proto"' $PYTHON_PROTOBUF_DIR/python/dist/BUILD.bazel ) # And regenerate! -# shellcheck disable=SC2086 -protoc_install/bin/protoc \ +PROTOC_VERSION=$(python3 -m grpc_tools.protoc --version) +echo $PROTOC_VERSION +python3 -m grpc_tools.protoc \ --proto_path="$PYTHON_PROTOBUF_DIR/src" \ --mypy_out="relax_strict_optional_primitives:$REPO_ROOT/stubs/protobuf" \ $PROTO_FILES @@ -76,7 +64,11 @@ rm -rf "$TMP_DIR" cd "$REPO_ROOT" sed --in-place="" \ - "s/extra_description = .*$/extra_description = \"Generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) on [protobuf v$PROTOBUF_VERSION](https:\/\/github.com\/protocolbuffers\/protobuf\/releases\/tag\/v$PROTOBUF_VERSION) (python protobuf==$PYTHON_PROTOBUF_VERSION)\"/" \ + "s/extra_description = .*$/extra_description = \"\ +Partially generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) \ +and $PROTOC_VERSION \ +on [protobuf v$PROTOBUF_VERSION](https:\/\/github.com\/protocolbuffers\/protobuf\/releases\/tag\/v$PROTOBUF_VERSION) \ +(python protobuf==$PYTHON_PROTOBUF_VERSION).\"/" \ stubs/protobuf/METADATA.toml # use `|| true` so the script still continues even if a pre-commit hook diff --git a/scripts/sync_tensorflow_protobuf_stubs.sh b/scripts/sync_tensorflow_protobuf_stubs.sh index 4813e3497b0b..432ee66d94a8 100755 --- a/scripts/sync_tensorflow_protobuf_stubs.sh +++ b/scripts/sync_tensorflow_protobuf_stubs.sh @@ -5,51 +5,39 @@ set -euxo pipefail -# Need protoc >= 3.15 for explicit optional -PROTOBUF_VERSION=25.3 # 4.25.3 # Whenever you update TENSORFLOW_VERSION here, version should be updated # in stubs/tensorflow/METADATA.toml and vice-versa. TENSORFLOW_VERSION=2.12.1 MYPY_PROTOBUF_VERSION=3.6.0 -if uname -a | grep Darwin; then - # brew install coreutils wget - PLAT=osx -else - PLAT=linux -fi +# brew install coreutils wget +# sudo apt install -y unzip REPO_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")"/..)" TMP_DIR="$(mktemp -d)" TENSORFLOW_FILENAME="v$TENSORFLOW_VERSION.zip" -PROTOC_FILENAME="protoc-$PROTOBUF_VERSION-$PLAT-x86_64.zip" -PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/$PROTOC_FILENAME" TENSORFLOW_URL="https://github.com/tensorflow/tensorflow/archive/refs/tags/$TENSORFLOW_FILENAME" +TENSORFLOW_DIR="tensorflow-$TENSORFLOW_VERSION" cd "$TMP_DIR" echo "Working in $TMP_DIR" -# Install protoc -wget "$PROTOC_URL" -mkdir protoc_install -unzip "$PROTOC_FILENAME" -d protoc_install -protoc_install/bin/protoc --version - # Fetch tensorflow (which contains all the .proto files) wget "$TENSORFLOW_URL" unzip "$TENSORFLOW_FILENAME" -TENSORFLOW_DIR="tensorflow-$TENSORFLOW_VERSION" # Prepare virtualenv python3 -m venv .venv source .venv/bin/activate -python3 -m pip install pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION" +python3 -m pip install grpcio-tools pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION" # Remove existing pyi find "$REPO_ROOT/stubs/tensorflow/" -name "*_pb2.pyi" -delete # Folders here cover the more commonly used protobufs externally and # their dependencies. Tensorflow has more protobufs and can be added if requested. -protoc_install/bin/protoc \ +PROTOC_VERSION=$(python3 -m grpc_tools.protoc --version) +echo $PROTOC_VERSION +python3 -m grpc_tools.protoc \ --proto_path="$TENSORFLOW_DIR" \ --mypy_out "relax_strict_optional_primitives:$REPO_ROOT/stubs/tensorflow" \ $TENSORFLOW_DIR/tensorflow/compiler/xla/*.proto \ @@ -89,8 +77,11 @@ rm \ stubs/tensorflow/tensorflow/core/util/example_proto_fast_parsing_test_pb2.pyi \ sed --in-place="" \ - "s/extra_description = .*$/extra_description = \"Partially generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) on tensorflow==$TENSORFLOW_VERSION\"/" \ - stubs/tensorflow/METADATA.toml + "s/extra_description = .*$/extra_description = \"\ +Partially generated using [mypy-protobuf==$MYPY_PROTOBUF_VERSION](https:\/\/github.com\/nipunn1313\/mypy-protobuf\/tree\/v$MYPY_PROTOBUF_VERSION) \ +and $PROTOC_VERSION \ +on tensorflow==$TENSORFLOW_VERSION.\"/" \ + stubs/tensorflow/METADATA.toml # use `|| true` so the script still continues even if a pre-commit hook # applies autofixes (which will result in a nonzero exit code) diff --git a/stubs/protobuf/METADATA.toml b/stubs/protobuf/METADATA.toml index b962f09c4bb8..ad7078f37e31 100644 --- a/stubs/protobuf/METADATA.toml +++ b/stubs/protobuf/METADATA.toml @@ -2,7 +2,7 @@ # in scripts/generate_proto_stubs.sh and vice-versa. version = "5.26.*" upstream_repository = "https://github.com/protocolbuffers/protobuf" -extra_description = "Generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) on [protobuf v26.1](https://github.com/protocolbuffers/protobuf/releases/tag/v26.1) (python protobuf==5.26.1)" +extra_description = "Generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 25.1 on [protobuf v26.1](https://github.com/protocolbuffers/protobuf/releases/tag/v26.1) (python protobuf==5.26.1)" partial_stub = true [tool.stubtest] diff --git a/stubs/tensorflow/METADATA.toml b/stubs/tensorflow/METADATA.toml index a040d3b32056..1d08d97fecd2 100644 --- a/stubs/tensorflow/METADATA.toml +++ b/stubs/tensorflow/METADATA.toml @@ -2,7 +2,7 @@ version = "2.15.*" upstream_repository = "https://github.com/tensorflow/tensorflow" # requires a version of numpy with a `py.typed` file requires = ["numpy>=1.20", "types-protobuf", "types-requests"] -extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) on tensorflow==2.12.1" +extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 25.1 on tensorflow==2.12.1." partial_stub = true [tool.stubtest]