Skip to content

Commit

Permalink
Bump protobuf to 5.26.* (python#11784)
Browse files Browse the repository at this point in the history 
Also update generate_proto_stubs script
  • Loading branch information
@Avasam
Avasam authored Apr 19, 2024
1 parent 51d6ecf commit ce049d5
Show file tree
Hide file tree
Showing 11 changed files with 99 additions and 93 deletions.
29 changes: 12 additions & 17 deletions scripts/generate_proto_stubs.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,15 @@ set -ex -o pipefail
#
# Whenever you update PROTOBUF_VERSION here, version should be updated
# in stubs/protobuf/METADATA.toml and vice-versa.
PROTOBUF_VERSION=25.3
PROTOBUF_VERSION=26.1
MYPY_PROTOBUF_VERSION=3.6.0

if uname -a | grep Darwin; then
# brew install coreutils wget
PLAT=osx
# brew install coreutils wget
PLAT=osx
else
PLAT=linux
# sudo apt install -y unzip
PLAT=linux
fi
REPO_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")"/..)"
TMP_DIR="$(mktemp -d)"
Expand Down Expand Up @@ -51,19 +52,13 @@ python3 -m pip install pre-commit mypy-protobuf=="$MYPY_PROTOBUF_VERSION"
find "$REPO_ROOT/stubs/protobuf/" -name '*_pb2.pyi' -delete

# Roughly reproduce the subset of .proto files on the public interface as described
# by find_package_modules in the protobuf setup.py.
# The logic (as of 3.20.1) can roughly be described as a allowlist of .proto files
# further limited to exclude *test* and internal/
# https://github.com/protocolbuffers/protobuf/blob/master/python/setup.py
PROTO_FILES=$(grep "GenProto.*google" $PYTHON_PROTOBUF_DIR/python/setup.py | \
cut -d\' -f2 | \
grep -v "test" | \
grep -v google/protobuf/internal/ | \
grep -v google/protobuf/pyext/python.proto | \
grep -v src/google/protobuf/util/json_format.proto | \
grep -v src/google/protobuf/util/json_format_proto3.proto | \
sed "s:^:$PYTHON_PROTOBUF_DIR/python/:" | \
xargs -L1 realpath --relative-to=. \
# in py_proto_library calls in
# https://github.com/protocolbuffers/protobuf/blob/main/python/dist/BUILD.bazel
PROTO_FILES=$(grep '"//:.*_proto"' $PYTHON_PROTOBUF_DIR/python/dist/BUILD.bazel | \
cut -d\" -f2 | \
sed "s://\::$PYTHON_PROTOBUF_DIR/src/google/protobuf/:" | \
sed "s:_proto:.proto:" | \
sed "s:compiler_:compiler/:" \
)

# And regenerate!
Expand Down
6 changes: 1 addition & 5 deletions stubs/protobuf/@tests/stubtest_allowlist.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,8 @@ google.protobuf.message.Message.DESCRIPTOR
# See https://github.com/protocolbuffers/protobuf/blob/3ea30d80847cd9561db570ae7f673afc15523545/python/google/protobuf/message.py#L67
google.protobuf.message.Message.Extensions

# These are typed as (self, **kwargs) at runtime as thin wrapper functions
# around the underlying true typing. We prefer the true typing
# Has *args that would fail at runtime with any positional argument
google.protobuf.internal.containers.BaseContainer.sort
google.protobuf.message.Message.SerializePartialToString
google.protobuf.message.Message.SerializeToString
google.protobuf.text_format.MessageToBytes

# Stubbed as static method, but actually exists as a property that's
# a function. Typeshed's typing is more useful
Expand Down
4 changes: 2 additions & 2 deletions stubs/protobuf/METADATA.toml
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# Whenever you update version here, PROTOBUF_VERSION should be updated
# in scripts/generate_proto_stubs.sh and vice-versa.
version = "4.25.*"
version = "5.26.*"
upstream_repository = "https://github.com/protocolbuffers/protobuf"
extra_description = "Generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) on [protobuf v25.3](https://github.com/protocolbuffers/protobuf/releases/tag/v25.3) (python protobuf==4.25.3)"
extra_description = "Generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) on [protobuf v26.1](https://github.com/protocolbuffers/protobuf/releases/tag/v26.1) (python protobuf==5.26.1)"
partial_stub = true

[tool.stubtest]
Expand Down
20 changes: 18 additions & 2 deletions stubs/protobuf/google/protobuf/compiler/plugin_pb2.pyi
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,8 @@ class CodeGeneratorResponse(google.protobuf.message.Message):

ERROR_FIELD_NUMBER: builtins.int
SUPPORTED_FEATURES_FIELD_NUMBER: builtins.int
MINIMUM_EDITION_FIELD_NUMBER: builtins.int
MAXIMUM_EDITION_FIELD_NUMBER: builtins.int
FILE_FIELD_NUMBER: builtins.int
error: builtins.str
"""Error message. If non-empty, code generation failed. The plugin process
Expand All @@ -253,16 +255,30 @@ class CodeGeneratorResponse(google.protobuf.message.Message):
"""A bitmask of supported features that the code generator supports.
This is a bitwise "or" of values from the Feature enum.
"""
minimum_edition: builtins.int
"""The minimum edition this plugin supports. This will be treated as an
Edition enum, but we want to allow unknown values. It should be specified
according the edition enum value, *not* the edition number. Only takes
effect for plugins that have FEATURE_SUPPORTS_EDITIONS set.
"""
maximum_edition: builtins.int
"""The maximum edition this plugin supports. This will be treated as an
Edition enum, but we want to allow unknown values. It should be specified
according the edition enum value, *not* the edition number. Only takes
effect for plugins that have FEATURE_SUPPORTS_EDITIONS set.
"""
@property
def file(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___CodeGeneratorResponse.File]: ...
def __init__(
self,
*,
error: builtins.str | None = ...,
supported_features: builtins.int | None = ...,
minimum_edition: builtins.int | None = ...,
maximum_edition: builtins.int | None = ...,
file: collections.abc.Iterable[global___CodeGeneratorResponse.File] | None = ...,
) -> None: ...
def HasField(self, field_name: typing.Literal["error", b"error", "supported_features", b"supported_features"]) -> builtins.bool: ...
def ClearField(self, field_name: typing.Literal["error", b"error", "file", b"file", "supported_features", b"supported_features"]) -> None: ...
def HasField(self, field_name: typing.Literal["error", b"error", "maximum_edition", b"maximum_edition", "minimum_edition", b"minimum_edition", "supported_features", b"supported_features"]) -> builtins.bool: ...
def ClearField(self, field_name: typing.Literal["error", b"error", "file", b"file", "maximum_edition", b"maximum_edition", "minimum_edition", b"minimum_edition", "supported_features", b"supported_features"]) -> None: ...

global___CodeGeneratorResponse = CodeGeneratorResponse
11 changes: 5 additions & 6 deletions stubs/protobuf/google/protobuf/descriptor.pyi
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,6 @@ class Descriptor(_NestedDescriptorBase):
extension_ranges: Any
oneofs: Any
oneofs_by_name: Any
@property
def syntax(self): ...
def __init__(
self,
name: str,
Expand Down Expand Up @@ -156,7 +154,8 @@ class FieldDescriptor(DescriptorBase):
number: Any
type: Any
cpp_type: Any
label: Any
@property
def label(self): ...
has_default_value: Any
default_value: Any
containing_type: Any
Expand Down Expand Up @@ -319,6 +318,7 @@ class FileDescriptor(DescriptorBase):
dependencies=None,
public_dependencies=None,
syntax=None,
edition=None,
pool=None,
create_key=None,
): ...
Expand All @@ -327,8 +327,6 @@ class FileDescriptor(DescriptorBase):
message_types_by_name: Any
name: Any
package: Any
@property
def syntax(self): ...
serialized_pb: Any
enum_types_by_name: Any
extensions_by_name: Any
Expand All @@ -345,11 +343,12 @@ class FileDescriptor(DescriptorBase):
dependencies=None,
public_dependencies=None,
syntax=None,
edition=None,
pool=None,
create_key=None,
) -> None: ...
def CopyToProto(self, proto): ...
def GetOptions(self) -> FileOptions: ...

def MakeDescriptor(desc_proto, package="", build_file_if_cpp=True, syntax=None): ...
def MakeDescriptor(desc_proto, package="", build_file_if_cpp=True, syntax=None, edition=None, file_desc=None): ...
def _ParseOptions(message: Message, string: bytes) -> Message: ...
65 changes: 33 additions & 32 deletions stubs/protobuf/google/protobuf/descriptor_pb2.pyi
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ class _EditionEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTy
should not be depended on, but they will always be time-ordered for easy
comparison.
"""
EDITION_2024: _Edition.ValueType # 1001
EDITION_1_TEST_ONLY: _Edition.ValueType # 1
"""Placeholder editions for testing feature resolution. These should not be
used or relyed on outside of tests.
Expand All @@ -55,6 +56,11 @@ class _EditionEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTy
EDITION_99997_TEST_ONLY: _Edition.ValueType # 99997
EDITION_99998_TEST_ONLY: _Edition.ValueType # 99998
EDITION_99999_TEST_ONLY: _Edition.ValueType # 99999
EDITION_MAX: _Edition.ValueType # 2147483647
"""Placeholder for specifying unbounded edition support. This should only
ever be used by plugins that can expect to never require any changes to
support a new edition.
"""

class Edition(_Edition, metaclass=_EditionEnumTypeWrapper):
"""The full set of known editions."""
Expand All @@ -73,6 +79,7 @@ EDITION_2023: Edition.ValueType # 1000
should not be depended on, but they will always be time-ordered for easy
comparison.
"""
EDITION_2024: Edition.ValueType # 1001
EDITION_1_TEST_ONLY: Edition.ValueType # 1
"""Placeholder editions for testing feature resolution. These should not be
used or relyed on outside of tests.
Expand All @@ -81,6 +88,11 @@ EDITION_2_TEST_ONLY: Edition.ValueType # 2
EDITION_99997_TEST_ONLY: Edition.ValueType # 99997
EDITION_99998_TEST_ONLY: Edition.ValueType # 99998
EDITION_99999_TEST_ONLY: Edition.ValueType # 99999
EDITION_MAX: Edition.ValueType # 2147483647
"""Placeholder for specifying unbounded edition support. This should only
ever be used by plugins that can expect to never require any changes to
support a new edition.
"""
global___Edition = Edition

@typing.final
Expand Down Expand Up @@ -557,12 +569,12 @@ class FieldDescriptorProto(google.protobuf.message.Message):
"""If true, this is a proto3 "optional". When a proto3 field is optional, it
tracks presence regardless of field type.
When proto3_optional is true, this field must be belong to a oneof to
signal to old proto3 clients that presence is tracked for this field. This
oneof is known as a "synthetic" oneof, and this field must be its sole
member (each proto3 optional field gets its own synthetic oneof). Synthetic
oneofs exist in the descriptor only, and do not generate any API. Synthetic
oneofs must be ordered after all "real" oneofs.
When proto3_optional is true, this field must belong to a oneof to signal
to old proto3 clients that presence is tracked for this field. This oneof
is known as a "synthetic" oneof, and this field must be its sole member
(each proto3 optional field gets its own synthetic oneof). Synthetic oneofs
exist in the descriptor only, and do not generate any API. Synthetic oneofs
must be ordered after all "real" oneofs.
For message fields, proto3_optional doesn't create any semantic change,
since non-repeated message fields always track presence. However it still
Expand Down Expand Up @@ -851,7 +863,6 @@ class FileOptions(google.protobuf.message.Message):
CC_GENERIC_SERVICES_FIELD_NUMBER: builtins.int
JAVA_GENERIC_SERVICES_FIELD_NUMBER: builtins.int
PY_GENERIC_SERVICES_FIELD_NUMBER: builtins.int
PHP_GENERIC_SERVICES_FIELD_NUMBER: builtins.int
DEPRECATED_FIELD_NUMBER: builtins.int
CC_ENABLE_ARENAS_FIELD_NUMBER: builtins.int
OBJC_CLASS_PREFIX_FIELD_NUMBER: builtins.int
Expand Down Expand Up @@ -916,7 +927,6 @@ class FileOptions(google.protobuf.message.Message):
"""
java_generic_services: builtins.bool
py_generic_services: builtins.bool
php_generic_services: builtins.bool
deprecated: builtins.bool
"""Is this file deprecated?
Depending on the target platform, this can emit Deprecated annotations
Expand Down Expand Up @@ -981,7 +991,6 @@ class FileOptions(google.protobuf.message.Message):
cc_generic_services: builtins.bool | None = ...,
java_generic_services: builtins.bool | None = ...,
py_generic_services: builtins.bool | None = ...,
php_generic_services: builtins.bool | None = ...,
deprecated: builtins.bool | None = ...,
cc_enable_arenas: builtins.bool | None = ...,
objc_class_prefix: builtins.str | None = ...,
Expand All @@ -994,8 +1003,8 @@ class FileOptions(google.protobuf.message.Message):
features: global___FeatureSet | None = ...,
uninterpreted_option: collections.abc.Iterable[global___UninterpretedOption] | None = ...,
) -> None: ...
def HasField(self, field_name: typing.Literal["cc_enable_arenas", b"cc_enable_arenas", "cc_generic_services", b"cc_generic_services", "csharp_namespace", b"csharp_namespace", "deprecated", b"deprecated", "features", b"features", "go_package", b"go_package", "java_generate_equals_and_hash", b"java_generate_equals_and_hash", "java_generic_services", b"java_generic_services", "java_multiple_files", b"java_multiple_files", "java_outer_classname", b"java_outer_classname", "java_package", b"java_package", "java_string_check_utf8", b"java_string_check_utf8", "objc_class_prefix", b"objc_class_prefix", "optimize_for", b"optimize_for", "php_class_prefix", b"php_class_prefix", "php_generic_services", b"php_generic_services", "php_metadata_namespace", b"php_metadata_namespace", "php_namespace", b"php_namespace", "py_generic_services", b"py_generic_services", "ruby_package", b"ruby_package", "swift_prefix", b"swift_prefix"]) -> builtins.bool: ...
def ClearField(self, field_name: typing.Literal["cc_enable_arenas", b"cc_enable_arenas", "cc_generic_services", b"cc_generic_services", "csharp_namespace", b"csharp_namespace", "deprecated", b"deprecated", "features", b"features", "go_package", b"go_package", "java_generate_equals_and_hash", b"java_generate_equals_and_hash", "java_generic_services", b"java_generic_services", "java_multiple_files", b"java_multiple_files", "java_outer_classname", b"java_outer_classname", "java_package", b"java_package", "java_string_check_utf8", b"java_string_check_utf8", "objc_class_prefix", b"objc_class_prefix", "optimize_for", b"optimize_for", "php_class_prefix", b"php_class_prefix", "php_generic_services", b"php_generic_services", "php_metadata_namespace", b"php_metadata_namespace", "php_namespace", b"php_namespace", "py_generic_services", b"py_generic_services", "ruby_package", b"ruby_package", "swift_prefix", b"swift_prefix", "uninterpreted_option", b"uninterpreted_option"]) -> None: ...
def HasField(self, field_name: typing.Literal["cc_enable_arenas", b"cc_enable_arenas", "cc_generic_services", b"cc_generic_services", "csharp_namespace", b"csharp_namespace", "deprecated", b"deprecated", "features", b"features", "go_package", b"go_package", "java_generate_equals_and_hash", b"java_generate_equals_and_hash", "java_generic_services", b"java_generic_services", "java_multiple_files", b"java_multiple_files", "java_outer_classname", b"java_outer_classname", "java_package", b"java_package", "java_string_check_utf8", b"java_string_check_utf8", "objc_class_prefix", b"objc_class_prefix", "optimize_for", b"optimize_for", "php_class_prefix", b"php_class_prefix", "php_metadata_namespace", b"php_metadata_namespace", "php_namespace", b"php_namespace", "py_generic_services", b"py_generic_services", "ruby_package", b"ruby_package", "swift_prefix", b"swift_prefix"]) -> builtins.bool: ...
def ClearField(self, field_name: typing.Literal["cc_enable_arenas", b"cc_enable_arenas", "cc_generic_services", b"cc_generic_services", "csharp_namespace", b"csharp_namespace", "deprecated", b"deprecated", "features", b"features", "go_package", b"go_package", "java_generate_equals_and_hash", b"java_generate_equals_and_hash", "java_generic_services", b"java_generic_services", "java_multiple_files", b"java_multiple_files", "java_outer_classname", b"java_outer_classname", "java_package", b"java_package", "java_string_check_utf8", b"java_string_check_utf8", "objc_class_prefix", b"objc_class_prefix", "optimize_for", b"optimize_for", "php_class_prefix", b"php_class_prefix", "php_metadata_namespace", b"php_metadata_namespace", "php_namespace", b"php_namespace", "py_generic_services", b"py_generic_services", "ruby_package", b"ruby_package", "swift_prefix", b"swift_prefix", "uninterpreted_option", b"uninterpreted_option"]) -> None: ...

global___FileOptions = FileOptions

Expand Down Expand Up @@ -1042,11 +1051,7 @@ class MessageOptions(google.protobuf.message.Message):
this is a formalization for deprecating messages.
"""
map_entry: builtins.bool
"""NOTE: Do not set the option in .proto files. Always use the maps syntax
instead. The option should only be implicitly set by the proto compiler
parser.
Whether the message is an automatically generated map entry type for the
"""Whether the message is an automatically generated map entry type for the
maps field.
For maps fields:
Expand All @@ -1063,6 +1068,10 @@ class MessageOptions(google.protobuf.message.Message):
use a native map in the target language to hold the keys and values.
The reflection APIs in such implementations still need to work as
if the field is a repeated message field.
NOTE: Do not set the option in .proto files. Always use the maps syntax
instead. The option should only be implicitly set by the proto compiler
parser.
"""
deprecated_legacy_json_field_conflicts: builtins.bool
"""Enable the legacy handling of JSON field name conflicts. This lowercases
Expand Down Expand Up @@ -1290,19 +1299,11 @@ class FieldOptions(google.protobuf.message.Message):
call from multiple threads concurrently, while non-const methods continue
to require exclusive access.
Note that implementations may choose not to check required fields within
a lazy sub-message. That is, calling IsInitialized() on the outer message
may return true even if the inner message has missing required fields.
This is necessary because otherwise the inner message would have to be
parsed in order to perform the check, defeating the purpose of lazy
parsing. An implementation which chooses not to check required fields
must be consistent about it. That is, for any particular sub-message, the
implementation must either *always* check its required fields, or *never*
check its required fields, regardless of whether or not the message has
been parsed.
As of May 2022, lazy verifies the contents of the byte stream during
parsing. An invalid byte stream will cause the overall parsing to fail.
Note that lazy message fields are still eagerly verified to check
ill-formed wireformat or missing required fields. Calling IsInitialized()
on the outer message would fail if the inner message has missing required
fields. Failed verification would result in parsing failure (except when
uninitialized messages are acceptable).
"""
unverified_lazy: builtins.bool
"""unverified_lazy does no correctness checks on the byte stream. This should
Expand Down Expand Up @@ -1712,13 +1713,13 @@ class FeatureSet(google.protobuf.message.Message):
class _Utf8ValidationEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[FeatureSet._Utf8Validation.ValueType], builtins.type):
DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
UTF8_VALIDATION_UNKNOWN: FeatureSet._Utf8Validation.ValueType # 0
NONE: FeatureSet._Utf8Validation.ValueType # 1
VERIFY: FeatureSet._Utf8Validation.ValueType # 2
NONE: FeatureSet._Utf8Validation.ValueType # 3

class Utf8Validation(_Utf8Validation, metaclass=_Utf8ValidationEnumTypeWrapper): ...
UTF8_VALIDATION_UNKNOWN: FeatureSet.Utf8Validation.ValueType # 0
NONE: FeatureSet.Utf8Validation.ValueType # 1
VERIFY: FeatureSet.Utf8Validation.ValueType # 2
NONE: FeatureSet.Utf8Validation.ValueType # 3

class _MessageEncoding:
ValueType = typing.NewType("ValueType", builtins.int)
Expand Down Expand Up @@ -1912,7 +1913,7 @@ class SourceCodeInfo(google.protobuf.message.Message):
location.
Each element is a field number or an index. They form a path from
the root FileDescriptorProto to the place where the definition occurs.
the root FileDescriptorProto to the place where the definition appears.
For example, this path:
[ 4, 3, 2, 7, 1 ]
refers to:
Expand Down
Loading

0 comments on commit ce049d5

Please sign in to comment.