From 1f66b1b032dee1608157e221d4f234522e5aae58 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 12 Oct 2021 10:19:10 -0700 Subject: [PATCH 1/6] Sync from Piper @402594188 PROTOBUF_SYNC_PIPER --- docs/third_party.md | 1 + editors/protobuf-mode.el | 2 +- java/lite.md | 2 +- java/util/pom.xml | 1 + src/README.md | 2 +- src/google/protobuf/arenastring.cc | 12 +- .../cpp/cpp_parse_function_generator.cc | 2 + tests.sh | 2 +- toolchain/cc_toolchain_config.bzl | 148 ++++++++++++++++++ toolchain/toolchains.bazelrc | 8 + 10 files changed, 172 insertions(+), 8 deletions(-) create mode 100644 toolchain/cc_toolchain_config.bzl create mode 100644 toolchain/toolchains.bazelrc diff --git a/docs/third_party.md b/docs/third_party.md index ca2ac63fd3f1d..06b31b1d25dae 100644 --- a/docs/third_party.md +++ b/docs/third_party.md @@ -129,6 +129,7 @@ GRPC (http://www.grpc.io/) is Google's RPC implementation for Protocol Buffers. * https://github.com/Yeolar/raster (C++) * https://github.com/jnordberg/wsrpc (JavaScript Node.js/Browser) * https://github.com/ppissias/xsrpcj (Java) +* https://github.com/twitchtv/twirp (Multiple languages) Inactive: diff --git a/editors/protobuf-mode.el b/editors/protobuf-mode.el index 8102771665638..aa31bd0c44566 100644 --- a/editors/protobuf-mode.el +++ b/editors/protobuf-mode.el @@ -68,7 +68,7 @@ (eval-when-compile (and (= emacs-major-version 24) (>= emacs-minor-version 4) - (require 'cl)) + (require 'cl-lib)) (require 'cc-langs) (require 'cc-fonts)) diff --git a/java/lite.md b/java/lite.md index f248ff6c3e82d..603609f6763b8 100644 --- a/java/lite.md +++ b/java/lite.md @@ -30,7 +30,7 @@ protobuf Java runtime. If you are using Maven, use the following: com.google.protobuf protobuf-javalite - 3.9.1 + 3.18.1 ``` diff --git a/java/util/pom.xml b/java/util/pom.xml index 06f6c1b737ee8..31ccf5d567a1c 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -49,6 +49,7 @@ org.easymock easymock + test com.google.truth diff --git a/src/README.md b/src/README.md index 51d9e2fe38c86..9db40fdde4672 100644 --- a/src/README.md +++ b/src/README.md @@ -19,7 +19,7 @@ To build protobuf from source, the following tools are needed: On Ubuntu/Debian, you can install them with: - $ sudo apt-get install autoconf automake libtool curl make g++ unzip + sudo apt-get install autoconf automake libtool curl make g++ unzip On other platforms, please use the corresponding package managing tool to install them before proceeding. diff --git a/src/google/protobuf/arenastring.cc b/src/google/protobuf/arenastring.cc index 7608b13c7a4c4..169f52729d616 100644 --- a/src/google/protobuf/arenastring.cc +++ b/src/google/protobuf/arenastring.cc @@ -256,6 +256,12 @@ void ArenaStringPtr::ClearToDefault(const LazyString& default_value, } } +inline void SetStrWithHeapBuffer(std::string* str, ArenaStringPtr* s) { + TaggedPtr res; + res.Set(str); + s->UnsafeSetTaggedPointer(res); +} + const char* EpsCopyInputStream::ReadArenaString(const char* ptr, ArenaStringPtr* s, Arena* arena) { @@ -264,13 +270,11 @@ const char* EpsCopyInputStream::ReadArenaString(const char* ptr, int size = ReadSize(&ptr); if (!ptr) return nullptr; - auto str = Arena::Create(arena); + auto* str = Arena::Create(arena); ptr = ReadString(ptr, size, str); GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); - TaggedPtr res; - res.Set(str); - s->UnsafeSetTaggedPointer(res); + SetStrWithHeapBuffer(str, s); return ptr; } diff --git a/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc b/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc index 826133840496f..810f240a89a17 100644 --- a/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/cpp_parse_function_generator.cc @@ -643,6 +643,8 @@ void ParseFunctionGenerator::GenerateArenaString(Formatter& format, ", ~0x$2$u", inlined_string_index / 32, strings::Hex(1u << (inlined_string_index % 32), strings::ZERO_PAD_8)); + } else { + GOOGLE_DCHECK(field->default_value_string().empty()); } format( ");\n" diff --git a/tests.sh b/tests.sh index 06cbc6ce6d2b7..fcfb75f865eba 100755 --- a/tests.sh +++ b/tests.sh @@ -222,7 +222,7 @@ internal_build_java() { cp -r java $dir cd $dir && $MVN clean # Skip tests here - callers will decide what tests they want to run - $MVN install -pl util -Dmaven.test.skip=true + $MVN install -pl core -Dmaven.test.skip=true } build_java() { diff --git a/toolchain/cc_toolchain_config.bzl b/toolchain/cc_toolchain_config.bzl new file mode 100644 index 0000000000000..4478a11416899 --- /dev/null +++ b/toolchain/cc_toolchain_config.bzl @@ -0,0 +1,148 @@ +load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") +load( + "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", + "feature", + "flag_group", + "flag_set", + "tool", + "tool_path", +) + +all_link_actions = [ + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, +] + +all_compile_actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, + ACTION_NAMES.lto_backend, +] + +def _impl(ctx): + tool_paths = [ + tool_path( + name = "gcc", + path = "/usr/local/bin/clang", + ), + tool_path( + name = "ld", + path = "/usr/bin/ld", + ), + tool_path( + name = "ar", + path = "/usr/bin/ar", + ), + tool_path( + name = "compat-ld", + path = "/usr/bin/ld", + ), + tool_path( + name = "cpp", + path = "/bin/false", + ), + tool_path( + name = "dwp", + path = "/bin/false", + ), + tool_path( + name = "gcov", + path = "/bin/false", + ), + tool_path( + name = "nm", + path = "/bin/false", + ), + tool_path( + name = "objcopy", + path = "/bin/false", + ), + tool_path( + name = "objdump", + path = "/bin/false", + ), + tool_path( + name = "strip", + path = "/bin/false", + ), + ] + + linker_flags = feature( + name = "default_linker_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = [ + "-lstdc++", + "--target=" + ctx.attr.target_full_name, + ], + ), + ], + ), + ], + ) + compiler_flags = feature( + name = "default_compile_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = all_compile_actions, + flag_groups = [ + flag_group( + flags = [ + ctx.attr.bit_flag, + "-Wall", + "-no-canonical-prefixes", + "--target=" + ctx.attr.target_full_name, + "-isystem", + ctx.attr.toolchain_dir, + ctx.attr.include_flag, + ], + ), + ], + ), + ], + ) + + return cc_common.create_cc_toolchain_config_info( + abi_libc_version = ctx.attr.target_cpu, + abi_version = ctx.attr.target_cpu, + ctx = ctx, + compiler = "clang", + cxx_builtin_include_directories = [ + ctx.attr.toolchain_dir, + "/usr/include", + "/usr/local/lib/clang", + ], + features = [linker_flags, compiler_flags], + host_system_name = "local", + target_cpu = ctx.attr.target_cpu, + target_libc = ctx.attr.target_cpu, + target_system_name = ctx.attr.target_full_name, + toolchain_identifier = ctx.attr.toolchain_name, + tool_paths = tool_paths, + ) + +cc_toolchain_config = rule( + implementation = _impl, + attrs = { + "bit_flag": attr.string(mandatory = True, values = ["-m32", "-m64"]), + "include_flag": attr.string(mandatory = False), + "target_cpu": attr.string(mandatory = True, values = ["aarch64", "ppc64", "systemz", "x86_32", "x86_64"]), + "target_full_name": attr.string(mandatory = True), + "toolchain_dir": attr.string(mandatory = True), + "toolchain_name": attr.string(mandatory = True), + }, + provides = [CcToolchainConfigInfo], +) diff --git a/toolchain/toolchains.bazelrc b/toolchain/toolchains.bazelrc new file mode 100644 index 0000000000000..f56e4cee3be91 --- /dev/null +++ b/toolchain/toolchains.bazelrc @@ -0,0 +1,8 @@ +build:cross_config --crosstool_top=//toolchain:clang_suite +build:cross_config --host_crosstool_top=@bazel_tools//tools/cpp:toolchain + +build:linux-aarch_64 --config=cross_config --cpu=linux-aarch_64 +build:linux-ppcle_64 --config=cross_config --cpu=linux-ppcle_64 +build:linux-s390_64 --config=cross_config --cpu=linux-s390_64 +build:linux-x86_32 --config=cross_config --cpu=linux-x86_32 +build:linux-x86_64 --config=cross_config --cpu=linux-x86_64 From 99612d0885f2fe76400803ccdb425af46f29f481 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 12 Oct 2021 10:21:22 -0700 Subject: [PATCH 2/6] Update CHANGES.txt --- CHANGES.txt | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 7682fbdd19707..66f1bc0f4c5f4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,19 @@ Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) - Protocol Compiler + + Python + * Proto2 DecodeError now includes message name in error message + + C++ * Make proto2::Message::DiscardUnknownFields() non-virtual + * Separate RepeatedPtrField into its own header file + * For default floating point values of 0, consider all bits significant + + Java + * For default floating point values of 0, consider all bits significant + * Annotate `//java/com/google/protobuf/util/...` with nullness annotations + + Kotlin + * Switch Kotlin proto DSLs to be implemented with inline value classes 2021-10-04 version 3.18.1 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) From 68c17dcde81a7dfe2633eef7e94a3916f851765f Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 12 Oct 2021 11:12:39 -0700 Subject: [PATCH 3/6] Tweak syntax of casting function to void GCC 4.9 seems to be unable to handle (void) syntax with a function, but it is OK with a static_cast to void. --- src/google/protobuf/repeated_ptr_field.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/protobuf/repeated_ptr_field.h b/src/google/protobuf/repeated_ptr_field.h index c96b17517c447..7d635e3bb50f5 100644 --- a/src/google/protobuf/repeated_ptr_field.h +++ b/src/google/protobuf/repeated_ptr_field.h @@ -1631,7 +1631,7 @@ class RepeatedPtrIterator { : it_(other.it_) { // Force a compiler error if the other type is not convertible to ours. if (false) { - (void)[](OtherElement* from) -> Element* { return from; }; + static_cast([](OtherElement* from) -> Element* { return from; }); } } From 7f79a416ff4950ed255423812e29004e9d1db182 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 12 Oct 2021 11:49:50 -0700 Subject: [PATCH 4/6] Update the file lists with ./update_file_lists.sh --- BUILD | 1 + cmake/extract_includes.bat.in | 1 + 2 files changed, 2 insertions(+) diff --git a/BUILD b/BUILD index 9fbf69dfca963..a9ed429f10ac4 100644 --- a/BUILD +++ b/BUILD @@ -147,6 +147,7 @@ cc_library( "src/google/protobuf/message_lite.cc", "src/google/protobuf/parse_context.cc", "src/google/protobuf/repeated_field.cc", + "src/google/protobuf/repeated_ptr_field.cc", "src/google/protobuf/stubs/bytestream.cc", "src/google/protobuf/stubs/common.cc", "src/google/protobuf/stubs/int128.cc", diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in index dd96c6ae31967..605c5f966ba30 100644 --- a/cmake/extract_includes.bat.in +++ b/cmake/extract_includes.bat.in @@ -93,6 +93,7 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\port_undef.inc" inclu copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\reflection.h" include\google\protobuf\reflection.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\reflection_ops.h" include\google\protobuf\reflection_ops.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_field.h" include\google\protobuf\repeated_field.h +copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_ptr_field.h" include\google\protobuf\repeated_ptr_field.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\service.h" include\google\protobuf\service.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\source_context.pb.h" include\google\protobuf\source_context.pb.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\struct.pb.h" include\google\protobuf\struct.pb.h From 454f0cccaa22e3c4958e25b76cbd25a677294cc1 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 12 Oct 2021 15:37:40 -0700 Subject: [PATCH 5/6] Add jsr305 dependency for Bazel --- WORKSPACE | 9 ++++++++- java/util/BUILD | 1 + maven_install.json | 22 +++------------------- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 2174cabf8c8fa..cc2120ec939b5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -38,13 +38,15 @@ bind( load("@rules_jvm_external//:defs.bzl", "maven_install") maven_install( artifacts = [ + "com.google.code.findbugs:jsr305:3.0.2", "com.google.code.gson:gson:2.8.6", "com.google.errorprone:error_prone_annotations:2.3.2", - "com.google.j2objc:j2obj_annotations:1.3", + "com.google.j2objc:j2objc-annotations:1.3", "com.google.guava:guava:30.1.1-jre", "com.google.truth:truth:1.1.2", "junit:junit:4.12", "org.easymock:easymock:3.2", + ], repositories = [ "https://repo1.maven.org/maven2", @@ -78,6 +80,11 @@ bind( actual = "@maven//:com_google_j2objc_j2objc_annotations", ) +bind( + name = "jsr305", + actual = "@maven//:com_google_code_findbugs_jsr305", +) + bind( name = "junit", actual = "@maven//:junit_junit", diff --git a/java/util/BUILD b/java/util/BUILD index 02e55496897f0..3855da96f01d0 100644 --- a/java/util/BUILD +++ b/java/util/BUILD @@ -14,6 +14,7 @@ java_library( "//external:error_prone_annotations", "//external:j2objc_annotations", "//external:gson", + "//external:jsr305", "//external:guava", "//java/core", "//java/lite", diff --git a/maven_install.json b/maven_install.json index f9342185c4e4e..6168aa4af246e 100644 --- a/maven_install.json +++ b/maven_install.json @@ -1,6 +1,8 @@ { "dependency_tree": { - "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": 1033791982, + "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL", + "__INPUT_ARTIFACTS_HASH": 1907885757, + "__RESOLVED_ARTIFACTS_HASH": 375457873, "conflict_resolution": { "com.google.errorprone:error_prone_annotations:2.3.2": "com.google.errorprone:error_prone_annotations:2.5.1", "junit:junit:4.12": "junit:junit:4.13.1" @@ -201,24 +203,6 @@ "sha256": "b3dd1cf5019f942d8cc2afad0aa6aef4b21532446fe90a6b68d567e3389763dd", "url": "https://repo1.maven.org/maven2/org/easymock/easymock/3.2/easymock-3.2.jar" }, - { - "coord": "org.easymock:easymockclassextension:3.2", - "dependencies": [ - "org.easymock:easymock:3.2", - "cglib:cglib-nodep:2.2.2", - "org.objenesis:objenesis:1.3" - ], - "directDependencies": [ - "org.easymock:easymock:3.2" - ], - "file": "v1/https/repo1.maven.org/maven2/org/easymock/easymockclassextension/3.2/easymockclassextension-3.2.jar", - "mirror_urls": [ - "https://repo1.maven.org/maven2/org/easymock/easymockclassextension/3.2/easymockclassextension-3.2.jar", - "https://repo.maven.apache.org/maven2/org/easymock/easymockclassextension/3.2/easymockclassextension-3.2.jar" - ], - "sha256": "e2aeb3ecec87d859b2f3072985d4b15873558bcf6410f422db0c0c5194c76c87", - "url": "https://repo1.maven.org/maven2/org/easymock/easymockclassextension/3.2/easymockclassextension-3.2.jar" - }, { "coord": "org.hamcrest:hamcrest-core:1.3", "dependencies": [], From 9aa1adc60c52c8682c21ad1fc98afa1d3b805563 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 12 Oct 2021 15:43:34 -0700 Subject: [PATCH 6/6] Removed unused references to easymock_classextension --- WORKSPACE | 5 ----- java/core/BUILD | 1 - 2 files changed, 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index cc2120ec939b5..c88d242db2bea 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -95,11 +95,6 @@ bind( actual = "@maven//:org_easymock_easymock", ) -bind( - name = "easymock_classextension", - actual = "@maven//:org_easymock_easymockclassextension", -) - bind( name = "truth", actual = "@maven//:com_google_truth_truth", diff --git a/java/core/BUILD b/java/core/BUILD index 42124bb4f011e..c65f10a4e1a5c 100644 --- a/java/core/BUILD +++ b/java/core/BUILD @@ -255,7 +255,6 @@ junit_tests( ":java_test_protos_java_proto", ":test_util", "//external:easymock", - "//external:easymock_classextension", "//external:guava", "//external:junit", "//external:truth",