Sync from Piper @343521567

PROTOBUF_SYNC_PIPER

BUILD

@@ -164,6 +164,7 @@ cc_library(
         # AUTOGEN(protobuf_lite_srcs)
         "src/google/protobuf/any_lite.cc",
         "src/google/protobuf/arena.cc",
+        "src/google/protobuf/arenastring.cc",
         "src/google/protobuf/extension_set.cc",
         "src/google/protobuf/generated_enum_util.cc",
         "src/google/protobuf/generated_message_table_driven_lite.cc",

CHANGES.txt

@@ -1,12 +1,15 @@
-Unreleased Changes
+2020-11-11 version 3.14.0 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
 
   Protocol Compiler
   * The proto compiler no longer requires a .proto filename when it is not
     generating code.
+  * Added flag `--deterministic_output` to `protoc --encode=...`.
+  * Fixed deadlock when using google.protobuf.Any embedded in aggregate options.
 
   C++
   * Arenas are now unconditionally enabled. cc_enable_arenas no longer has
     any effect.
+  * Removed inlined string support, which is incompatible with arenas.
   * Fix a memory corruption bug in reflection when mixing optional and
     non-optional fields.
   * Make SpaceUsed() calculation more thorough for map fields.
@@ -20,18 +23,55 @@ Unreleased Changes
   * When running under ASAN, skip a test that makes huge allocations.
   * Fixed a crash that could happen when creating more than 256 extensions in
     a single message.
+  * Fix a crash in BuildFile when passing in invalid descriptor proto.
+  * Parser security fix when operating with CodedInputStream.
+  * Warn against the use of AllowUnknownExtension.
+  * Migrated to C++11 for-range loops instead of index-based loops where
+    possible. This fixes a lot of warnings when compiling with -Wsign-compare.
+  * Fix segment fault for proto3 optional (#7805)
+  * Adds a CMake option to build `libprotoc` separately (#7949)
 
   Java
   * Bugfix in mergeFrom() when a oneof has multiple message fields.
+  * Fix RopeByteString.RopeInputStream.read() returning -1 when told to read
+    0 bytes when not at EOF.
+  * Redefine remove(Object) on primitive repeated field Lists to avoid
+    autoboxing.
+  * Support "\u" escapes in textformat string literals.
+  * Trailing empty spaces are no longer ignored for FieldMask.
+  * Fix FieldMaskUtil.subtract to recursively remove mask.
+  * Mark enums with `@java.lang.Deprecated` if the proto enum has option
+    `deprecated = true;`.
+  * Adding forgotten duration.proto to the lite library (#7738)
 
   Python
   * Print google.protobuf.NullValue as null instead of "NULL_VALUE" when it is
     used outside WKT Value/Struct.
   * Fix bug occurring when attempting to deep copy an enum type in python 3.
+  * Add a setuptools extension for generating Python protobufs (#7783)
+  * Remove uses of pkg_resources in non-namespace packages. (#7902)
+  * [bazel/py] Omit google/__init__.py from the Protobuf runtime. (#7908)
+  * Removed the unnecessary setuptools package dependency for Python package (#7511)
+  * Fix PyUnknownFields memory leak (#7928)
+
+  PHP
+  * Added support for "==" to the PHP C extension (#7883)
+  * Added `==` operators for Map and Array. (#7900)
+  * Native C well-known types (#7944)
+  * Optimized away hex2bin() call in generated code (#8006)
+  * New version of upb, and a new hash function wyhash in third_party. (#8000)
+  * add missing hasOneof method to check presence of oneof fields (#8003)
 
   Go:
   * Update go_package options to reference google.golang.org/protobuf module.
 
+  C#:
+  * annotate ByteString.CopyFrom(ReadOnlySpan<byte>) as SecuritySafeCritical (#7701)
+  * Fix C# optional field reflection when there are regular fields too (#7705)
+  * Fix parsing negative Int32Value that crosses segment boundary (#8035)
+
+  Javascript:
+  * JS: parse (un)packed fields conditionally (#7379)
 
 2020-07-14 version 3.13.0 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
 

Protobuf.podspec

@@ -5,7 +5,7 @@
 # dependent projects use the :git notation to refer to the library.
 Pod::Spec.new do |s|
   s.name     = 'Protobuf'
-  s.version  = '3.13.0'
+  s.version  = '3.14.0'
   s.summary  = 'Protocol Buffers v.3 runtime library for Objective-C.'
   s.homepage = 'https://github.com/protocolbuffers/protobuf'
   s.license  = '3-Clause BSD License'
@@ -34,7 +34,7 @@ Pod::Spec.new do |s|
   s.user_target_xcconfig = { 'GCC_PREPROCESSOR_DEFINITIONS' => '$(inherited) GPB_USE_PROTOBUF_FRAMEWORK_IMPORTS=1' }
   s.pod_target_xcconfig = { 'GCC_PREPROCESSOR_DEFINITIONS' => '$(inherited) GPB_USE_PROTOBUF_FRAMEWORK_IMPORTS=1' }
 
-  s.ios.deployment_target = '7.0'
+  s.ios.deployment_target = '9.0'
   s.osx.deployment_target = '10.9'
   s.tvos.deployment_target = '9.0'
   s.watchos.deployment_target = '2.0'

cmake/CMakeLists.txt

@@ -44,6 +44,7 @@ option(protobuf_BUILD_TESTS "Build tests" ON)
 option(protobuf_BUILD_CONFORMANCE "Build conformance tests" OFF)
 option(protobuf_BUILD_EXAMPLES "Build examples" OFF)
 option(protobuf_BUILD_PROTOC_BINARIES "Build libprotoc and protoc compiler" ON)
+option(protobuf_BUILD_LIBPROTOC "Build libprotoc" OFF)
 if (BUILD_SHARED_LIBS)
   set(protobuf_BUILD_SHARED_LIBS_DEFAULT ON)
 else (BUILD_SHARED_LIBS)
@@ -64,8 +65,6 @@ include(protobuf-options.cmake)
 # Overrides for option dependencies
 if (protobuf_BUILD_PROTOC_BINARIES OR protobuf_BUILD_TESTS)
   set(protobuf_BUILD_LIBPROTOC ON)
-else()
-  set(protobuf_BUILD_LIBPROTOC OFF)
 endif ()
 # Path to main configure script
 set(protobuf_CONFIGURE_SCRIPT "../configure.ac")

cmake/extract_includes.bat.in

@@ -87,7 +87,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\bytestream.h" i
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\callback.h" include\google\protobuf\stubs\callback.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\casts.h" include\google\protobuf\stubs\casts.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\common.h" include\google\protobuf\stubs\common.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\fastmem.h" include\google\protobuf\stubs\fastmem.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\hash.h" include\google\protobuf\stubs\hash.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\logging.h" include\google\protobuf\stubs\logging.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\macros.h" include\google\protobuf\stubs\macros.h

cmake/install.cmake

@@ -6,9 +6,9 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/protobuf-lite.pc.cmake
                ${CMAKE_CURRENT_BINARY_DIR}/protobuf-lite.pc @ONLY)
 
 set(_protobuf_libraries libprotobuf-lite libprotobuf)
-if (protobuf_BUILD_PROTOC_BINARIES)
+if (protobuf_BUILD_LIBPROTOC)
     list(APPEND _protobuf_libraries libprotoc)
-endif (protobuf_BUILD_PROTOC_BINARIES)
+endif (protobuf_BUILD_LIBPROTOC)
 
 foreach(_library ${_protobuf_libraries})
   set_property(TARGET ${_library}

cmake/libprotobuf-lite.cmake

@@ -69,6 +69,9 @@ target_link_libraries(libprotobuf-lite ${CMAKE_THREAD_LIBS_INIT})
 if(protobuf_LINK_LIBATOMIC)
   target_link_libraries(libprotobuf-lite atomic)
 endif()
+if(${CMAKE_SYSTEM_NAME} STREQUAL "Android")
+	target_link_libraries(libprotobuf-lite log)
+endif()
 target_include_directories(libprotobuf-lite PUBLIC ${protobuf_source_dir}/src)
 if(MSVC AND protobuf_BUILD_SHARED_LIBS)
   target_compile_definitions(libprotobuf-lite

cmake/libprotobuf.cmake

@@ -2,7 +2,6 @@ set(libprotobuf_files
   ${protobuf_source_dir}/src/google/protobuf/any.cc
   ${protobuf_source_dir}/src/google/protobuf/any.pb.cc
   ${protobuf_source_dir}/src/google/protobuf/api.pb.cc
-  ${protobuf_source_dir}/src/google/protobuf/arenastring.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/importer.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/parser.cc
   ${protobuf_source_dir}/src/google/protobuf/descriptor.cc
@@ -122,6 +121,9 @@ endif()
 if(protobuf_LINK_LIBATOMIC)
   target_link_libraries(libprotobuf atomic)
 endif()
+if(${CMAKE_SYSTEM_NAME} STREQUAL "Android")
+	target_link_libraries(libprotobuf log)
+endif()
 target_include_directories(libprotobuf PUBLIC ${protobuf_source_dir}/src)
 if(MSVC AND protobuf_BUILD_SHARED_LIBS)
   target_compile_definitions(libprotobuf

cmake/protobuf-module.cmake.in

@@ -97,6 +97,10 @@ function(_protobuf_find_libraries name filename)
   else()
     get_target_property(${name}_LIBRARY_RELEASE protobuf::lib${filename}
       LOCATION_RELEASE)
+    get_target_property(${name}_LIBRARY_RELWITHDEBINFO protobuf::lib${filename}
+      LOCATION_RELWITHDEBINFO)
+    get_target_property(${name}_LIBRARY_MINSIZEREL protobuf::lib${filename}
+      LOCATION_MINSIZEREL)
     get_target_property(${name}_LIBRARY_DEBUG protobuf::lib${filename}
       LOCATION_DEBUG)
 
@@ -146,6 +150,14 @@ get_target_property(Protobuf_INCLUDE_DIRS protobuf::libprotobuf
 # Set the protoc Executable
 get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc
   IMPORTED_LOCATION_RELEASE)
+if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
+  get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc
+    IMPORTED_LOCATION_RELWITHDEBINFO)
+endif()
+if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
+  get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc
+    IMPORTED_LOCATION_MINSIZEREL)
+endif()
 if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
   get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc
     IMPORTED_LOCATION_DEBUG)

conformance/Makefile.am

@@ -333,7 +333,7 @@ conformance-php-c:
 
 # Targets for actually running tests.
 test_cpp: protoc_middleman conformance-test-runner conformance-cpp
-	./conformance-test-runner --enforce_recommended --failure_list failure_list_cpp.txt ./conformance-cpp
+	./conformance-test-runner --enforce_recommended --failure_list failure_list_cpp.txt --text_format_failure_list text_format_failure_list_cpp.txt ./conformance-cpp
 
 test_java: protoc_middleman conformance-test-runner conformance-java
 	./conformance-test-runner --enforce_recommended --failure_list failure_list_java.txt --text_format_failure_list text_format_failure_list_java.txt ./conformance-java
@@ -359,10 +359,24 @@ test_php_c_32: protoc_middleman conformance-test-runner conformance-php-c $(othe
 # These depend on library paths being properly set up.  The easiest way to
 # run them is to just use "tox" from the python dir.
 test_python: protoc_middleman conformance-test-runner
-	./conformance-test-runner --enforce_recommended --failure_list failure_list_python.txt --text_format_failure_list text_format_failure_list_python.txt ./conformance_python.py
+	VERSION="$(shell python --version 2>&1)"; \
+	if [[ "$$VERSION"  == "Python 2.7"* ]]; then \
+	  echo "Using Python 2.7 failure list."; \
+	  ./conformance-test-runner --enforce_recommended --failure_list failure_list_python.txt --text_format_failure_list text_format_failure_list_python_2.7.txt ./conformance_python.py; \
+	else \
+	  echo "Using Python >2.7 failure list."; \
+	  ./conformance-test-runner --enforce_recommended --failure_list failure_list_python.txt --text_format_failure_list text_format_failure_list_python.txt ./conformance_python.py; \
+	fi
 
 test_python_cpp: protoc_middleman conformance-test-runner
-	./conformance-test-runner --enforce_recommended --failure_list failure_list_python_cpp.txt ./conformance_python.py
+	VERSION="$(shell python --version 2>&1)"; \
+	if [[ "$$VERSION"  == "Python 2.7"* ]]; then \
+	  echo "Using Python 2.7 failure list."; \
+	  ./conformance-test-runner --enforce_recommended --failure_list failure_list_python_cpp.txt --text_format_failure_list text_format_failure_list_python_cpp_2.7.txt ./conformance_python.py; \
+	else \
+	  echo "Using Python >2.7 failure list."; \
+	  ./conformance-test-runner --enforce_recommended --failure_list failure_list_python_cpp.txt --text_format_failure_list text_format_failure_list_python_cpp.txt ./conformance_python.py; \
+	fi
 
 test_nodejs: protoc_middleman conformance-test-runner $(other_language_protoc_outputs)
 	NODE_PATH=../js:. ./conformance-test-runner --enforce_recommended --failure_list failure_list_js.txt ./conformance_nodejs.js

conformance/conformance_cpp.cc

@@ -36,6 +36,7 @@
 #include <google/protobuf/text_format.h>
 #include <google/protobuf/util/json_util.h>
 #include <google/protobuf/util/type_resolver_util.h>
+#include <google/protobuf/stubs/status.h>
 #include "conformance.pb.h"
 #include <google/protobuf/test_messages_proto2.pb.h>
 #include <google/protobuf/test_messages_proto3.pb.h>
@@ -125,9 +126,9 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) {
       options.ignore_unknown_fields =
           (request.test_category() ==
               conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST);
-      Status status = JsonToBinaryString(type_resolver, *type_url,
-                                         request.json_payload(), &proto_binary,
-                                         options);
+      util::Status status =
+          JsonToBinaryString(type_resolver, *type_url, request.json_payload(),
+                             &proto_binary, options);
       if (!status.ok()) {
         response->set_parse_error(string("Parse error: ") +
                                   std::string(status.error_message()));
@@ -179,8 +180,9 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) {
     case conformance::JSON: {
       string proto_binary;
       GOOGLE_CHECK(test_message->SerializeToString(&proto_binary));
-      Status status = BinaryToJsonString(type_resolver, *type_url, proto_binary,
-                                         response->mutable_json_payload());
+      util::Status status =
+          BinaryToJsonString(type_resolver, *type_url, proto_binary,
+                             response->mutable_json_payload());
       if (!status.ok()) {
         response->set_serialize_error(
             string("Failed to serialize JSON output: ") +

conformance/failure_list_php_c.txt

@@ -1,4 +1,2 @@
 Recommended.Proto2.JsonInput.FieldNameExtension.Validator
-Recommended.Proto3.JsonInput.NullValueInOtherOneofNewFormat.Validator
-Recommended.Proto3.JsonInput.NullValueInOtherOneofOldFormat.Validator
 Required.Proto2.JsonInput.StoresDefaultPrimitive.Validator

conformance/text_format_conformance_suite.cc

@@ -259,8 +259,7 @@ void TextFormatConformanceTestSuite::RunSuiteImpl() {
                          "optional_float: 18446744073709551616");
 
   // String literals x {Strings, Bytes}
-  for (const auto& field_type :
-       std::vector<const std::string>{"String", "Bytes"}) {
+  for (const auto& field_type : std::vector<std::string>{"String", "Bytes"}) {
     const std::string field_name =
         field_type == "String" ? "optional_string" : "optional_bytes";
     RunValidTextFormatTest(

conformance/text_format_failure_list_python_2.7.txt

@@ -0,0 +1,36 @@
+# This is the list of text format conformance tests that are known to fail right
+# now.
+# TODO: These should be fixed.
+Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput
+Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput
+
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput
+Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes
+Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString

conformance/text_format_failure_list_python_cpp_2.7.txt

@@ -0,0 +1,30 @@
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput
+Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes
+Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString