Integrate from Piper for C++, Java, and Python

CHANGES.txt

@@ -1,3 +1,26 @@
+Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
+
+  C++
+  * MessageDifferencer: fixed bug when using custom ignore with multiple
+    unknown fields
+  * Use init_seg in MSVC to push initialization to an earlier phase.
+  * Runtime no longer triggers -Wsign-compare warnings.
+  * Fixed -Wtautological-constant-out-of-range-compare warning.
+  * DynamicCastToGenerated works for nullptr input for even if RTTI is disabled
+  * Arena is refactored and optimized.
+  * Change the signature of Any::PackFrom() to return false on error.
+
+  Java
+  * Avoid possible UnsupportedOperationException when using CodedInputSteam
+    with a direct ByteBuffer.
+  * Make Durations.comparator() and Timestamps.comparator() Serializable.
+  * Add more detailed error information for dynamic message field type
+    validation failure
+
+  Python
+  * Provided an override for the reverse() method that will reverse the internal
+    collection directly instead of using the other methods of the BaseContainer.
+
 2020-11-11 version 3.14.0 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
 
   Protocol Compiler

conformance/conformance_cpp.cc

@@ -36,6 +36,7 @@
 #include <google/protobuf/text_format.h>
 #include <google/protobuf/util/json_util.h>
 #include <google/protobuf/util/type_resolver_util.h>
+#include <google/protobuf/stubs/status.h>
 #include "conformance.pb.h"
 #include <google/protobuf/test_messages_proto2.pb.h>
 #include <google/protobuf/test_messages_proto3.pb.h>
@@ -125,9 +126,9 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) {
       options.ignore_unknown_fields =
           (request.test_category() ==
               conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST);
-      Status status = JsonToBinaryString(type_resolver, *type_url,
-                                         request.json_payload(), &proto_binary,
-                                         options);
+      util::Status status =
+          JsonToBinaryString(type_resolver, *type_url, request.json_payload(),
+                             &proto_binary, options);
       if (!status.ok()) {
         response->set_parse_error(string("Parse error: ") +
                                   std::string(status.error_message()));
@@ -179,8 +180,9 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) {
     case conformance::JSON: {
       string proto_binary;
       GOOGLE_CHECK(test_message->SerializeToString(&proto_binary));
-      Status status = BinaryToJsonString(type_resolver, *type_url, proto_binary,
-                                         response->mutable_json_payload());
+      util::Status status =
+          BinaryToJsonString(type_resolver, *type_url, proto_binary,
+                             response->mutable_json_payload());
       if (!status.ok()) {
         response->set_serialize_error(
             string("Failed to serialize JSON output: ") +

java/core/src/main/java/com/google/protobuf/FieldSet.java

@@ -119,7 +119,6 @@ boolean isEmpty() {
   }
 
   /** Make this FieldSet immutable from this point forward. */
-  @SuppressWarnings("unchecked")
   public void makeImmutable() {
     if (isImmutable) {
       return;
@@ -286,11 +285,11 @@ public void setField(final T descriptor, Object value) {
       final List newList = new ArrayList();
       newList.addAll((List) value);
       for (final Object element : newList) {
-        verifyType(descriptor.getLiteType(), element);
+        verifyType(descriptor, element);
       }
       value = newList;
     } else {
-      verifyType(descriptor.getLiteType(), value);
+      verifyType(descriptor, value);
     }
 
     if (value instanceof LazyField) {
@@ -354,7 +353,7 @@ public void setRepeatedField(final T descriptor, final int index, final Object v
       throw new IndexOutOfBoundsException();
     }
 
-    verifyType(descriptor.getLiteType(), value);
+    verifyType(descriptor, value);
     ((List<Object>) list).set(index, value);
   }
 
@@ -369,7 +368,7 @@ public void addRepeatedField(final T descriptor, final Object value) {
           "addRepeatedField() can only be called on repeated fields.");
     }
 
-    verifyType(descriptor.getLiteType(), value);
+    verifyType(descriptor, value);
 
     final Object existingValue = getField(descriptor);
     List<Object> list;
@@ -390,8 +389,8 @@ public void addRepeatedField(final T descriptor, final Object value) {
    *
    * @throws IllegalArgumentException The value is not of the right type.
    */
-  private void verifyType(final WireFormat.FieldType type, final Object value) {
-    if (!isValidType(type, value)) {
+  private void verifyType(final T descriptor, final Object value) {
+    if (!isValidType(descriptor.getLiteType(), value)) {
       // TODO(kenton):  When chaining calls to setField(), it can be hard to
       //   tell from the stack trace which exact call failed, since the whole
       //   chain is considered one line of code.  It would be nice to print
@@ -400,10 +399,16 @@ private void verifyType(final WireFormat.FieldType type, final Object value) {
       //   isn't a big deal, though, since it would only really apply when using
       //   reflection and generally people don't chain reflection setters.
       throw new IllegalArgumentException(
-          "Wrong object type used with protocol message reflection.");
+          String.format(
+              "Wrong object type used with protocol message reflection.\n"
+              + "Field number: %d, field java type: %s, value type: %s\n",
+              descriptor.getNumber(),
+              descriptor.getLiteType().getJavaType(),
+              value.getClass().getName()));
     }
   }
 
+
   private static boolean isValidType(final WireFormat.FieldType type, final Object value) {
     checkNotNull(value);
     switch (type.getJavaType()) {
@@ -1081,12 +1086,12 @@ public void setField(final T descriptor, Object value) {
         final List newList = new ArrayList();
         newList.addAll((List) value);
         for (final Object element : newList) {
-          verifyType(descriptor.getLiteType(), element);
+          verifyType(descriptor, element);
           hasNestedBuilders = hasNestedBuilders || element instanceof MessageLite.Builder;
         }
         value = newList;
       } else {
-        verifyType(descriptor.getLiteType(), value);
+        verifyType(descriptor, value);
       }
 
       if (value instanceof LazyField) {
@@ -1172,7 +1177,7 @@ public void setRepeatedField(final T descriptor, final int index, final Object v
         throw new IndexOutOfBoundsException();
       }
 
-      verifyType(descriptor.getLiteType(), value);
+      verifyType(descriptor, value);
       ((List<Object>) list).set(index, value);
     }
 
@@ -1190,7 +1195,7 @@ public void addRepeatedField(final T descriptor, final Object value) {
 
       hasNestedBuilders = hasNestedBuilders || value instanceof MessageLite.Builder;
 
-      verifyType(descriptor.getLiteType(), value);
+      verifyType(descriptor, value);
 
       final Object existingValue = getField(descriptor);
       List<Object> list;
@@ -1211,15 +1216,20 @@ public void addRepeatedField(final T descriptor, final Object value) {
      *
      * @throws IllegalArgumentException The value is not of the right type.
      */
-    private static void verifyType(final WireFormat.FieldType type, final Object value) {
-      if (!FieldSet.isValidType(type, value)) {
+    private void verifyType(final T descriptor, final Object value) {
+      if (!FieldSet.isValidType(descriptor.getLiteType(), value)) {
         // Builder can accept Message.Builder values even though FieldSet will reject.
-        if (type.getJavaType() == WireFormat.JavaType.MESSAGE
+        if (descriptor.getLiteType().getJavaType() == WireFormat.JavaType.MESSAGE
             && value instanceof MessageLite.Builder) {
           return;
         }
         throw new IllegalArgumentException(
-            "Wrong object type used with protocol message reflection.");
+            String.format(
+                "Wrong object type used with protocol message reflection.\n"
+                + "Field number: %d, field java type: %s, value type: %s\n",
+                descriptor.getNumber(),
+                descriptor.getLiteType().getJavaType(),
+                value.getClass().getName()));
       }
     }
 

java/core/src/main/java/com/google/protobuf/TextFormat.java

@@ -61,6 +61,7 @@ private TextFormat() {}
 
   private static final Logger logger = Logger.getLogger(TextFormat.class.getName());
 
+
   /**
    * Outputs a textual representation of the Protocol Message supplied into the parameter output.
    * (This representation is the new version of the classic "ProtocolPrinter" output from the
@@ -727,9 +728,9 @@ private void printSingleField(
           // Groups must be serialized with their original capitalization.
           generator.print(field.getMessageType().getName());
         } else {
-          generator.print(field.getName());
+            generator.print(field.getName());
+          }
         }
-      }
 
       if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
         generator.print(" {");
@@ -1811,16 +1812,16 @@ private void mergeField(
         extension = target.findExtensionByName(extensionRegistry, name.toString());
 
         if (extension == null) {
-          String message =
-              (tokenizer.getPreviousLine() + 1)
-                  + ":"
-                  + (tokenizer.getPreviousColumn() + 1)
-                  + ":\t"
-                  + type.getFullName()
-                  + ".["
-                  + name
-                  + "]";
-          unknownFields.add(new UnknownField(message, UnknownField.Type.EXTENSION));
+            String message =
+                (tokenizer.getPreviousLine() + 1)
+                    + ":"
+                    + (tokenizer.getPreviousColumn() + 1)
+                    + ":\t"
+                    + type.getFullName()
+                    + ".["
+                    + name
+                    + "]";
+            unknownFields.add(new UnknownField(message, UnknownField.Type.EXTENSION));
         } else {
           if (extension.descriptor.getContainingType() != type) {
             throw tokenizer.parseExceptionPreviousToken(

java/core/src/main/java/com/google/protobuf/UnsafeUtil.java

@@ -388,7 +388,7 @@ private static boolean supportsUnsafeByteBufferOperations() {
       }
 
       if (Android.isOnAndroidDevice()) {
-        return true;
+        return false;
       }
       clazz.getMethod("getByte", long.class);
       clazz.getMethod("putByte", long.class, byte.class);

java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java

@@ -184,17 +184,17 @@ private void assertMapValuesUpdated(TestMap message) {
   }
 
   private void assertMapValuesCleared(TestMapOrBuilder testMapOrBuilder) {
-    assertEquals(0, testMapOrBuilder.getInt32ToInt32Field().size());
+    assertEquals(0, testMapOrBuilder.getInt32ToInt32FieldMap().size());
     assertEquals(0, testMapOrBuilder.getInt32ToInt32FieldCount());
-    assertEquals(0, testMapOrBuilder.getInt32ToStringField().size());
+    assertEquals(0, testMapOrBuilder.getInt32ToStringFieldMap().size());
     assertEquals(0, testMapOrBuilder.getInt32ToStringFieldCount());
-    assertEquals(0, testMapOrBuilder.getInt32ToBytesField().size());
+    assertEquals(0, testMapOrBuilder.getInt32ToBytesFieldMap().size());
     assertEquals(0, testMapOrBuilder.getInt32ToBytesFieldCount());
-    assertEquals(0, testMapOrBuilder.getInt32ToEnumField().size());
+    assertEquals(0, testMapOrBuilder.getInt32ToEnumFieldMap().size());
     assertEquals(0, testMapOrBuilder.getInt32ToEnumFieldCount());
-    assertEquals(0, testMapOrBuilder.getInt32ToMessageField().size());
+    assertEquals(0, testMapOrBuilder.getInt32ToMessageFieldMap().size());
     assertEquals(0, testMapOrBuilder.getInt32ToMessageFieldCount());
-    assertEquals(0, testMapOrBuilder.getStringToInt32Field().size());
+    assertEquals(0, testMapOrBuilder.getStringToInt32FieldMap().size());
     assertEquals(0, testMapOrBuilder.getStringToInt32FieldCount());
   }
 
@@ -226,13 +226,13 @@ public void testGetMapIsImmutable() {
   }
 
   private void assertMapsAreImmutable(TestMapOrBuilder testMapOrBuilder) {
-    assertImmutable(testMapOrBuilder.getInt32ToInt32Field(), 1, 2);
-    assertImmutable(testMapOrBuilder.getInt32ToStringField(), 1, "2");
-    assertImmutable(testMapOrBuilder.getInt32ToBytesField(), 1, TestUtil.toBytes("2"));
-    assertImmutable(testMapOrBuilder.getInt32ToEnumField(), 1, TestMap.EnumValue.FOO);
+    assertImmutable(testMapOrBuilder.getInt32ToInt32FieldMap(), 1, 2);
+    assertImmutable(testMapOrBuilder.getInt32ToStringFieldMap(), 1, "2");
+    assertImmutable(testMapOrBuilder.getInt32ToBytesFieldMap(), 1, TestUtil.toBytes("2"));
+    assertImmutable(testMapOrBuilder.getInt32ToEnumFieldMap(), 1, TestMap.EnumValue.FOO);
     assertImmutable(
-        testMapOrBuilder.getInt32ToMessageField(), 1, MessageValue.getDefaultInstance());
-    assertImmutable(testMapOrBuilder.getStringToInt32Field(), "1", 2);
+        testMapOrBuilder.getInt32ToMessageFieldMap(), 1, MessageValue.getDefaultInstance());
+    assertImmutable(testMapOrBuilder.getStringToInt32FieldMap(), "1", 2);
   }
 
   private <K, V> void assertImmutable(Map<K, V> map, K key, V value) {