Merge pull request #9090 from acozzette/sync-stage

Integrate from Piper for C++, Java, and Python

BUILD

@@ -147,6 +147,7 @@ cc_library(
         "src/google/protobuf/message_lite.cc",
         "src/google/protobuf/parse_context.cc",
         "src/google/protobuf/repeated_field.cc",
+        "src/google/protobuf/repeated_ptr_field.cc",
         "src/google/protobuf/stubs/bytestream.cc",
         "src/google/protobuf/stubs/common.cc",
         "src/google/protobuf/stubs/int128.cc",

CHANGES.txt

@@ -1,6 +1,19 @@
 Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
-  Protocol Compiler
+
+  Python
+  * Proto2 DecodeError now includes message name in error message
+
+  C++
   * Make proto2::Message::DiscardUnknownFields() non-virtual
+  * Separate RepeatedPtrField into its own header file
+  * For default floating point values of 0, consider all bits significant
+
+  Java
+  * For default floating point values of 0, consider all bits significant
+  * Annotate `//java/com/google/protobuf/util/...` with nullness annotations
+
+  Kotlin
+  * Switch Kotlin proto DSLs to be implemented with inline value classes
 
 2021-10-04 version 3.18.1 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
 

WORKSPACE

@@ -38,13 +38,15 @@ bind(
 load("@rules_jvm_external//:defs.bzl", "maven_install")
 maven_install(
     artifacts = [
+        "com.google.code.findbugs:jsr305:3.0.2",
         "com.google.code.gson:gson:2.8.6",
         "com.google.errorprone:error_prone_annotations:2.3.2",
-        "com.google.j2objc:j2obj_annotations:1.3",
+        "com.google.j2objc:j2objc-annotations:1.3",
         "com.google.guava:guava:30.1.1-jre",
         "com.google.truth:truth:1.1.2",
         "junit:junit:4.12",
         "org.easymock:easymock:3.2",
+
     ],
     repositories = [
         "https://repo1.maven.org/maven2",
@@ -78,6 +80,11 @@ bind(
     actual = "@maven//:com_google_j2objc_j2objc_annotations",
 )
 
+bind(
+    name = "jsr305",
+    actual = "@maven//:com_google_code_findbugs_jsr305",
+)
+
 bind(
     name = "junit",
     actual = "@maven//:junit_junit",
@@ -88,11 +95,6 @@ bind(
     actual = "@maven//:org_easymock_easymock",
 )
 
-bind(
-    name = "easymock_classextension",
-    actual = "@maven//:org_easymock_easymockclassextension",
-)
-
 bind(
     name = "truth",
     actual = "@maven//:com_google_truth_truth",

cmake/extract_includes.bat.in

@@ -93,6 +93,7 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\port_undef.inc" inclu
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\reflection.h" include\google\protobuf\reflection.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\reflection_ops.h" include\google\protobuf\reflection_ops.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_field.h" include\google\protobuf\repeated_field.h
+copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\repeated_ptr_field.h" include\google\protobuf\repeated_ptr_field.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\service.h" include\google\protobuf\service.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\source_context.pb.h" include\google\protobuf\source_context.pb.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\struct.pb.h" include\google\protobuf\struct.pb.h

cmake/libprotobuf-lite.cmake

@@ -19,6 +19,7 @@ set(libprotobuf_lite_files
   ${protobuf_source_dir}/src/google/protobuf/message_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/parse_context.cc
   ${protobuf_source_dir}/src/google/protobuf/repeated_field.cc
+  ${protobuf_source_dir}/src/google/protobuf/repeated_ptr_field.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/bytestream.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/common.cc
   ${protobuf_source_dir}/src/google/protobuf/stubs/int128.cc
@@ -65,6 +66,7 @@ set(libprotobuf_lite_includes
   ${protobuf_source_dir}/src/google/protobuf/parse_context.h
   ${protobuf_source_dir}/src/google/protobuf/port.h
   ${protobuf_source_dir}/src/google/protobuf/repeated_field.h
+  ${protobuf_source_dir}/src/google/protobuf/repeated_ptr_field.h
   ${protobuf_source_dir}/src/google/protobuf/stubs/bytestream.h
   ${protobuf_source_dir}/src/google/protobuf/stubs/callback.h
   ${protobuf_source_dir}/src/google/protobuf/stubs/casts.h

java/core/BUILD

@@ -255,7 +255,6 @@ junit_tests(
         ":java_test_protos_java_proto",
         ":test_util",
         "//external:easymock",
-        "//external:easymock_classextension",
         "//external:guava",
         "//external:junit",
         "//external:truth",

java/core/src/main/java/com/google/protobuf/Descriptors.java

@@ -49,6 +49,7 @@
 import com.google.protobuf.DescriptorProtos.ServiceDescriptorProto;
 import com.google.protobuf.DescriptorProtos.ServiceOptions;
 import com.google.protobuf.Descriptors.FileDescriptor.Syntax;
+import java.lang.ref.ReferenceQueue;
 import java.lang.ref.WeakReference;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -59,7 +60,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.WeakHashMap;
 import java.util.logging.Logger;
 
 /**
@@ -1816,6 +1816,15 @@ public EnumValueDescriptor findValueByNumber(final int number) {
           valuesSortedByNumber, distinctNumbers, EnumValueDescriptor.NUMBER_GETTER, number);
     }
 
+    private static class UnknownEnumValueReference extends WeakReference<EnumValueDescriptor> {
+      private final int number;
+
+      private UnknownEnumValueReference(int number, EnumValueDescriptor descriptor) {
+        super(descriptor);
+        this.number = number;
+      }
+    }
+
     /**
      * Get the enum value for a number. If no enum value has this number, construct an
      * EnumValueDescriptor for it.
@@ -1827,43 +1836,28 @@ public EnumValueDescriptor findValueByNumberCreatingIfUnknown(final int number)
       }
       // The number represents an unknown enum value.
       synchronized (this) {
-        // Descriptors are compared by object identity so for the same number
-        // we need to return the same EnumValueDescriptor object. This means
-        // we have to store created EnumValueDescriptors. However, as there
-        // are potentially 2G unknown enum values, storing all of these
-        // objects persistently will consume lots of memory for long-running
-        // services and it's also unnecessary as not many EnumValueDescriptors
-        // will be used at the same time.
-        //
-        // To solve the problem we take advantage of Java's weak references and
-        // rely on gc to release unused descriptors.
-        //
-        // Here is how it works:
-        //   * We store unknown EnumValueDescriptors in a WeakHashMap with the
-        //     value being a weak reference to the descriptor.
-        //   * The descriptor holds a strong reference to the key so as long
-        //     as the EnumValueDescriptor is in use, the key will be there
-        //     and the corresponding map entry will be there. Following-up
-        //     queries with the same number will return the same descriptor.
-        //   * If the user no longer uses an unknown EnumValueDescriptor,
-        //     it will be gc-ed since we only hold a weak reference to it in
-        //     the map. The key in the corresponding map entry will also be
-        //     gc-ed as the only strong reference to it is in the descriptor
-        //     which is just gc-ed. With the key being gone WeakHashMap will
-        //     then remove the whole entry. This way unknown descriptors will
-        //     be freed automatically and we don't need to do anything to
-        //     clean-up unused map entries.
-
-        // Note: We must use "new Integer(number)" here because we don't want
-        // these Integer objects to be cached.
-        Integer key = new Integer(number);
-        WeakReference<EnumValueDescriptor> reference = unknownValues.get(key);
-        if (reference != null) {
-          result = reference.get();
+        if (cleanupQueue == null) {
+          cleanupQueue = new ReferenceQueue<EnumValueDescriptor>();
+          unknownValues = new HashMap<Integer, WeakReference<EnumValueDescriptor>>();
+        } else {
+          while (true) {
+            UnknownEnumValueReference toClean = (UnknownEnumValueReference) cleanupQueue.poll();
+            if (toClean == null) {
+              break;
+            }
+            unknownValues.remove(toClean.number);
+          }
         }
+
+        // There are two ways we can be missing a value: it wasn't in the map, or the reference
+        // has been GC'd.  (It may even have been GC'd since we cleaned up the references a few
+        // lines of code ago.)  So get out the reference, if it's still present...
+        WeakReference<EnumValueDescriptor> reference = unknownValues.get(number);
+        result = (reference == null) ? null : reference.get();
+
         if (result == null) {
-          result = new EnumValueDescriptor(this, key);
-          unknownValues.put(key, new WeakReference<EnumValueDescriptor>(result));
+          result = new EnumValueDescriptor(this, number);
+          unknownValues.put(number, new UnknownEnumValueReference(number, result));
         }
       }
       return result;
@@ -1882,8 +1876,8 @@ int getUnknownEnumValueDescriptorCount() {
     private final EnumValueDescriptor[] values;
     private final EnumValueDescriptor[] valuesSortedByNumber;
     private final int distinctNumbers;
-    private final WeakHashMap<Integer, WeakReference<EnumValueDescriptor>> unknownValues =
-        new WeakHashMap<>();
+    private Map<Integer, WeakReference<EnumValueDescriptor>> unknownValues = null;
+    private ReferenceQueue<EnumValueDescriptor> cleanupQueue = null;
 
     private EnumDescriptor(
         final EnumDescriptorProto proto,

java/core/src/main/java/com/google/protobuf/Internal.java

@@ -54,6 +54,7 @@ public final class Internal {
 
   private Internal() {}
 
+  static final Charset US_ASCII = Charset.forName("US-ASCII");
   static final Charset UTF_8 = Charset.forName("UTF-8");
   static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1");
 

java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java

@@ -134,7 +134,7 @@ static InvalidWireTypeException invalidWireType() {
     return new InvalidWireTypeException("Protocol message tag had invalid wire type.");
   }
 
-  /** Exception indicating that and unexpected wire type was encountered for a field. */
+  /** Exception indicating that an unexpected wire type was encountered for a field. */
   @ExperimentalApi
   public static class InvalidWireTypeException extends InvalidProtocolBufferException {
     private static final long serialVersionUID = 3283890091615336259L;

java/core/src/main/java/com/google/protobuf/MessageLiteToString.java

@@ -187,10 +187,10 @@ private static boolean isDefaultValue(Object o) {
       return ((Integer) o) == 0;
     }
     if (o instanceof Float) {
-      return ((Float) o) == 0f;
+      return Float.floatToRawIntBits((Float) o) == 0;
     }
     if (o instanceof Double) {
-      return ((Double) o) == 0d;
+      return Double.doubleToRawLongBits((Double) o) == 0;
     }
     if (o instanceof String) {
       return o.equals("");

java/core/src/main/java/com/google/protobuf/MessageSchema.java

@@ -5808,9 +5808,9 @@ private boolean isFieldPresent(T message, int pos) {
       final long offset = offset(typeAndOffset);
       switch (type(typeAndOffset)) {
         case 0: // DOUBLE:
-          return UnsafeUtil.getDouble(message, offset) != 0D;
+            return Double.doubleToRawLongBits(UnsafeUtil.getDouble(message, offset)) != 0L;
         case 1: // FLOAT:
-          return UnsafeUtil.getFloat(message, offset) != 0F;
+            return Float.floatToRawIntBits(UnsafeUtil.getFloat(message, offset)) != 0;
         case 2: // INT64:
           return UnsafeUtil.getLong(message, offset) != 0L;
         case 3: // UINT64:

java/core/src/main/java/com/google/protobuf/SchemaUtil.java

@@ -68,13 +68,13 @@ public static void requireGeneratedMessage(Class<?> messageType) {
   }
 
   public static void writeDouble(int fieldNumber, double value, Writer writer) throws IOException {
-    if (Double.compare(value, 0.0) != 0) {
+    if (Double.doubleToRawLongBits(value) != 0) {
       writer.writeDouble(fieldNumber, value);
     }
   }
 
   public static void writeFloat(int fieldNumber, float value, Writer writer) throws IOException {
-    if (Float.compare(value, 0.0f) != 0) {
+    if (Float.floatToRawIntBits(value) != 0) {
       writer.writeFloat(fieldNumber, value);
     }
   }

java/core/src/main/java/com/google/protobuf/TextFormatParseInfoTree.java

@@ -90,7 +90,7 @@ private TextFormatParseInfoTree(
   /**
    * Retrieve all the locations of a field.
    *
-   * @param fieldDescriptor the @{link FieldDescriptor} of the desired field
+   * @param fieldDescriptor the {@link FieldDescriptor} of the desired field
    * @return a list of the locations of values of the field. If there are not values or the field
    *     doesn't exist, an empty list is returned.
    */
@@ -105,7 +105,7 @@ public List<TextFormatParseLocation> getLocations(final FieldDescriptor fieldDes
    * <p>Returns the {@link TextFormatParseLocation} for index-th value of the field in the parsed
    * text.
    *
-   * @param fieldDescriptor the @{link FieldDescriptor} of the desired field
+   * @param fieldDescriptor the {@link FieldDescriptor} of the desired field
    * @param index the index of the value.
    * @return the {@link TextFormatParseLocation} of the value
    * @throws IllegalArgumentException index is out of range
@@ -117,7 +117,7 @@ public TextFormatParseLocation getLocation(final FieldDescriptor fieldDescriptor
   /**
    * Retrieve a list of all the location information trees for a sub message field.
    *
-   * @param fieldDescriptor the @{link FieldDescriptor} of the desired field
+   * @param fieldDescriptor the {@link FieldDescriptor} of the desired field
    * @return A list of {@link TextFormatParseInfoTree}
    */
   public List<TextFormatParseInfoTree> getNestedTrees(final FieldDescriptor fieldDescriptor) {
@@ -128,7 +128,7 @@ public List<TextFormatParseInfoTree> getNestedTrees(final FieldDescriptor fieldD
   /**
    * Returns the parse info tree for the given field, which must be a message type.
    *
-   * @param fieldDescriptor the @{link FieldDescriptor} of the desired sub message
+   * @param fieldDescriptor the {@link FieldDescriptor} of the desired sub message
    * @param index the index of message value.
    * @return the {@code ParseInfoTree} of the message value. {@code null} is returned if the field
    *     doesn't exist or the index is out of range.

java/core/src/main/java/com/google/protobuf/Utf8.java

@@ -1371,29 +1371,39 @@ String decodeUtf8(byte[] bytes, int index, int size) throws InvalidProtocolBuffe
             String.format("buffer length=%d, index=%d, size=%d", bytes.length, index, size));
       }
 
-      int offset = index;
-      final int limit = offset + size;
-
-      // The longest possible resulting String is the same as the number of input bytes, when it is
-      // all ASCII. For other cases, this over-allocates and we will truncate in the end.
-      char[] resultArr = new char[size];
-      int resultPos = 0;
+      int offset = index + unsafeEstimateConsecutiveAscii(bytes, index, size);
+      final int limit = index + size;
 
-      // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this).
-      // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII).
+      // get an "exact" consecutive ASCII
       while (offset < limit) {
         byte b = UnsafeUtil.getByte(bytes, offset);
-        if (!DecodeUtil.isOneByte(b)) {
+        if (b < 0) {
           break;
         }
         offset++;
-        DecodeUtil.handleOneByte(b, resultArr, resultPos++);
+      }
+
+      if (offset == limit) {
+        // The entire byte sequence is ASCII.  Don't bother copying to a char[], JVMs using
+        // compact strings will just turn it back into the same byte[].
+        return new String(bytes, index, size, Internal.US_ASCII);
+      }
+
+      // It's not all ASCII, at this point.  This may over-allocate, but we will truncate in the
+      // end.
+      char[] resultArr = new char[size];
+      int resultPos = 0;
+
+      // Copy over the initial run of ASCII.
+      for (int i = index; i < offset; i++) {
+        DecodeUtil.handleOneByte(UnsafeUtil.getByte(bytes, i), resultArr, resultPos++);
       }
 
       while (offset < limit) {
         byte byte1 = UnsafeUtil.getByte(bytes, offset++);
         if (DecodeUtil.isOneByte(byte1)) {
           DecodeUtil.handleOneByte(byte1, resultArr, resultPos++);
+
           // It's common for there to be multiple ASCII characters in a run mixed in, so add an
           // extra optimized loop to take care of these runs.
           while (offset < limit) {
@@ -1656,7 +1666,17 @@ private static int unsafeEstimateConsecutiveAscii(
         return 0;
       }
 
-      for (int i = 0; i < maxChars; i++) {
+      int i;
+      for (i = 0; i + 8 <= maxChars; i += 8) {
+        if ((UnsafeUtil.getLong(bytes, UnsafeUtil.BYTE_ARRAY_BASE_OFFSET + offset)
+                & ASCII_MASK_LONG)
+            != 0L) {
+          break;
+        }
+        offset += 8;
+      }
+
+      for (; i < maxChars; i++) {
         if (UnsafeUtil.getByte(bytes, offset++) < 0) {
           return i;
         }

java/core/src/test/java/com/google/protobuf/CheckUtf8Test.java

@@ -49,7 +49,7 @@
 @RunWith(JUnit4.class)
 public class CheckUtf8Test {
 
-  private static final String UTF8_BYTE_STRING_TEXT = "some text";
+  private static final String UTF8_BYTE_STRING_TEXT = "some text π \uD83D\uDE00";
   private static final ByteString UTF8_BYTE_STRING = ByteString.copyFromUtf8(UTF8_BYTE_STRING_TEXT);
   private static final ByteString NON_UTF8_BYTE_STRING =
       ByteString.copyFrom(new byte[] {(byte) 0x80}); // A lone continuation byte.