getLocations(final FieldDescriptor fieldDes
* Returns the {@link TextFormatParseLocation} for index-th value of the field in the parsed
* text.
*
- * @param fieldDescriptor the @{link FieldDescriptor} of the desired field
+ * @param fieldDescriptor the {@link FieldDescriptor} of the desired field
* @param index the index of the value.
* @return the {@link TextFormatParseLocation} of the value
* @throws IllegalArgumentException index is out of range
@@ -117,7 +117,7 @@ public TextFormatParseLocation getLocation(final FieldDescriptor fieldDescriptor
/**
* Retrieve a list of all the location information trees for a sub message field.
*
- * @param fieldDescriptor the @{link FieldDescriptor} of the desired field
+ * @param fieldDescriptor the {@link FieldDescriptor} of the desired field
* @return A list of {@link TextFormatParseInfoTree}
*/
public List getNestedTrees(final FieldDescriptor fieldDescriptor) {
@@ -128,7 +128,7 @@ public List getNestedTrees(final FieldDescriptor fieldD
/**
* Returns the parse info tree for the given field, which must be a message type.
*
- * @param fieldDescriptor the @{link FieldDescriptor} of the desired sub message
+ * @param fieldDescriptor the {@link FieldDescriptor} of the desired sub message
* @param index the index of message value.
* @return the {@code ParseInfoTree} of the message value. {@code null} is returned if the field
* doesn't exist or the index is out of range.
diff --git a/java/core/src/main/java/com/google/protobuf/Utf8.java b/java/core/src/main/java/com/google/protobuf/Utf8.java
index 7d56ddd6e640..7c9133e16b32 100644
--- a/java/core/src/main/java/com/google/protobuf/Utf8.java
+++ b/java/core/src/main/java/com/google/protobuf/Utf8.java
@@ -1371,29 +1371,39 @@ String decodeUtf8(byte[] bytes, int index, int size) throws InvalidProtocolBuffe
String.format("buffer length=%d, index=%d, size=%d", bytes.length, index, size));
}
- int offset = index;
- final int limit = offset + size;
-
- // The longest possible resulting String is the same as the number of input bytes, when it is
- // all ASCII. For other cases, this over-allocates and we will truncate in the end.
- char[] resultArr = new char[size];
- int resultPos = 0;
+ int offset = index + unsafeEstimateConsecutiveAscii(bytes, index, size);
+ final int limit = index + size;
- // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this).
- // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII).
+ // get an "exact" consecutive ASCII
while (offset < limit) {
byte b = UnsafeUtil.getByte(bytes, offset);
- if (!DecodeUtil.isOneByte(b)) {
+ if (b < 0) {
break;
}
offset++;
- DecodeUtil.handleOneByte(b, resultArr, resultPos++);
+ }
+
+ if (offset == limit) {
+ // The entire byte sequence is ASCII. Don't bother copying to a char[], JVMs using
+ // compact strings will just turn it back into the same byte[].
+ return new String(bytes, index, size, Internal.US_ASCII);
+ }
+
+ // It's not all ASCII, at this point. This may over-allocate, but we will truncate in the
+ // end.
+ char[] resultArr = new char[size];
+ int resultPos = 0;
+
+ // Copy over the initial run of ASCII.
+ for (int i = index; i < offset; i++) {
+ DecodeUtil.handleOneByte(UnsafeUtil.getByte(bytes, i), resultArr, resultPos++);
}
while (offset < limit) {
byte byte1 = UnsafeUtil.getByte(bytes, offset++);
if (DecodeUtil.isOneByte(byte1)) {
DecodeUtil.handleOneByte(byte1, resultArr, resultPos++);
+
// It's common for there to be multiple ASCII characters in a run mixed in, so add an
// extra optimized loop to take care of these runs.
while (offset < limit) {
@@ -1656,7 +1666,17 @@ private static int unsafeEstimateConsecutiveAscii(
return 0;
}
- for (int i = 0; i < maxChars; i++) {
+ int i;
+ for (i = 0; i + 8 <= maxChars; i += 8) {
+ if ((UnsafeUtil.getLong(bytes, UnsafeUtil.BYTE_ARRAY_BASE_OFFSET + offset)
+ & ASCII_MASK_LONG)
+ != 0L) {
+ break;
+ }
+ offset += 8;
+ }
+
+ for (; i < maxChars; i++) {
if (UnsafeUtil.getByte(bytes, offset++) < 0) {
return i;
}
diff --git a/java/core/src/test/java/com/google/protobuf/CheckUtf8Test.java b/java/core/src/test/java/com/google/protobuf/CheckUtf8Test.java
index 458529c4c318..800623ac741b 100644
--- a/java/core/src/test/java/com/google/protobuf/CheckUtf8Test.java
+++ b/java/core/src/test/java/com/google/protobuf/CheckUtf8Test.java
@@ -49,7 +49,7 @@
@RunWith(JUnit4.class)
public class CheckUtf8Test {
- private static final String UTF8_BYTE_STRING_TEXT = "some text";
+ private static final String UTF8_BYTE_STRING_TEXT = "some text π \uD83D\uDE00";
private static final ByteString UTF8_BYTE_STRING = ByteString.copyFromUtf8(UTF8_BYTE_STRING_TEXT);
private static final ByteString NON_UTF8_BYTE_STRING =
ByteString.copyFrom(new byte[] {(byte) 0x80}); // A lone continuation byte.
diff --git a/java/core/src/test/java/com/google/protobuf/DecodeUtf8Test.java b/java/core/src/test/java/com/google/protobuf/DecodeUtf8Test.java
index 359d4d74c02b..5a345aa173fd 100644
--- a/java/core/src/test/java/com/google/protobuf/DecodeUtf8Test.java
+++ b/java/core/src/test/java/com/google/protobuf/DecodeUtf8Test.java
@@ -1,3 +1,33 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc. All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
package com.google.protobuf;
import com.google.protobuf.Utf8.Processor;
@@ -28,12 +58,13 @@ public void testRoundTripAllValidChars() throws Exception {
public void testOneByte() throws Exception {
int valid = 0;
+ ByteBuffer buffer = ByteBuffer.allocateDirect(1);
for (int i = Byte.MIN_VALUE; i <= Byte.MAX_VALUE; i++) {
- ByteString bs = ByteString.copyFrom(new byte[] { (byte) i });
- if (!bs.isValidUtf8()) {
- assertInvalid(bs.toByteArray());
- } else {
+ ByteString bs = ByteString.copyFrom(new byte[] {(byte) i});
+ if (bs.isValidUtf8()) {
valid++;
+ } else {
+ assertInvalid(bs.toByteArray(), buffer);
}
}
assertEquals(IsValidUtf8TestUtil.EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT, valid);
@@ -41,13 +72,14 @@ public void testOneByte() throws Exception {
public void testTwoBytes() throws Exception {
int valid = 0;
+ ByteBuffer buffer = ByteBuffer.allocateDirect(2);
for (int i = Byte.MIN_VALUE; i <= Byte.MAX_VALUE; i++) {
for (int j = Byte.MIN_VALUE; j <= Byte.MAX_VALUE; j++) {
- ByteString bs = ByteString.copyFrom(new byte[]{(byte) i, (byte) j});
- if (!bs.isValidUtf8()) {
- assertInvalid(bs.toByteArray());
- } else {
+ ByteString bs = ByteString.copyFrom(new byte[] {(byte) i, (byte) j});
+ if (bs.isValidUtf8()) {
valid++;
+ } else {
+ assertInvalid(bs.toByteArray(), buffer);
}
}
}
@@ -55,34 +87,30 @@ public void testTwoBytes() throws Exception {
}
public void testThreeBytes() throws Exception {
- // Travis' OOM killer doesn't like this test
- if (System.getenv("TRAVIS") == null) {
- int count = 0;
- int valid = 0;
- for (int i = Byte.MIN_VALUE; i <= Byte.MAX_VALUE; i++) {
- for (int j = Byte.MIN_VALUE; j <= Byte.MAX_VALUE; j++) {
- for (int k = Byte.MIN_VALUE; k <= Byte.MAX_VALUE; k++) {
- byte[] bytes = new byte[]{(byte) i, (byte) j, (byte) k};
- ByteString bs = ByteString.copyFrom(bytes);
- if (!bs.isValidUtf8()) {
- assertInvalid(bytes);
- } else {
- valid++;
- }
- count++;
- if (count % 1000000L == 0) {
- logger.info("Processed " + (count / 1000000L) + " million characters");
- }
+ int count = 0;
+ int valid = 0;
+ ByteBuffer buffer = ByteBuffer.allocateDirect(3);
+ for (int i = Byte.MIN_VALUE; i <= Byte.MAX_VALUE; i++) {
+ for (int j = Byte.MIN_VALUE; j <= Byte.MAX_VALUE; j++) {
+ for (int k = Byte.MIN_VALUE; k <= Byte.MAX_VALUE; k++) {
+ byte[] bytes = new byte[] {(byte) i, (byte) j, (byte) k};
+ ByteString bs = ByteString.copyFrom(bytes);
+ if (bs.isValidUtf8()) {
+ valid++;
+ } else {
+ assertInvalid(bytes, buffer);
+ }
+ count++;
+ if (count % 1000000L == 0) {
+ logger.info("Processed " + (count / 1000000L) + " million characters");
}
}
}
- assertEquals(IsValidUtf8TestUtil.EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT, valid);
}
+ assertEquals(IsValidUtf8TestUtil.EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT, valid);
}
- /**
- * Tests that round tripping of a sample of four byte permutations work.
- */
+ /** Tests that round tripping of a sample of four byte permutations work. */
public void testInvalid_4BytesSamples() throws Exception {
// Bad trailing bytes
assertInvalid(0xF0, 0xA4, 0xAD, 0x7F);
@@ -99,8 +127,7 @@ public void testRealStrings() throws Exception {
// German
assertRoundTrips("Quizdeltagerne spiste jordb\u00e6r med fl\u00f8de, mens cirkusklovnen");
// Japanese
- assertRoundTrips(
- "\u3044\u308d\u306f\u306b\u307b\u3078\u3068\u3061\u308a\u306c\u308b\u3092");
+ assertRoundTrips("\u3044\u308d\u306f\u306b\u307b\u3078\u3068\u3061\u308a\u306c\u308b\u3092");
// Hebrew
assertRoundTrips(
"\u05d3\u05d2 \u05e1\u05e7\u05e8\u05df \u05e9\u05d8 \u05d1\u05d9\u05dd "
@@ -115,9 +142,10 @@ public void testRealStrings() throws Exception {
assertRoundTrips(
"\u8fd4\u56de\u94fe\u4e2d\u7684\u4e0b\u4e00\u4e2a\u4ee3\u7406\u9879\u9009\u62e9\u5668");
// Chinese with 4-byte chars
- assertRoundTrips("\uD841\uDF0E\uD841\uDF31\uD841\uDF79\uD843\uDC53\uD843\uDC78"
- + "\uD843\uDC96\uD843\uDCCF\uD843\uDCD5\uD843\uDD15\uD843\uDD7C\uD843\uDD7F"
- + "\uD843\uDE0E\uD843\uDE0F\uD843\uDE77\uD843\uDE9D\uD843\uDEA2");
+ assertRoundTrips(
+ "\uD841\uDF0E\uD841\uDF31\uD841\uDF79\uD843\uDC53\uD843\uDC78"
+ + "\uD843\uDC96\uD843\uDCCF\uD843\uDCD5\uD843\uDD15\uD843\uDD7C\uD843\uDD7F"
+ + "\uD843\uDE0E\uD843\uDE0F\uD843\uDE77\uD843\uDE9D\uD843\uDEA2");
// Mixed
assertRoundTrips(
"The quick brown \u3044\u308d\u306f\u306b\u307b\u3078\u8fd4\u56de\u94fe"
@@ -132,7 +160,7 @@ public void testOverlong() throws Exception {
// Max overlong
assertInvalid(0xc1, 0xbf);
assertInvalid(0xe0, 0x9f, 0xbf);
- assertInvalid(0xf0 ,0x8f, 0xbf, 0xbf);
+ assertInvalid(0xf0, 0x8f, 0xbf, 0xbf);
// null overlong
assertInvalid(0xc0, 0x80);
@@ -168,7 +196,7 @@ public void testBufferSlice() throws Exception {
}
public void testInvalidBufferSlice() throws Exception {
- byte[] bytes = "The quick brown fox jumps over the lazy dog".getBytes(Internal.UTF_8);
+ byte[] bytes = "The quick brown fox jumps over the lazy dog".getBytes(Internal.UTF_8);
assertInvalidSlice(bytes, bytes.length - 3, 4);
assertInvalidSlice(bytes, bytes.length, 1);
assertInvalidSlice(bytes, bytes.length + 1, 0);
@@ -180,10 +208,14 @@ private void assertInvalid(int... bytesAsInt) throws Exception {
for (int i = 0; i < bytesAsInt.length; i++) {
bytes[i] = (byte) bytesAsInt[i];
}
- assertInvalid(bytes);
+ assertInvalid(bytes, null);
}
- private void assertInvalid(byte[] bytes) throws Exception {
+ // Attempts to decode the byte array in several ways and asserts that it always generates an
+ // exception. Allocating a direct ByteBuffer is slow, so the caller can optionally provide a
+ // buffer to reuse. If buffer is non-null, it must be a direct-allocated ByteBuffer of the
+ // appropriate size.
+ private void assertInvalid(byte[] bytes, ByteBuffer buffer) throws Exception {
try {
UNSAFE_PROCESSOR.decodeUtf8(bytes, 0, bytes.length);
fail();
@@ -197,37 +229,24 @@ private void assertInvalid(byte[] bytes) throws Exception {
// Expected.
}
- ByteBuffer direct = ByteBuffer.allocateDirect(bytes.length);
- direct.put(bytes);
- direct.flip();
- try {
- UNSAFE_PROCESSOR.decodeUtf8(direct, 0, bytes.length);
- fail();
- } catch (InvalidProtocolBufferException e) {
- // Expected.
- }
- try {
- SAFE_PROCESSOR.decodeUtf8(direct, 0, bytes.length);
- fail();
- } catch (InvalidProtocolBufferException e) {
- // Expected.
+ if (buffer == null) {
+ buffer = ByteBuffer.allocateDirect(bytes.length);
}
-
- ByteBuffer heap = ByteBuffer.allocate(bytes.length);
- heap.put(bytes);
- heap.flip();
+ buffer.put(bytes);
+ buffer.flip();
try {
- UNSAFE_PROCESSOR.decodeUtf8(heap, 0, bytes.length);
+ UNSAFE_PROCESSOR.decodeUtf8(buffer, 0, bytes.length);
fail();
} catch (InvalidProtocolBufferException e) {
// Expected.
}
try {
- SAFE_PROCESSOR.decodeUtf8(heap, 0, bytes.length);
+ SAFE_PROCESSOR.decodeUtf8(buffer, 0, bytes.length);
fail();
} catch (InvalidProtocolBufferException e) {
// Expected.
}
+ buffer.clear();
}
private void assertInvalidSlice(byte[] bytes, int index, int size) throws Exception {
@@ -286,25 +305,31 @@ private void assertRoundTrips(String str, int index, int size) throws Exception
if (size == -1) {
size = bytes.length;
}
- assertDecode(new String(bytes, index, size, Internal.UTF_8),
+ assertDecode(
+ new String(bytes, index, size, Internal.UTF_8),
UNSAFE_PROCESSOR.decodeUtf8(bytes, index, size));
- assertDecode(new String(bytes, index, size, Internal.UTF_8),
+ assertDecode(
+ new String(bytes, index, size, Internal.UTF_8),
SAFE_PROCESSOR.decodeUtf8(bytes, index, size));
ByteBuffer direct = ByteBuffer.allocateDirect(bytes.length);
direct.put(bytes);
direct.flip();
- assertDecode(new String(bytes, index, size, Internal.UTF_8),
+ assertDecode(
+ new String(bytes, index, size, Internal.UTF_8),
UNSAFE_PROCESSOR.decodeUtf8(direct, index, size));
- assertDecode(new String(bytes, index, size, Internal.UTF_8),
+ assertDecode(
+ new String(bytes, index, size, Internal.UTF_8),
SAFE_PROCESSOR.decodeUtf8(direct, index, size));
ByteBuffer heap = ByteBuffer.allocate(bytes.length);
heap.put(bytes);
heap.flip();
- assertDecode(new String(bytes, index, size, Internal.UTF_8),
+ assertDecode(
+ new String(bytes, index, size, Internal.UTF_8),
UNSAFE_PROCESSOR.decodeUtf8(heap, index, size));
- assertDecode(new String(bytes, index, size, Internal.UTF_8),
+ assertDecode(
+ new String(bytes, index, size, Internal.UTF_8),
SAFE_PROCESSOR.decodeUtf8(heap, index, size));
}
@@ -321,5 +346,4 @@ private List codepoints(String str) {
}
return codepoints;
}
-
}
diff --git a/java/core/src/test/java/com/google/protobuf/IsValidUtf8Test.java b/java/core/src/test/java/com/google/protobuf/IsValidUtf8Test.java
index bbbe2cd2b392..46e56a9ccbbd 100644
--- a/java/core/src/test/java/com/google/protobuf/IsValidUtf8Test.java
+++ b/java/core/src/test/java/com/google/protobuf/IsValidUtf8Test.java
@@ -36,6 +36,7 @@
import static com.google.protobuf.IsValidUtf8TestUtil.EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT;
import static com.google.protobuf.IsValidUtf8TestUtil.HEAP_NIO_FACTORY;
import static com.google.protobuf.IsValidUtf8TestUtil.LITERAL_FACTORY;
+import static com.google.protobuf.IsValidUtf8TestUtil.ROPE_FACTORY;
import static com.google.protobuf.IsValidUtf8TestUtil.testBytes;
import com.google.protobuf.IsValidUtf8TestUtil.ByteStringFactory;
@@ -61,6 +62,7 @@ public void testIsValidUtf8_1Byte() {
testBytes(LITERAL_FACTORY, 1, EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT);
testBytes(HEAP_NIO_FACTORY, 1, EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT);
testBytes(DIRECT_NIO_FACTORY, 1, EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT);
+ testBytes(ROPE_FACTORY, 1, EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT);
}
/** Tests that round tripping of all two byte permutations work. */
@@ -69,17 +71,16 @@ public void testIsValidUtf8_2Bytes() {
testBytes(LITERAL_FACTORY, 2, IsValidUtf8TestUtil.EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT);
testBytes(HEAP_NIO_FACTORY, 2, IsValidUtf8TestUtil.EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT);
testBytes(DIRECT_NIO_FACTORY, 2, IsValidUtf8TestUtil.EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT);
+ testBytes(ROPE_FACTORY, 2, IsValidUtf8TestUtil.EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT);
}
/** Tests that round tripping of all three byte permutations work. */
@Test
public void testIsValidUtf8_3Bytes() {
- // Travis' OOM killer doesn't like this test
- if (System.getenv("TRAVIS") == null) {
- testBytes(LITERAL_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT);
- testBytes(HEAP_NIO_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT);
- testBytes(DIRECT_NIO_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT);
- }
+ testBytes(LITERAL_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT);
+ testBytes(HEAP_NIO_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT);
+ testBytes(DIRECT_NIO_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT);
+ testBytes(ROPE_FACTORY, 3, EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT);
}
/**
@@ -168,12 +169,14 @@ private void assertValidUtf8(int... bytes) {
assertValidUtf8(LITERAL_FACTORY, bytes, false);
assertValidUtf8(HEAP_NIO_FACTORY, bytes, false);
assertValidUtf8(DIRECT_NIO_FACTORY, bytes, false);
+ assertValidUtf8(ROPE_FACTORY, bytes, false);
}
private void assertInvalidUtf8(int... bytes) {
assertValidUtf8(LITERAL_FACTORY, bytes, true);
assertValidUtf8(HEAP_NIO_FACTORY, bytes, true);
assertValidUtf8(DIRECT_NIO_FACTORY, bytes, true);
+ assertValidUtf8(ROPE_FACTORY, bytes, true);
}
private static ByteString asBytes(String s) {
diff --git a/java/core/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java b/java/core/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java
index 94a9ffb5d26e..f1a671164d18 100644
--- a/java/core/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java
+++ b/java/core/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java
@@ -35,16 +35,10 @@
import java.lang.ref.SoftReference;
import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CharsetEncoder;
-import java.nio.charset.CoderResult;
-import java.nio.charset.CodingErrorAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
-import java.util.logging.Logger;
/**
* Shared testing code for {@link IsValidUtf8Test} and {@link IsValidUtf8FourByteTest}.
@@ -53,8 +47,6 @@
* @author martinrb@google.com (Martin Buchholz)
*/
final class IsValidUtf8TestUtil {
- private static final Logger logger = Logger.getLogger(IsValidUtf8TestUtil.class.getName());
-
private IsValidUtf8TestUtil() {}
static interface ByteStringFactory {
@@ -102,6 +94,29 @@ public ByteString newByteString(byte[] bytes) {
}
};
+ static final ByteStringFactory ROPE_FACTORY =
+ new ByteStringFactory() {
+ // Seed the random number generator with 0 so that the tests are deterministic.
+ private final Random random = new Random(0);
+
+ @Override
+ public ByteString newByteString(byte[] bytes) {
+ // We split the byte array into three pieces (some possibly empty) by choosing two random
+ // cut points i and j.
+ int i = random.nextInt(bytes.length);
+ int j = random.nextInt(bytes.length);
+ if (j < i) {
+ int tmp = i;
+ i = j;
+ j = tmp;
+ }
+ return RopeByteString.newInstanceForTest(
+ ByteString.wrap(bytes, 0, i),
+ RopeByteString.newInstanceForTest(
+ ByteString.wrap(bytes, i, j - i), ByteString.wrap(bytes, j, bytes.length - j)));
+ }
+ };
+
// 128 - [chars 0x0000 to 0x007f]
static final long ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS = 0x007f - 0x0000 + 1;
@@ -247,13 +262,11 @@ static void testBytes(ByteStringFactory factory, int numBytes, long expectedCoun
*/
static void testBytes(
ByteStringFactory factory, int numBytes, long expectedCount, long start, long lim) {
- Random rnd = new Random();
byte[] bytes = new byte[numBytes];
if (lim == -1) {
lim = 1L << (numBytes * 8);
}
- long count = 0;
long countRoundTripped = 0;
for (long byteChar = start; byteChar < lim; byteChar++) {
long tmpByteChar = byteChar;
@@ -271,166 +284,13 @@ static void testBytes(
outputFailure(byteChar, bytes, bytesReencoded);
}
- // Check agreement with static Utf8 methods.
+ // Check agreement with Utf8.isValidUtf8.
assertThat(Utf8.isValidUtf8(bytes)).isEqualTo(isRoundTrippable);
- assertThat(Utf8.isValidUtf8(bytes, 0, numBytes)).isEqualTo(isRoundTrippable);
-
- try {
- assertThat(Utf8.decodeUtf8(bytes, 0, numBytes)).isEqualTo(s);
- } catch (InvalidProtocolBufferException e) {
- if (isRoundTrippable) {
- System.out.println("Could not decode utf-8");
- outputFailure(byteChar, bytes, bytesReencoded);
- }
- }
-
- // Test partial sequences.
- // Partition numBytes into three segments (not necessarily non-empty).
- int i = rnd.nextInt(numBytes);
- int j = rnd.nextInt(numBytes);
- if (j < i) {
- int tmp = i;
- i = j;
- j = tmp;
- }
- int state1 = Utf8.partialIsValidUtf8(Utf8.COMPLETE, bytes, 0, i);
- int state2 = Utf8.partialIsValidUtf8(state1, bytes, i, j);
- int state3 = Utf8.partialIsValidUtf8(state2, bytes, j, numBytes);
- if (isRoundTrippable != (state3 == Utf8.COMPLETE)) {
- System.out.printf("state=%04x %04x %04x i=%d j=%d%n", state1, state2, state3, i, j);
- outputFailure(byteChar, bytes, bytesReencoded);
- }
- assertThat((state3 == Utf8.COMPLETE)).isEqualTo(isRoundTrippable);
-
- // Test ropes built out of small partial sequences
- ByteString rope =
- RopeByteString.newInstanceForTest(
- bs.substring(0, i),
- RopeByteString.newInstanceForTest(bs.substring(i, j), bs.substring(j, numBytes)));
- assertThat(rope.getClass()).isSameInstanceAs(RopeByteString.class);
-
- ByteString[] byteStrings = {bs, bs.substring(0, numBytes), rope};
- for (ByteString x : byteStrings) {
- assertThat(x.isValidUtf8()).isEqualTo(isRoundTrippable);
- assertThat(x.partialIsValidUtf8(Utf8.COMPLETE, 0, numBytes)).isEqualTo(state3);
-
- assertThat(x.partialIsValidUtf8(Utf8.COMPLETE, 0, i)).isEqualTo(state1);
- assertThat(x.substring(0, i).partialIsValidUtf8(Utf8.COMPLETE, 0, i)).isEqualTo(state1);
- assertThat(x.partialIsValidUtf8(state1, i, j - i)).isEqualTo(state2);
- assertThat(x.substring(i, j).partialIsValidUtf8(state1, 0, j - i)).isEqualTo(state2);
- assertThat(x.partialIsValidUtf8(state2, j, numBytes - j)).isEqualTo(state3);
- assertThat(x.substring(j, numBytes).partialIsValidUtf8(state2, 0, numBytes - j))
- .isEqualTo(state3);
- }
-
- // ByteString reduplication should not affect its UTF-8 validity.
- ByteString ropeADope = RopeByteString.newInstanceForTest(bs, bs.substring(0, numBytes));
- assertThat(ropeADope.isValidUtf8()).isEqualTo(isRoundTrippable);
-
- if (isRoundTrippable) {
- countRoundTripped++;
- }
- count++;
- if (byteChar != 0 && byteChar % 1000000L == 0) {
- logger.info("Processed " + (byteChar / 1000000L) + " million characters");
- }
- }
- logger.info("Round tripped " + countRoundTripped + " of " + count);
- assertThat(countRoundTripped).isEqualTo(expectedCount);
- }
-
- /**
- * Variation of {@link #testBytes} that does less allocation using the low-level encoders/decoders
- * directly. Checked in because it's useful for debugging when trying to process bytes faster, but
- * since it doesn't use the actual String class, it's possible for incompatibilities to develop
- * (although unlikely).
- *
- * @param factory the factory for {@link ByteString} instances.
- * @param numBytes the number of bytes in the byte array
- * @param expectedCount the expected number of roundtrippable permutations
- * @param start the starting bytes encoded as a long as big-endian
- * @param lim the limit of bytes to process encoded as a long as big-endian, or -1 to mean the max
- * limit for numBytes
- */
- static void testBytesUsingByteBuffers(
- ByteStringFactory factory, int numBytes, long expectedCount, long start, long lim) {
- CharsetDecoder decoder =
- Internal.UTF_8
- .newDecoder()
- .onMalformedInput(CodingErrorAction.REPLACE)
- .onUnmappableCharacter(CodingErrorAction.REPLACE);
- CharsetEncoder encoder =
- Internal.UTF_8
- .newEncoder()
- .onMalformedInput(CodingErrorAction.REPLACE)
- .onUnmappableCharacter(CodingErrorAction.REPLACE);
- byte[] bytes = new byte[numBytes];
- int maxChars = (int) (decoder.maxCharsPerByte() * numBytes) + 1;
- char[] charsDecoded = new char[(int) (decoder.maxCharsPerByte() * numBytes) + 1];
- int maxBytes = (int) (encoder.maxBytesPerChar() * maxChars) + 1;
- byte[] bytesReencoded = new byte[maxBytes];
-
- ByteBuffer bb = ByteBuffer.wrap(bytes);
- CharBuffer cb = CharBuffer.wrap(charsDecoded);
- ByteBuffer bbReencoded = ByteBuffer.wrap(bytesReencoded);
- if (lim == -1) {
- lim = 1L << (numBytes * 8);
- }
- long count = 0;
- long countRoundTripped = 0;
- for (long byteChar = start; byteChar < lim; byteChar++) {
- bb.rewind();
- bb.limit(bytes.length);
- cb.rewind();
- cb.limit(charsDecoded.length);
- bbReencoded.rewind();
- bbReencoded.limit(bytesReencoded.length);
- encoder.reset();
- decoder.reset();
- long tmpByteChar = byteChar;
- for (int i = 0; i < bytes.length; i++) {
- bytes[bytes.length - i - 1] = (byte) tmpByteChar;
- tmpByteChar = tmpByteChar >> 8;
- }
- boolean isRoundTrippable = factory.newByteString(bytes).isValidUtf8();
- CoderResult result = decoder.decode(bb, cb, true);
- assertThat(result.isError()).isFalse();
- result = decoder.flush(cb);
- assertThat(result.isError()).isFalse();
-
- int charLen = cb.position();
- cb.rewind();
- cb.limit(charLen);
- result = encoder.encode(cb, bbReencoded, true);
- assertThat(result.isError()).isFalse();
- result = encoder.flush(bbReencoded);
- assertThat(result.isError()).isFalse();
-
- boolean bytesEqual = true;
- int bytesLen = bbReencoded.position();
- if (bytesLen != numBytes) {
- bytesEqual = false;
- } else {
- for (int i = 0; i < numBytes; i++) {
- if (bytes[i] != bytesReencoded[i]) {
- bytesEqual = false;
- break;
- }
- }
- }
- if (bytesEqual != isRoundTrippable) {
- outputFailure(byteChar, bytes, bytesReencoded, bytesLen);
- }
- count++;
if (isRoundTrippable) {
countRoundTripped++;
}
- if (byteChar != 0 && byteChar % 1000000 == 0) {
- logger.info("Processed " + (byteChar / 1000000) + " million characters");
- }
}
- logger.info("Round tripped " + countRoundTripped + " of " + count);
assertThat(countRoundTripped).isEqualTo(expectedCount);
}
diff --git a/java/core/src/test/java/com/google/protobuf/MessageTest.java b/java/core/src/test/java/com/google/protobuf/MessageTest.java
index 9c34161b6516..fa3643393ffc 100644
--- a/java/core/src/test/java/com/google/protobuf/MessageTest.java
+++ b/java/core/src/test/java/com/google/protobuf/MessageTest.java
@@ -380,4 +380,34 @@ public void testDynamicRepeatedMessageNotNull() throws Exception {
result.getDescriptorForType().findFieldByName("repeated_foreign_message")))
.isEqualTo(2);
}
+
+ @Test
+ public void testPreservesFloatingPointNegative0() throws Exception {
+ proto3_unittest.UnittestProto3.TestAllTypes message =
+ proto3_unittest.UnittestProto3.TestAllTypes.newBuilder()
+ .setOptionalFloat(-0.0f)
+ .setOptionalDouble(-0.0)
+ .build();
+ assertThat(
+ proto3_unittest.UnittestProto3.TestAllTypes.parseFrom(
+ message.toByteString(), ExtensionRegistry.getEmptyRegistry()))
+ .isEqualTo(message);
+ }
+
+ @Test
+ public void testNegative0FloatingPointEquality() throws Exception {
+ // Like Double#equals and Float#equals, we treat -0.0 as not being equal to +0.0 even though
+ // IEEE 754 mandates that they are equivalent. This test asserts that behavior.
+ proto3_unittest.UnittestProto3.TestAllTypes message1 =
+ proto3_unittest.UnittestProto3.TestAllTypes.newBuilder()
+ .setOptionalFloat(-0.0f)
+ .setOptionalDouble(-0.0)
+ .build();
+ proto3_unittest.UnittestProto3.TestAllTypes message2 =
+ proto3_unittest.UnittestProto3.TestAllTypes.newBuilder()
+ .setOptionalFloat(0.0f)
+ .setOptionalDouble(0.0)
+ .build();
+ assertThat(message1).isNotEqualTo(message2);
+ }
}
diff --git a/java/core/src/test/java/com/google/protobuf/ServiceTest.java b/java/core/src/test/java/com/google/protobuf/ServiceTest.java
index d7996782f71b..33ce537a4aaf 100644
--- a/java/core/src/test/java/com/google/protobuf/ServiceTest.java
+++ b/java/core/src/test/java/com/google/protobuf/ServiceTest.java
@@ -316,7 +316,6 @@ public WrapsCallback(MockCallback callback) {
}
@Override
- @SuppressWarnings("unchecked")
public boolean matches(Object actual) {
if (!(actual instanceof RpcCallback)) {
return false;
diff --git a/java/core/src/test/java/com/google/protobuf/TextFormatParseInfoTreeTest.java b/java/core/src/test/java/com/google/protobuf/TextFormatParseInfoTreeTest.java
index 5d0eb62c5d52..b6241912a32f 100644
--- a/java/core/src/test/java/com/google/protobuf/TextFormatParseInfoTreeTest.java
+++ b/java/core/src/test/java/com/google/protobuf/TextFormatParseInfoTreeTest.java
@@ -41,7 +41,7 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-/** Test @{link TextFormatParseInfoTree}. */
+/** Test {@link TextFormatParseInfoTree}. */
@RunWith(JUnit4.class)
public class TextFormatParseInfoTreeTest {
diff --git a/java/core/src/test/java/com/google/protobuf/TextFormatParseLocationTest.java b/java/core/src/test/java/com/google/protobuf/TextFormatParseLocationTest.java
index ed20df5cbe3a..f85ee6c5a1ba 100644
--- a/java/core/src/test/java/com/google/protobuf/TextFormatParseLocationTest.java
+++ b/java/core/src/test/java/com/google/protobuf/TextFormatParseLocationTest.java
@@ -37,7 +37,7 @@
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
-/** Test @{link TextFormatParseLocation}. */
+/** Test {@link TextFormatParseLocation}. */
@RunWith(JUnit4.class)
public class TextFormatParseLocationTest {
diff --git a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java
index e755111f0f9e..f9b7da46b7cd 100644
--- a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java
+++ b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java
@@ -1815,4 +1815,15 @@ public void testSortMapFields() throws Exception {
+ "}\n";
assertThat(TextFormat.printer().printToString(message)).isEqualTo(text);
}
+
+ @Test
+ public void testPreservesFloatingPointNegative0() throws Exception {
+ proto3_unittest.UnittestProto3.TestAllTypes message =
+ proto3_unittest.UnittestProto3.TestAllTypes.newBuilder()
+ .setOptionalFloat(-0.0f)
+ .setOptionalDouble(-0.0)
+ .build();
+ assertThat(TextFormat.printer().printToString(message))
+ .isEqualTo("optional_float: -0.0\noptional_double: -0.0\n");
+ }
}
diff --git a/java/lite/src/test/java/com/google/protobuf/LiteTest.java b/java/lite/src/test/java/com/google/protobuf/LiteTest.java
index 3680cc23bc07..b0972111acd8 100644
--- a/java/lite/src/test/java/com/google/protobuf/LiteTest.java
+++ b/java/lite/src/test/java/com/google/protobuf/LiteTest.java
@@ -2752,6 +2752,36 @@ public void testUnpairedSurrogatesReplacedByQuestionMark() throws InvalidProtoco
.isTrue();
}
+ @Test
+ public void testPreservesFloatingPointNegative0() throws Exception {
+ proto3_unittest.UnittestProto3.TestAllTypes message =
+ proto3_unittest.UnittestProto3.TestAllTypes.newBuilder()
+ .setOptionalFloat(-0.0f)
+ .setOptionalDouble(-0.0)
+ .build();
+ assertThat(
+ proto3_unittest.UnittestProto3.TestAllTypes.parseFrom(
+ message.toByteString(), ExtensionRegistryLite.getEmptyRegistry()))
+ .isEqualTo(message);
+ }
+
+ @Test
+ public void testNegative0FloatingPointEquality() throws Exception {
+ // Like Double#equals and Float#equals, we treat -0.0 as not being equal to +0.0 even though
+ // IEEE 754 mandates that they are equivalent. This test asserts that behavior.
+ proto3_unittest.UnittestProto3.TestAllTypes message1 =
+ proto3_unittest.UnittestProto3.TestAllTypes.newBuilder()
+ .setOptionalFloat(-0.0f)
+ .setOptionalDouble(-0.0)
+ .build();
+ proto3_unittest.UnittestProto3.TestAllTypes message2 =
+ proto3_unittest.UnittestProto3.TestAllTypes.newBuilder()
+ .setOptionalFloat(0.0f)
+ .setOptionalDouble(0.0)
+ .build();
+ assertThat(message1).isNotEqualTo(message2);
+ }
+
private String encodeHex(ByteString bytes) {
String hexDigits = "0123456789abcdef";
StringBuilder stringBuilder = new StringBuilder(bytes.size() * 2);
diff --git a/java/util/BUILD b/java/util/BUILD
index 02e55496897f..3855da96f01d 100644
--- a/java/util/BUILD
+++ b/java/util/BUILD
@@ -14,6 +14,7 @@ java_library(
"//external:error_prone_annotations",
"//external:j2objc_annotations",
"//external:gson",
+ "//external:jsr305",
"//external:guava",
"//java/core",
"//java/lite",
diff --git a/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java b/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java
index a6cdcb995765..9f36494a710d 100644
--- a/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java
+++ b/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java
@@ -46,6 +46,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import javax.annotation.Nullable;
/**
* Utility helper functions to work with {@link com.google.protobuf.FieldMask}.
@@ -230,10 +231,8 @@ public static boolean isValid(Class type, String path) {
return isValid(descriptor, path);
}
- /**
- * Checks whether paths in a given fields mask are valid.
- */
- public static boolean isValid(Descriptor descriptor, String path) {
+ /** Checks whether paths in a given fields mask are valid. */
+ public static boolean isValid(@Nullable Descriptor descriptor, String path) {
String[] parts = path.split(FIELD_SEPARATOR_REGEX);
if (parts.length == 0) {
return false;
diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java
index d94fb3d85948..ddfbc9ed900e 100644
--- a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java
+++ b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java
@@ -88,6 +88,7 @@
import java.util.Set;
import java.util.TreeMap;
import java.util.logging.Logger;
+import javax.annotation.Nullable;
/**
* Utility classes to convert protobuf messages to/from JSON format. The JSON
@@ -521,10 +522,12 @@ public static Builder newBuilder() {
* Find a type by its full name. Returns null if it cannot be found in this {@link
* TypeRegistry}.
*/
+ @Nullable
public Descriptor find(String name) {
return types.get(name);
}
+ @Nullable
Descriptor getDescriptorForTypeUrl(String typeUrl) throws InvalidProtocolBufferException {
return find(getTypeName(typeUrl));
}
@@ -546,7 +549,7 @@ private Builder() {}
*/
@CanIgnoreReturnValue
public Builder add(Descriptor messageType) {
- if (types == null) {
+ if (built) {
throw new IllegalStateException("A TypeRegistry.Builder can only be used once.");
}
addFile(messageType.getFile());
@@ -559,7 +562,7 @@ public Builder add(Descriptor messageType) {
*/
@CanIgnoreReturnValue
public Builder add(Iterable messageTypes) {
- if (types == null) {
+ if (built) {
throw new IllegalStateException("A TypeRegistry.Builder can only be used once.");
}
for (Descriptor type : messageTypes) {
@@ -573,10 +576,8 @@ public Builder add(Iterable messageTypes) {
* one Builder.
*/
public TypeRegistry build() {
- TypeRegistry result = new TypeRegistry(types);
- // Make sure the built {@link TypeRegistry} is immutable.
- types = null;
- return result;
+ built = true;
+ return new TypeRegistry(types);
}
private void addFile(FileDescriptor file) {
@@ -607,6 +608,7 @@ private void addMessage(Descriptor message) {
private final Set files = new HashSet();
private Map types = new HashMap();
+ private boolean built = false;
}
}
@@ -984,7 +986,7 @@ private void printListValue(MessageOrBuilder message) throws IOException {
}
/** Prints a regular message with an optional type URL. */
- private void print(MessageOrBuilder message, String typeUrl) throws IOException {
+ private void print(MessageOrBuilder message, @Nullable String typeUrl) throws IOException {
generator.print("{" + blankOrNewLine);
generator.indent();
@@ -1340,7 +1342,9 @@ void merge(String json, Message.Builder builder) throws InvalidProtocolBufferExc
throw e;
} catch (Exception e) {
// We convert all exceptions from JSON parsing to our own exceptions.
- throw new InvalidProtocolBufferException(e.getMessage());
+ InvalidProtocolBufferException toThrow = new InvalidProtocolBufferException(e.getMessage());
+ toThrow.initCause(e);
+ throw toThrow;
}
}
@@ -1899,6 +1903,7 @@ private ByteString parseBytes(JsonElement json) throws InvalidProtocolBufferExce
}
}
+ @Nullable
private EnumValueDescriptor parseEnum(EnumDescriptor enumDescriptor, JsonElement json)
throws InvalidProtocolBufferException {
String value = json.getAsString();
@@ -1926,6 +1931,7 @@ private EnumValueDescriptor parseEnum(EnumDescriptor enumDescriptor, JsonElement
return result;
}
+ @Nullable
private Object parseFieldValue(FieldDescriptor field, JsonElement json, Message.Builder builder)
throws InvalidProtocolBufferException {
if (json instanceof JsonNull) {
diff --git a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java
index bd58cc9a8ea6..f00bbb1b4c9e 100644
--- a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java
+++ b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java
@@ -1887,4 +1887,12 @@ public void testPrintingEnumsAsIntsChainedAfterIncludingDefaultValueFields() thr
.print(message))
.isEqualTo("{\n" + " \"optionalBool\": false\n" + "}");
}
+
+ @Test
+ public void testPreservesFloatingPointNegative0() throws Exception {
+ TestAllTypes message =
+ TestAllTypes.newBuilder().setOptionalFloat(-0.0f).setOptionalDouble(-0.0).build();
+ assertThat(JsonFormat.printer().print(message))
+ .isEqualTo("{\n \"optionalFloat\": -0.0,\n \"optionalDouble\": -0.0\n}");
+ }
}
diff --git a/js/binary/utils_test.js b/js/binary/utils_test.js
index bc1707d9e3bd..70585db46e05 100644
--- a/js/binary/utils_test.js
+++ b/js/binary/utils_test.js
@@ -394,7 +394,7 @@ describe('binaryUtilsTest', function() {
// corner cases
test(0.9999999762949594, 0x3f800000);
test(7.99999999999999, 0x41000000);
- test(Math.sin(30 * Math.PI / 180), 0x3f000000); // sin(30 degrees)
+ test(Math.sin(30 * Math.PI / 180), 0x3f000000); // sin(30 degrees)
// Various positive values.
var cursor = f32_eps * 10;
diff --git a/maven_install.json b/maven_install.json
index f9342185c4e4..6168aa4af246 100644
--- a/maven_install.json
+++ b/maven_install.json
@@ -1,6 +1,8 @@
{
"dependency_tree": {
- "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": 1033791982,
+ "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL",
+ "__INPUT_ARTIFACTS_HASH": 1907885757,
+ "__RESOLVED_ARTIFACTS_HASH": 375457873,
"conflict_resolution": {
"com.google.errorprone:error_prone_annotations:2.3.2": "com.google.errorprone:error_prone_annotations:2.5.1",
"junit:junit:4.12": "junit:junit:4.13.1"
@@ -201,24 +203,6 @@
"sha256": "b3dd1cf5019f942d8cc2afad0aa6aef4b21532446fe90a6b68d567e3389763dd",
"url": "https://repo1.maven.org/maven2/org/easymock/easymock/3.2/easymock-3.2.jar"
},
- {
- "coord": "org.easymock:easymockclassextension:3.2",
- "dependencies": [
- "org.easymock:easymock:3.2",
- "cglib:cglib-nodep:2.2.2",
- "org.objenesis:objenesis:1.3"
- ],
- "directDependencies": [
- "org.easymock:easymock:3.2"
- ],
- "file": "v1/https/repo1.maven.org/maven2/org/easymock/easymockclassextension/3.2/easymockclassextension-3.2.jar",
- "mirror_urls": [
- "https://repo1.maven.org/maven2/org/easymock/easymockclassextension/3.2/easymockclassextension-3.2.jar",
- "https://repo.maven.apache.org/maven2/org/easymock/easymockclassextension/3.2/easymockclassextension-3.2.jar"
- ],
- "sha256": "e2aeb3ecec87d859b2f3072985d4b15873558bcf6410f422db0c0c5194c76c87",
- "url": "https://repo1.maven.org/maven2/org/easymock/easymockclassextension/3.2/easymockclassextension-3.2.jar"
- },
{
"coord": "org.hamcrest:hamcrest-core:1.3",
"dependencies": [],
diff --git a/python/google/protobuf/internal/api_implementation.py b/python/google/protobuf/internal/api_implementation.py
index a3667318c1b7..7fef2376707d 100644
--- a/python/google/protobuf/internal/api_implementation.py
+++ b/python/google/protobuf/internal/api_implementation.py
@@ -41,46 +41,15 @@
# The compile-time constants in the _api_implementation module can be used to
# switch to a certain implementation of the Python API at build time.
_api_version = _api_implementation.api_version
- _proto_extension_modules_exist_in_build = True
except ImportError:
_api_version = -1 # Unspecified by compiler flags.
- _proto_extension_modules_exist_in_build = False
if _api_version == 1:
raise ValueError('api_version=1 is no longer supported.')
-if _api_version < 0: # Still unspecified?
- try:
- # The presence of this module in a build allows the proto implementation to
- # be upgraded merely via build deps rather than a compiler flag or the
- # runtime environment variable.
- # pylint: disable=g-import-not-at-top
- from google.protobuf import _use_fast_cpp_protos
- # Work around a known issue in the classic bootstrap .par import hook.
- if not _use_fast_cpp_protos:
- raise ImportError('_use_fast_cpp_protos import succeeded but was None')
- del _use_fast_cpp_protos
- _api_version = 2
- from google.protobuf import use_pure_python
- raise RuntimeError(
- 'Conflicting deps on both :use_fast_cpp_protos and :use_pure_python.\n'
- ' go/build_deps_on_BOTH_use_fast_cpp_protos_AND_use_pure_python\n'
- 'This should be impossible via a link error at build time...')
- except ImportError:
- try:
- # pylint: disable=g-import-not-at-top
- from google.protobuf import use_pure_python
- del use_pure_python # Avoids a pylint error and namespace pollution.
- _api_version = 0
- except ImportError:
- # TODO(b/74017912): It's unsafe to enable :use_fast_cpp_protos by default;
- # it can cause data loss if you have any Python-only extensions to any
- # message passed back and forth with C++ code.
- #
- # TODO(b/17427486): Once that bug is fixed, we want to make both Python 2
- # and Python 3 default to `_api_version = 2` (C++ implementation V2).
- pass
-
-_default_implementation_type = ('python' if _api_version <= 0 else 'cpp')
+
+
+_default_implementation_type = ('cpp' if _api_version > 0 else 'python')
+
# This environment variable can be used to switch to a certain implementation
# of the Python API, overriding the compile-time constants in the
@@ -97,21 +66,6 @@
'Falling back to the python implementation.')
_implementation_type = 'python'
-# This environment variable can be used to switch between the two
-# 'cpp' implementations, overriding the compile-time constants in the
-# _api_implementation module. Right now only '2' is supported. Any other
-# value will cause an error to be raised.
-_implementation_version_str = os.getenv(
- 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION', '2')
-
-if _implementation_version_str != '2':
- raise ValueError(
- 'unsupported PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION: "' +
- _implementation_version_str + '" (supported versions: 2)'
- )
-
-_implementation_version = int(_implementation_version_str)
-
# Detect if serialization should be deterministic by default
try:
@@ -150,7 +104,7 @@ def _SetType(implementation_type):
# See comment on 'Type' above.
def Version():
- return _implementation_version
+ return 2
# For internal use only
diff --git a/python/google/protobuf/message.py b/python/google/protobuf/message.py
index 224d2fc49178..ee46d0e4c9c6 100644
--- a/python/google/protobuf/message.py
+++ b/python/google/protobuf/message.py
@@ -359,6 +359,14 @@ def ByteSize(self):
"""
raise NotImplementedError
+ @classmethod
+ def FromString(cls, s):
+ raise NotImplementedError
+
+ @staticmethod
+ def RegisterExtension(extension_handle):
+ raise NotImplementedError
+
def _SetListener(self, message_listener):
"""Internal method used by the protocol message implementation.
Clients should not call this directly.
diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc
index 4340afc4e152..cb48faa440bf 100644
--- a/python/google/protobuf/pyext/message.cc
+++ b/python/google/protobuf/pyext/message.cc
@@ -1919,7 +1919,9 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) {
// explicit in our correctness checks.
if (ptr == nullptr || ctx.BytesUntilLimit(ptr) < 0) {
// Parse error or the parser overshoot the limit.
- PyErr_Format(DecodeError_class, "Error parsing message");
+ PyErr_Format(
+ DecodeError_class, "Error parsing message with type '%s'",
+ self->GetMessageClass()->message_descriptor->full_name().c_str());
return NULL;
}
// ctx has an explicit limit set (length of string_view), so we have to
diff --git a/python/setup.py b/python/setup.py
index 63118c23cb85..ab42b6fe173d 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -211,11 +211,11 @@ def get_option_from_sys_argv(option_str):
extra_compile_args = []
if sys.platform != 'win32':
- extra_compile_args.append('-Wno-write-strings')
- extra_compile_args.append('-Wno-invalid-offsetof')
- extra_compile_args.append('-Wno-sign-compare')
- extra_compile_args.append('-Wno-unused-variable')
- extra_compile_args.append('-std=c++11')
+ extra_compile_args.append('-Wno-write-strings')
+ extra_compile_args.append('-Wno-invalid-offsetof')
+ extra_compile_args.append('-Wno-sign-compare')
+ extra_compile_args.append('-Wno-unused-variable')
+ extra_compile_args.append('-std=c++11')
if sys.platform == 'darwin':
extra_compile_args.append("-Wno-shorten-64-to-32");
@@ -285,21 +285,20 @@ def get_option_from_sys_argv(option_str):
maintainer_email='protobuf@googlegroups.com',
license='3-Clause BSD License',
classifiers=[
- "Programming Language :: Python",
- "Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3.3",
- "Programming Language :: Python :: 3.4",
- "Programming Language :: Python :: 3.5",
- "Programming Language :: Python :: 3.6",
- "Programming Language :: Python :: 3.7",
- ],
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.3",
+ "Programming Language :: Python :: 3.4",
+ "Programming Language :: Python :: 3.5",
+ "Programming Language :: Python :: 3.6",
+ "Programming Language :: Python :: 3.7",
+ ],
namespace_packages=['google'],
packages=find_packages(
exclude=[
'import_test_package',
'protobuf_distutils',
- ],
- ),
+ ],),
test_suite='google.protobuf.internal',
cmdclass={
'clean': clean,
@@ -309,5 +308,5 @@ def get_option_from_sys_argv(option_str):
},
install_requires=install_requires,
ext_modules=ext_module_list,
- python_requires='>=3.5',
+ python_requires=">=3.5",
)
diff --git a/src/Makefile.am b/src/Makefile.am
index 102af0441794..232955f90493 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -18,7 +18,7 @@ else
PTHREAD_DEF =
endif
-PROTOBUF_VERSION = 29:1:0
+PROTOBUF_VERSION = 28:3:0
if GCC
# Turn on all warnings except for sign comparison (we ignore sign comparison
@@ -145,6 +145,7 @@ nobase_include_HEADERS = \
google/protobuf/reflection.h \
google/protobuf/reflection_ops.h \
google/protobuf/repeated_field.h \
+ google/protobuf/repeated_ptr_field.h \
google/protobuf/service.h \
google/protobuf/source_context.pb.h \
google/protobuf/struct.pb.h \
@@ -210,6 +211,7 @@ libprotobuf_lite_la_SOURCES = \
google/protobuf/message_lite.cc \
google/protobuf/parse_context.cc \
google/protobuf/repeated_field.cc \
+ google/protobuf/repeated_ptr_field.cc \
google/protobuf/string_member_robber.h \
google/protobuf/stubs/bytestream.cc \
google/protobuf/stubs/common.cc \
@@ -541,8 +543,8 @@ EXTRA_DIST = \
google/protobuf/compiler/package_info.h \
google/protobuf/compiler/ruby/ruby_generated_code.proto \
google/protobuf/compiler/ruby/ruby_generated_code_pb.rb \
- google/protobuf/compiler/ruby/ruby_generated_code_proto2_import.proto \
google/protobuf/compiler/ruby/ruby_generated_code_proto2.proto \
+ google/protobuf/compiler/ruby/ruby_generated_code_proto2_import.proto \
google/protobuf/compiler/ruby/ruby_generated_code_proto2_pb.rb \
google/protobuf/compiler/ruby/ruby_generated_pkg_explicit.proto \
google/protobuf/compiler/ruby/ruby_generated_pkg_explicit_legacy.proto \
diff --git a/src/google/protobuf/arena_impl.h b/src/google/protobuf/arena_impl.h
index d38e31167eac..2ffac319b411 100644
--- a/src/google/protobuf/arena_impl.h
+++ b/src/google/protobuf/arena_impl.h
@@ -202,13 +202,6 @@ class PROTOBUF_EXPORT SerialArena {
void (*cleanup)(void*); // Function pointer to the destructor or deleter.
};
- // Creates a new SerialArena inside mem using the remaining memory as for
- // future allocations.
- static SerialArena* New(SerialArena::Memory mem, void* owner);
- // Free SerialArena returning the memory passed in to New
- template
- Memory Free(Deallocator deallocator);
-
void CleanupList();
uint64_t SpaceAllocated() const {
return space_allocated_.load(std::memory_order_relaxed);
@@ -281,6 +274,16 @@ class PROTOBUF_EXPORT SerialArena {
void set_next(SerialArena* next) { next_ = next; }
private:
+ friend class ThreadSafeArena;
+ friend class ArenaBenchmark;
+
+ // Creates a new SerialArena inside mem using the remaining memory as for
+ // future allocations.
+ static SerialArena* New(SerialArena::Memory mem, void* owner);
+ // Free SerialArena returning the memory passed in to New
+ template
+ Memory Free(Deallocator deallocator);
+
// Blocks are variable length malloc-ed objects. The following structure
// describes the common header for all blocks.
struct Block {
diff --git a/src/google/protobuf/arenastring.cc b/src/google/protobuf/arenastring.cc
index 7608b13c7a4c..169f52729d61 100644
--- a/src/google/protobuf/arenastring.cc
+++ b/src/google/protobuf/arenastring.cc
@@ -256,6 +256,12 @@ void ArenaStringPtr::ClearToDefault(const LazyString& default_value,
}
}
+inline void SetStrWithHeapBuffer(std::string* str, ArenaStringPtr* s) {
+ TaggedPtr res;
+ res.Set(str);
+ s->UnsafeSetTaggedPointer(res);
+}
+
const char* EpsCopyInputStream::ReadArenaString(const char* ptr,
ArenaStringPtr* s,
Arena* arena) {
@@ -264,13 +270,11 @@ const char* EpsCopyInputStream::ReadArenaString(const char* ptr,
int size = ReadSize(&ptr);
if (!ptr) return nullptr;
- auto str = Arena::Create(arena);
+ auto* str = Arena::Create(arena);
ptr = ReadString(ptr, size, str);
GOOGLE_PROTOBUF_PARSER_ASSERT(ptr);
- TaggedPtr res;
- res.Set(str);
- s->UnsafeSetTaggedPointer(res);
+ SetStrWithHeapBuffer(str, s);
return ptr;
}
diff --git a/src/google/protobuf/compiler/cpp/cpp_enum_field.cc b/src/google/protobuf/compiler/cpp/cpp_enum_field.cc
index cdcd9a6da225..70311dde40ed 100644
--- a/src/google/protobuf/compiler/cpp/cpp_enum_field.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_enum_field.cc
@@ -334,106 +334,6 @@ void RepeatedEnumFieldGenerator::GenerateConstructorCode(
// Not needed for repeated fields.
}
-void RepeatedEnumFieldGenerator::GenerateMergeFromCodedStream(
- io::Printer* printer) const {
- Formatter format(printer, variables_);
- // Don't use ReadRepeatedPrimitive here so that the enum can be validated.
- format(
- "int value = 0;\n"
- "DO_((::$proto_ns$::internal::WireFormatLite::ReadPrimitive<\n"
- " int, ::$proto_ns$::internal::WireFormatLite::TYPE_ENUM>(\n"
- " input, &value)));\n");
- if (HasPreservingUnknownEnumSemantics(descriptor_)) {
- format("add_$name$(static_cast< $type$ >(value));\n");
- } else {
- format(
- "if ($type$_IsValid(value)) {\n"
- " add_$name$(static_cast< $type$ >(value));\n");
- if (UseUnknownFieldSet(descriptor_->file(), options_)) {
- format(
- "} else {\n"
- " mutable_unknown_fields()->AddVarint(\n"
- " $number$, static_cast<$uint64$>(value));\n");
- } else {
- format(
- "} else {\n"
- " unknown_fields_stream.WriteVarint32(tag);\n"
- " unknown_fields_stream.WriteVarint32(\n"
- " static_cast<$uint32$>(value));\n");
- }
- format("}\n");
- }
-}
-
-void RepeatedEnumFieldGenerator::GenerateMergeFromCodedStreamWithPacking(
- io::Printer* printer) const {
- Formatter format(printer, variables_);
- if (!descriptor_->is_packed()) {
- // This path is rarely executed, so we use a non-inlined implementation.
- if (HasPreservingUnknownEnumSemantics(descriptor_)) {
- format(
- "DO_((::$proto_ns$::internal::"
- "WireFormatLite::ReadPackedEnumPreserveUnknowns(\n"
- " input,\n"
- " $number$,\n"
- " nullptr,\n"
- " nullptr,\n"
- " this->_internal_mutable_$name$())));\n");
- } else if (UseUnknownFieldSet(descriptor_->file(), options_)) {
- format(
- "DO_((::$proto_ns$::internal::WireFormat::"
- "ReadPackedEnumPreserveUnknowns(\n"
- " input,\n"
- " $number$,\n"
- " $type$_IsValid,\n"
- " mutable_unknown_fields(),\n"
- " this->_internal_mutable_$name$())));\n");
- } else {
- format(
- "DO_((::$proto_ns$::internal::"
- "WireFormatLite::ReadPackedEnumPreserveUnknowns(\n"
- " input,\n"
- " $number$,\n"
- " $type$_IsValid,\n"
- " &unknown_fields_stream,\n"
- " this->_internal_mutable_$name$())));\n");
- }
- } else {
- format(
- "$uint32$ length;\n"
- "DO_(input->ReadVarint32(&length));\n"
- "::$proto_ns$::io::CodedInputStream::Limit limit = "
- "input->PushLimit(static_cast(length));\n"
- "while (input->BytesUntilLimit() > 0) {\n"
- " int value = 0;\n"
- " DO_((::$proto_ns$::internal::WireFormatLite::ReadPrimitive<\n"
- " int, ::$proto_ns$::internal::WireFormatLite::TYPE_ENUM>(\n"
- " input, &value)));\n");
- if (HasPreservingUnknownEnumSemantics(descriptor_)) {
- format(" add_$name$(static_cast< $type$ >(value));\n");
- } else {
- format(
- " if ($type$_IsValid(value)) {\n"
- " _internal_add_$name$(static_cast< $type$ >(value));\n"
- " } else {\n");
- if (UseUnknownFieldSet(descriptor_->file(), options_)) {
- format(
- " mutable_unknown_fields()->AddVarint(\n"
- " $number$, static_cast<$uint64$>(value));\n");
- } else {
- format(
- " unknown_fields_stream.WriteVarint32(tag);\n"
- " unknown_fields_stream.WriteVarint32(\n"
- " static_cast<$uint32$>(value));\n");
- }
- format(" }\n");
- }
- format(
- "}\n"
- "input->PopLimit(limit);\n");
- }
-}
-
void RepeatedEnumFieldGenerator::GenerateSerializeWithCachedSizesToArray(
io::Printer* printer) const {
Formatter format(printer, variables_);
diff --git a/src/google/protobuf/compiler/cpp/cpp_enum_field.h b/src/google/protobuf/compiler/cpp/cpp_enum_field.h
index 793ab2d7933b..e65ec0f5c075 100644
--- a/src/google/protobuf/compiler/cpp/cpp_enum_field.h
+++ b/src/google/protobuf/compiler/cpp/cpp_enum_field.h
@@ -98,8 +98,6 @@ class RepeatedEnumFieldGenerator : public FieldGenerator {
void GenerateSwappingCode(io::Printer* printer) const override;
void GenerateConstructorCode(io::Printer* printer) const override;
void GenerateCopyConstructorCode(io::Printer* printer) const override {}
- void GenerateMergeFromCodedStream(io::Printer* printer) const;
- void GenerateMergeFromCodedStreamWithPacking(io::Printer* printer) const;
void GenerateSerializeWithCachedSizesToArray(
io::Printer* printer) const override;
void GenerateByteSize(io::Printer* printer) const override;
diff --git a/src/google/protobuf/compiler/cpp/cpp_field.cc b/src/google/protobuf/compiler/cpp/cpp_field.cc
index 48ae2903b3e7..a95dd33e91b9 100644
--- a/src/google/protobuf/compiler/cpp/cpp_field.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_field.cc
@@ -340,7 +340,7 @@ FieldGenerator* FieldGeneratorMap::MakeGenerator(
switch (field->cpp_type()) {
case FieldDescriptor::CPPTYPE_MESSAGE:
if (field->is_map()) {
- return new MapFieldGenerator(field, options);
+ return new MapFieldGenerator(field, options, scc_analyzer);
} else {
return new RepeatedMessageFieldGenerator(field, options,
scc_analyzer);
diff --git a/src/google/protobuf/compiler/cpp/cpp_field.h b/src/google/protobuf/compiler/cpp/cpp_field.h
index 185fa8c62626..e0eb679b4c91 100644
--- a/src/google/protobuf/compiler/cpp/cpp_field.h
+++ b/src/google/protobuf/compiler/cpp/cpp_field.h
@@ -181,6 +181,10 @@ class FieldGenerator {
// are placed in the message's ByteSize() method.
virtual void GenerateByteSize(io::Printer* printer) const = 0;
+ // Generates lines to call IsInitialized() for eligible message fields. Non
+ // message fields won't need to override this function.
+ virtual void GenerateIsInitialized(io::Printer* printer) const {}
+
virtual bool IsInlined() const { return false; }
void SetHasBitIndex(int32_t has_bit_index);
diff --git a/src/google/protobuf/compiler/cpp/cpp_helpers.h b/src/google/protobuf/compiler/cpp/cpp_helpers.h
index bae5bf2ffa18..bd4f48bc8ed1 100644
--- a/src/google/protobuf/compiler/cpp/cpp_helpers.h
+++ b/src/google/protobuf/compiler/cpp/cpp_helpers.h
@@ -743,6 +743,27 @@ class PROTOC_EXPORT Formatter {
void Outdent() const { printer_->Outdent(); }
io::Printer* printer() const { return printer_; }
+ class PROTOC_EXPORT ScopedIndenter {
+ public:
+ explicit ScopedIndenter(Formatter* format) : format_(format) {
+ format_->Indent();
+ }
+ ~ScopedIndenter() { format_->Outdent(); }
+
+ private:
+ Formatter* format_;
+ };
+
+ PROTOBUF_NODISCARD ScopedIndenter ScopedIndent() {
+ return ScopedIndenter(this);
+ }
+ template
+ PROTOBUF_NODISCARD ScopedIndenter ScopedIndent(const char* format,
+ const Args&&... args) {
+ (*this)(format, static_cast(args)...);
+ return ScopedIndenter(this);
+ }
+
class PROTOC_EXPORT SaveState {
public:
explicit SaveState(Formatter* format)
diff --git a/src/google/protobuf/compiler/cpp/cpp_map_field.cc b/src/google/protobuf/compiler/cpp/cpp_map_field.cc
index 49f65f3a5dea..130e90ebbe0a 100644
--- a/src/google/protobuf/compiler/cpp/cpp_map_field.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_map_field.cc
@@ -84,8 +84,11 @@ void SetMessageVariables(const FieldDescriptor* descriptor,
}
MapFieldGenerator::MapFieldGenerator(const FieldDescriptor* descriptor,
- const Options& options)
- : FieldGenerator(descriptor, options) {
+ const Options& options,
+ MessageSCCAnalyzer* scc_analyzer)
+ : FieldGenerator(descriptor, options),
+ has_required_fields_(
+ scc_analyzer->HasRequiredFields(descriptor->message_type())) {
SetMessageVariables(descriptor, &variables_, options);
}
@@ -293,6 +296,15 @@ void MapFieldGenerator::GenerateByteSize(io::Printer* printer) const {
"}\n");
}
+void MapFieldGenerator::GenerateIsInitialized(io::Printer* printer) const {
+ if (!has_required_fields_) return;
+
+ Formatter format(printer, variables_);
+ format(
+ "if (!::$proto_ns$::internal::AllAreInitialized($name$_)) return "
+ "false;\n");
+}
+
void MapFieldGenerator::GenerateConstinitInitializer(
io::Printer* printer) const {
Formatter format(printer, variables_);
diff --git a/src/google/protobuf/compiler/cpp/cpp_map_field.h b/src/google/protobuf/compiler/cpp/cpp_map_field.h
index 5ea04290e61b..c01ae498b1aa 100644
--- a/src/google/protobuf/compiler/cpp/cpp_map_field.h
+++ b/src/google/protobuf/compiler/cpp/cpp_map_field.h
@@ -34,6 +34,7 @@
#include