From 8eff3ed0720643e4fec186d2ad40914acf4e8d5d Mon Sep 17 00:00:00 2001 From: Google Java Core Libraries Date: Wed, 15 Dec 2021 15:49:06 -0800 Subject: [PATCH] Start with a smaller but faster growing buffer when reading to a byte array RELNOTES=n/a PiperOrigin-RevId: 416664799 --- .../src/com/google/common/io/ByteStreams.java | 32 +++++++++++++------ .../src/com/google/common/io/ByteStreams.java | 32 +++++++++++++------ 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/android/guava/src/com/google/common/io/ByteStreams.java b/android/guava/src/com/google/common/io/ByteStreams.java index 388188030861..65b4b671d89f 100644 --- a/android/guava/src/com/google/common/io/ByteStreams.java +++ b/android/guava/src/com/google/common/io/ByteStreams.java @@ -18,6 +18,8 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkPositionIndex; import static com.google.common.base.Preconditions.checkPositionIndexes; +import static java.lang.Math.max; +import static java.lang.Math.min; import com.google.common.annotations.Beta; import com.google.common.annotations.GwtIncompatible; @@ -170,13 +172,18 @@ public static long copy(ReadableByteChannel from, WritableByteChannel to) throws */ private static byte[] toByteArrayInternal(InputStream in, Queue bufs, int totalLen) throws IOException { - // Starting with an 8k buffer, double the size of each successive buffer. Buffers are retained - // in a deque so that there's no copying between buffers while reading and so all of the bytes - // in each new allocated buffer are available for reading from the stream. - for (int bufSize = BUFFER_SIZE; + // Roughly size to match what has been read already. Some file systems, such as procfs, return 0 + // as their length. These files are very small, so it's wasteful to allocate an 8KB buffer. + int initialBufferSize = min(BUFFER_SIZE, max(128, Integer.highestOneBit(totalLen) * 2)); + // Starting with an 8k buffer, double the size of each successive buffer. Smaller buffers + // quadruple in size until they reach 8k, to minimize the number of small reads for longer + // streams. Buffers are retained in a deque so that there's no copying between buffers while + // reading and so all of the bytes in each new allocated buffer are available for reading from + // the stream. + for (int bufSize = initialBufferSize; totalLen < MAX_ARRAY_LEN; - bufSize = IntMath.saturatedMultiply(bufSize, 2)) { - byte[] buf = new byte[Math.min(bufSize, MAX_ARRAY_LEN - totalLen)]; + bufSize = IntMath.saturatedMultiply(bufSize, bufSize < 4096 ? 4 : 2)) { + byte[] buf = new byte[min(bufSize, MAX_ARRAY_LEN - totalLen)]; bufs.add(buf); int off = 0; while (off < buf.length) { @@ -200,11 +207,18 @@ private static byte[] toByteArrayInternal(InputStream in, Queue bufs, in } private static byte[] combineBuffers(Queue bufs, int totalLen) { - byte[] result = new byte[totalLen]; - int remaining = totalLen; + if (bufs.isEmpty()) { + return new byte[0]; + } + byte[] result = bufs.remove(); + if (result.length == totalLen) { + return result; + } + int remaining = totalLen - result.length; + result = Arrays.copyOf(result, totalLen); while (remaining > 0) { byte[] buf = bufs.remove(); - int bytesToCopy = Math.min(remaining, buf.length); + int bytesToCopy = min(remaining, buf.length); int resultOffset = totalLen - remaining; System.arraycopy(buf, 0, result, resultOffset, bytesToCopy); remaining -= bytesToCopy; diff --git a/guava/src/com/google/common/io/ByteStreams.java b/guava/src/com/google/common/io/ByteStreams.java index 388188030861..65b4b671d89f 100644 --- a/guava/src/com/google/common/io/ByteStreams.java +++ b/guava/src/com/google/common/io/ByteStreams.java @@ -18,6 +18,8 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkPositionIndex; import static com.google.common.base.Preconditions.checkPositionIndexes; +import static java.lang.Math.max; +import static java.lang.Math.min; import com.google.common.annotations.Beta; import com.google.common.annotations.GwtIncompatible; @@ -170,13 +172,18 @@ public static long copy(ReadableByteChannel from, WritableByteChannel to) throws */ private static byte[] toByteArrayInternal(InputStream in, Queue bufs, int totalLen) throws IOException { - // Starting with an 8k buffer, double the size of each successive buffer. Buffers are retained - // in a deque so that there's no copying between buffers while reading and so all of the bytes - // in each new allocated buffer are available for reading from the stream. - for (int bufSize = BUFFER_SIZE; + // Roughly size to match what has been read already. Some file systems, such as procfs, return 0 + // as their length. These files are very small, so it's wasteful to allocate an 8KB buffer. + int initialBufferSize = min(BUFFER_SIZE, max(128, Integer.highestOneBit(totalLen) * 2)); + // Starting with an 8k buffer, double the size of each successive buffer. Smaller buffers + // quadruple in size until they reach 8k, to minimize the number of small reads for longer + // streams. Buffers are retained in a deque so that there's no copying between buffers while + // reading and so all of the bytes in each new allocated buffer are available for reading from + // the stream. + for (int bufSize = initialBufferSize; totalLen < MAX_ARRAY_LEN; - bufSize = IntMath.saturatedMultiply(bufSize, 2)) { - byte[] buf = new byte[Math.min(bufSize, MAX_ARRAY_LEN - totalLen)]; + bufSize = IntMath.saturatedMultiply(bufSize, bufSize < 4096 ? 4 : 2)) { + byte[] buf = new byte[min(bufSize, MAX_ARRAY_LEN - totalLen)]; bufs.add(buf); int off = 0; while (off < buf.length) { @@ -200,11 +207,18 @@ private static byte[] toByteArrayInternal(InputStream in, Queue bufs, in } private static byte[] combineBuffers(Queue bufs, int totalLen) { - byte[] result = new byte[totalLen]; - int remaining = totalLen; + if (bufs.isEmpty()) { + return new byte[0]; + } + byte[] result = bufs.remove(); + if (result.length == totalLen) { + return result; + } + int remaining = totalLen - result.length; + result = Arrays.copyOf(result, totalLen); while (remaining > 0) { byte[] buf = bufs.remove(); - int bytesToCopy = Math.min(remaining, buf.length); + int bytesToCopy = min(remaining, buf.length); int resultOffset = totalLen - remaining; System.arraycopy(buf, 0, result, resultOffset, bytesToCopy); remaining -= bytesToCopy;