Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[7.1.0] Use a larger buffer size for java.util.zip.*Stream classes #20642

Merged
merged 1 commit into from
Jan 11, 2024
Merged

[7.1.0] Use a larger buffer size for java.util.zip.*Stream classes #20642

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
* <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP format</a>
*/
public class ZipCombiner implements AutoCloseable {
private static final int INFLATER_BUFFER_BYTES = 8192;
public static final Date DOS_EPOCH = new Date(ZipUtil.DOS_EPOCH);
/**
* Whether to compress or decompress entries.
Expand Down Expand Up @@ -440,7 +441,7 @@ public void addZip(File zipFile) throws IOException {
entries.put(filename, null);
InputStream in = zip.getRawInputStream(entry);
if (entry.getMethod() == Compression.DEFLATED) {
in = new InflaterInputStream(in, getInflater());
in = new InflaterInputStream(in, getInflater(), INFLATER_BUFFER_BYTES);
}
action.getStrategy().merge(in, action.getMergeBuffer());
break;
Expand Down Expand Up @@ -492,7 +493,9 @@ private void writeEntryFromBuffer(ZipFileEntry entry, byte[] uncompressed) throw
writeEntry(entry, new ByteArrayInputStream(uncompressed));
} else {
ByteArrayOutputStream compressed = new ByteArrayOutputStream();
copyStream(new DeflaterInputStream(new ByteArrayInputStream(uncompressed), getDeflater()),
copyStream(
new DeflaterInputStream(
new ByteArrayInputStream(uncompressed), getDeflater(), INFLATER_BUFFER_BYTES),
compressed);
entry.setMethod(Compression.DEFLATED);
entry.setCompressedSize(compressed.size());
Expand Down Expand Up @@ -529,14 +532,19 @@ private void writeEntry(ZipReader zip, ZipFileEntry entry, EntryAction action)
// from the raw file data and deflate to a temporary byte array to determine the deflated
// size. Then use this byte array as the input stream for writing the entry.
ByteArrayOutputStream tmp = new ByteArrayOutputStream();
copyStream(new DeflaterInputStream(zip.getRawInputStream(entry), getDeflater()), tmp);
copyStream(
new DeflaterInputStream(
zip.getRawInputStream(entry), getDeflater(), INFLATER_BUFFER_BYTES),
tmp);
data = new ByteArrayInputStream(tmp.toByteArray());
outEntry.setMethod(Compression.DEFLATED);
outEntry.setCompressedSize(tmp.size());
} else if (mode == OutputMode.FORCE_STORED && entry.getMethod() != Compression.STORED) {
// The output mode is stored, but the entry compression is not; create an inflater stream
// from the raw file data.
data = new InflaterInputStream(zip.getRawInputStream(entry), getInflater());
// from the raw file data.
data =
new InflaterInputStream(
zip.getRawInputStream(entry), getInflater(), INFLATER_BUFFER_BYTES);
outEntry.setMethod(Compression.STORED);
outEntry.setCompressedSize(entry.getSize());
} else {
Expand Down
4 changes: 2 additions & 2 deletions src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipEntryInputStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
package com.google.devtools.build.zip;

import com.google.devtools.build.zip.ZipFileEntry.Compression;

import java.io.IOException;
import java.io.InputStream;
import java.util.zip.Inflater;
Expand All @@ -24,6 +23,7 @@

/** An input stream for reading the file data of a ZIP file entry. */
class ZipEntryInputStream extends InputStream {
private static final int INFLATER_BUFFER_BYTES = 8192;
private InputStream stream;
private long rem;

Expand Down Expand Up @@ -61,7 +61,7 @@ class ZipEntryInputStream extends InputStream {
rem = zipEntry.getSize();
}
if (!raw && zipEntry.getMethod() == Compression.DEFLATED) {
stream = new InflaterInputStream(stream, new Inflater(true));
stream = new InflaterInputStream(stream, new Inflater(true), INFLATER_BUFFER_BYTES);
}
}

Expand Down
7 changes: 4 additions & 3 deletions src/main/java/com/google/devtools/build/lib/analysis/actions/FileWriteAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ protected void computeKey(

private static final class CompressedFileWriteAction extends FileWriteAction {
private static final String GUID = "5bfba914-2251-11ee-be56-0242ac120002";
private static final int GZIP_BYTES_BUFFER = 8192;

private final byte[] compressedBytes;
private final int uncompressedSize;
Expand All @@ -252,7 +253,7 @@ private static final class CompressedFileWriteAction extends FileWriteAction {
// Presize on the small end to avoid over-allocating memory.
ByteArrayOutputStream byteStream = new ByteArrayOutputStream(dataToCompress.length / 100);

try (GZIPOutputStream zipStream = new GZIPOutputStream(byteStream)) {
try (GZIPOutputStream zipStream = new GZIPOutputStream(byteStream, GZIP_BYTES_BUFFER)) {
zipStream.write(dataToCompress);
} catch (IOException e) {
// This should be impossible since we're writing to a byte array.
Expand All @@ -268,7 +269,7 @@ private static final class CompressedFileWriteAction extends FileWriteAction {
public String getFileContents() {
byte[] uncompressedBytes = new byte[uncompressedSize];
try (GZIPInputStream zipStream =
new GZIPInputStream(new ByteArrayInputStream(compressedBytes))) {
new GZIPInputStream(new ByteArrayInputStream(compressedBytes), GZIP_BYTES_BUFFER)) {
int read;
int totalRead = 0;
while (totalRead < uncompressedSize
Expand All @@ -293,7 +294,7 @@ public String getFileContents() {
public DeterministicWriter newDeterministicWriter(ActionExecutionContext ctx) {
return out -> {
try (GZIPInputStream gzipIn =
new GZIPInputStream(new ByteArrayInputStream(compressedBytes))) {
new GZIPInputStream(new ByteArrayInputStream(compressedBytes), GZIP_BYTES_BUFFER)) {
ByteStreams.copy(gzipIn, out);
}
};
Expand Down
6 changes: 4 additions & 2 deletions src/main/java/com/google/devtools/build/lib/rules/genquery/GenQueryOutputStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ class GenQueryOutputStream extends OutputStream {
*/
private static final int COMPRESSION_THRESHOLD = 1 << 20;

private static final int GZIP_BYTES_BUFFER = 8192;

/**
* Encapsulates the output of a {@link GenQuery}'s query. CPU and memory overhead of individual
* methods depends on the underlying content and settings.
Expand Down Expand Up @@ -83,7 +85,7 @@ interface GenQueryResult {
GenQueryOutputStream(boolean compressedOutputRequested) throws IOException {
this.compressedOutputRequested = compressedOutputRequested;
if (compressedOutputRequested) {
this.out = new GZIPOutputStream(bytesOut);
this.out = new GZIPOutputStream(bytesOut, GZIP_BYTES_BUFFER);
this.outputWasCompressed = true;
} else {
this.out = bytesOut;
Expand Down Expand Up @@ -138,7 +140,7 @@ private void maybeStartCompression(int additionalBytes) throws IOException {
}

ByteString.Output compressedBytesOut = ByteString.newOutput();
GZIPOutputStream gzipOut = new GZIPOutputStream(compressedBytesOut);
GZIPOutputStream gzipOut = new GZIPOutputStream(compressedBytesOut, GZIP_BYTES_BUFFER);
bytesOut.writeTo(gzipOut);
bytesOut = compressedBytesOut;
out = gzipOut;
Expand Down
Loading