Allow storage.reader to read gzip blobs in compressed chunks (#1301)

googleapis · Oct 2, 2016 · 88b8b4a · 88b8b4a
1 parent dcee473
commit 88b8b4a
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 1 deletion.
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/DefaultStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/DefaultStorageRpc.java
@@ -51,9 +51,11 @@
 import com.google.api.client.http.HttpResponseException;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.InputStreamContent;
+import com.google.api.client.http.LowLevelHttpResponse;
 import com.google.api.client.http.json.JsonHttpContent;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson.JacksonFactory;
+import com.google.api.client.util.IOUtils;
 import com.google.api.services.storage.Storage;
 import com.google.api.services.storage.Storage.Objects.Get;
 import com.google.api.services.storage.Storage.Objects.Insert;
@@ -65,6 +67,7 @@
 import com.google.api.services.storage.model.ObjectAccessControl;
 import com.google.api.services.storage.model.Objects;
 import com.google.api.services.storage.model.StorageObject;
+import com.google.cloud.BaseServiceException;
 import com.google.cloud.storage.StorageException;
 import com.google.cloud.storage.StorageOptions;
 import com.google.common.base.Function;
@@ -78,6 +81,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.lang.reflect.Field;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.LinkedList;
@@ -500,7 +504,24 @@ public Tuple<String, byte[]> read(StorageObject from, Map<Option, ?> options, lo
       requestHeaders.setRange(range.toString());
       setEncryptionHeaders(requestHeaders, ENCRYPTION_KEY_PREFIX, options);
       ByteArrayOutputStream output = new ByteArrayOutputStream(bytes);
-      req.executeMedia().download(output);
+      HttpResponse httpResponse = req.executeMedia();
+      // todo(mziccard) remove when
+      // https://github.com/GoogleCloudPlatform/google-cloud-java/issues/982 is fixed
+      String contentEncoding = httpResponse.getContentEncoding();
+      if (contentEncoding != null && contentEncoding.contains("gzip")) {
+        try {
+          Field responseField = httpResponse.getClass().getDeclaredField("response");
+          responseField.setAccessible(true);
+          LowLevelHttpResponse lowLevelHttpResponse =
+              (LowLevelHttpResponse) responseField.get(httpResponse);
+          IOUtils.copy(lowLevelHttpResponse.getContent(), output);
+        } catch (IllegalAccessException|NoSuchFieldException ex) {
+          throw new StorageException(
+              BaseServiceException.UNKNOWN_CODE, "Error parsing gzip response", ex);
+        }
+      } else {
+        httpResponse.download(output);
+      }
       String etag = req.getLastResponseHeaders().getETag();
       return Tuple.of(etag, output.toByteArray());
     } catch (IOException ex) {

diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java
@@ -53,6 +53,7 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.google.common.io.BaseEncoding;
+import com.google.common.io.ByteStreams;
 
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -79,6 +80,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.zip.GZIPInputStream;
 
 import javax.crypto.spec.SecretKeySpec;
 
@@ -96,6 +98,8 @@ public class ITStorageTest {
   private static final String OTHER_BASE64_KEY = "IcOIQGlliNr5pr3vJb63l+XMqc7NjXqjfw/deBoNxPA=";
   private static final Key KEY =
       new SecretKeySpec(BaseEncoding.base64().decode(BASE64_KEY), "AES256");
+  private static final byte[] COMPRESSED_CONTENT = BaseEncoding.base64()
+      .decode("H4sIAAAAAAAAAPNIzcnJV3DPz0/PSVVwzskvTVEILskvSkxPVQQA/LySchsAAAA=");
 
   @BeforeClass
   public static void beforeClass() throws NoSuchAlgorithmException, InvalidKeySpecException {
@@ -1381,4 +1385,33 @@ public void testBlobAcl() {
       // expected
     }
   }
+
+  @Test
+  public void testReadCompressedBlob() throws IOException {
+    String blobName = "test-read-compressed-blob";
+    BlobInfo blobInfo = BlobInfo.builder(BlobId.of(BUCKET, blobName))
+        .contentType("text/plain")
+        .contentEncoding("gzip")
+        .build();
+    Blob blob = storage.create(blobInfo, COMPRESSED_CONTENT);
+    try (ByteArrayOutputStream output = new ByteArrayOutputStream()) {
+      try (ReadChannel reader = storage.reader(BlobId.of(BUCKET, blobName))) {
+        reader.chunkSize(8);
+        ByteBuffer buffer = ByteBuffer.allocate(8);
+        while (reader.read(buffer) != -1) {
+          buffer.flip();
+          output.write(buffer.array(), 0, buffer.limit());
+          buffer.clear();
+        }
+      }
+      assertArrayEquals(BLOB_STRING_CONTENT.getBytes(UTF_8),
+          storage.readAllBytes(BUCKET, blobName));
+      assertArrayEquals(COMPRESSED_CONTENT, output.toByteArray());
+      try (GZIPInputStream zipInput =
+               new GZIPInputStream(new ByteArrayInputStream(output.toByteArray()))) {
+        assertArrayEquals(BLOB_STRING_CONTENT.getBytes(UTF_8), ByteStreams.toByteArray(zipInput));
+      }
+    }
+    blob.delete();
+  }
 }