Change checksum algorithm to CRC32C. And localize checksum handling t…

…o one file. (#506)

datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/ChecksumEnforcingInputStream.java

@@ -19,25 +19,22 @@
 import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.io.InputStream;
-import java.security.MessageDigest;
 
 /** This class provides End-to-End Checksum API for http protocol. */
 class ChecksumEnforcingInputStream extends InputStream {
   private final InputStream delegate;
-  private final MessageDigest messageDigest;
+  private final EndToEndChecksumHandler endToEndChecksumHandler;
   private final String expectedChecksum;
 
-  ChecksumEnforcingInputStream(
-      InputStream originalInputStream, HttpResponse response, MessageDigest digest) {
-    this(originalInputStream, EndToEndChecksumHandler.getChecksumHeader(response), digest);
+  ChecksumEnforcingInputStream(InputStream originalInputStream, HttpResponse response) {
+    this(originalInputStream, EndToEndChecksumHandler.getChecksumHeader(response));
   }
 
   @VisibleForTesting
-  ChecksumEnforcingInputStream(
-      InputStream originalInputStream, String checksum, MessageDigest digest) {
+  ChecksumEnforcingInputStream(InputStream originalInputStream, String checksum) {
     delegate = originalInputStream;
     expectedChecksum = checksum;
-    messageDigest = digest;
+    endToEndChecksumHandler = new EndToEndChecksumHandler();
   }
 
   @Override
@@ -76,11 +73,10 @@ public int read(byte[] b, int off, int len) throws IOException {
     if (len <= 0) return 0;
     int i = delegate.read(b, off, len);
     if (i > 0) {
-      messageDigest.update(b, off, i);
+      endToEndChecksumHandler.update(b, off, i);
     } else {
       // no more payload to read. compute checksum and verify
-      if (!expectedChecksum.equalsIgnoreCase(
-          com.google.common.io.BaseEncoding.base16().encode(messageDigest.digest()))) {
+      if (!expectedChecksum.equalsIgnoreCase(endToEndChecksumHandler.hash())) {
         throw new IOException("possible memory corruption on payload detected");
       }
     }

datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/EndToEndChecksumHandler.java

@@ -16,31 +16,35 @@
 package com.google.datastore.v1.client;
 
 import com.google.api.client.http.HttpResponse;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
 
 /** This class provides End-to-End Checksum API for http protocol. */
 class EndToEndChecksumHandler {
   /** The checksum http header on http requests */
   static final String HTTP_REQUEST_CHECKSUM_HEADER = "x-request-checksum-348659783";
   /** The checksum http header on http responses */
   static final String HTTP_RESPONSE_CHECKSUM_HEADER = "x-response-checksum-348659783";
-  /** Algorithm used for checksum */
-  private static final String MD5 = "MD5";
+
+  final Hasher hasher = EndToEndChecksumHandler.getNewCrc32cHasher();
 
   /**
    * Create and return checksum as a string value for the input 'bytes'.
    *
    * @param bytes raw message for which the checksum is being computed
    * @return computed checksum as a hex string
-   * @throws RuntimeException if MD5 Algorithm is not found in the VM
    */
   static String computeChecksum(byte[] bytes) {
     if (bytes == null || (bytes.length == 0)) {
       return null;
     }
-    return com.google.common.io.BaseEncoding.base16()
-        .encode(getMessageDigestInstance().digest(bytes));
+    HashCode hc = getNewCrc32cHasher().putBytes(bytes).hash();
+    return hc.toString();
+  }
+
+  private static Hasher getNewCrc32cHasher() {
+    return Hashing.crc32c().newHasher();
   }
 
   /**
@@ -58,14 +62,6 @@ static boolean validateChecksum(String checksum, byte[] bytes) {
         && checksum.equalsIgnoreCase(computeChecksum(bytes));
   }
 
-  static MessageDigest getMessageDigestInstance() {
-    try {
-      return MessageDigest.getInstance(MD5);
-    } catch (NoSuchAlgorithmException e) {
-      throw new RuntimeException("MD5 algorithm is not found when computing checksum!");
-    }
-  }
-
   static boolean hasChecksumHeader(HttpResponse response) {
     String checksum = getChecksumHeader(response);
     return checksum != null && !checksum.isEmpty();
@@ -74,4 +70,12 @@ static boolean hasChecksumHeader(HttpResponse response) {
   static String getChecksumHeader(HttpResponse response) {
     return response.getHeaders().getFirstHeaderStringValue(HTTP_RESPONSE_CHECKSUM_HEADER);
   }
+
+  void update(byte[] bytes, int off, int len) {
+    hasher.putBytes(bytes, off, len);
+  }
+
+  String hash() {
+    return hasher.hash().toString();
+  }
 }

datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/RemoteRpc.java

@@ -109,8 +109,7 @@ public InputStream call(String methodName, MessageLite request) throws Datastore
         }
         InputStream inputStream = httpResponse.getContent();
         return enableE2EChecksum && EndToEndChecksumHandler.hasChecksumHeader(httpResponse)
-            ? new ChecksumEnforcingInputStream(
-                inputStream, httpResponse, EndToEndChecksumHandler.getMessageDigestInstance())
+            ? new ChecksumEnforcingInputStream(inputStream, httpResponse)
             : inputStream;
       } catch (SocketTimeoutException e) {
         throw makeException(url, methodName, Code.DEADLINE_EXCEEDED, "Deadline exceeded", e);

datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/ChecksumEnforcingInputStreamTest.java

@@ -21,16 +21,13 @@
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
-import java.security.MessageDigest;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /** Test for {@link ChecksumEnforcingInputStream}. */
 @RunWith(JUnit4.class)
 public class ChecksumEnforcingInputStreamTest {
-  private final MessageDigest digest = EndToEndChecksumHandler.getMessageDigestInstance();
-
   public void test(int payloadSize) throws Exception {
     // read 1000 bytes at a time
     // Since checksum should be correct, do not expect IOException
@@ -40,7 +37,7 @@ public void test(int payloadSize) throws Exception {
         // do nothing with the bytes read
       }
     } catch (IOException e) {
-      fail("checksum verification failed!");
+      fail("checksum verification failed! " + e.getMessage());
     }
   }
 
@@ -66,9 +63,7 @@ public void read_withInvalidChecksum() {
     // Since checksum should be correct, do not expect IOException
     try (ChecksumEnforcingInputStream instance =
         new ChecksumEnforcingInputStream(
-            new ByteArrayInputStream("hello there".getBytes(UTF_8)),
-            "this checksum is invalid",
-            digest)) {
+            new ByteArrayInputStream("hello there".getBytes(UTF_8)), "this checksum is invalid")) {
       byte[] buf = new byte[1000];
       while (instance.read(buf, 0, 1000) != -1) {
         // do nothing with the bytes read
@@ -103,7 +98,6 @@ private ChecksumEnforcingInputStream setUpData(int payloadSize) throws Exception
     }
     byte[] bytes = payload.getBytes(UTF_8);
     String expectedChecksum = EndToEndChecksumHandler.computeChecksum(bytes);
-    return new ChecksumEnforcingInputStream(
-        new ByteArrayInputStream(bytes), expectedChecksum, digest);
+    return new ChecksumEnforcingInputStream(new ByteArrayInputStream(bytes), expectedChecksum);
   }
 }

datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/RemoteRpcTest.java

@@ -174,7 +174,7 @@ public void testHttpHeaders_expectE2eChecksumHeader() throws IOException {
         httpRequest
             .getHeaders()
             .getFirstHeaderStringValue(EndToEndChecksumHandler.HTTP_REQUEST_CHECKSUM_HEADER);
-    assertEquals(32, header.length());
+    assertEquals(8, header.length());
   }
 
   @Test