#1849: Hash each digest algo in separated threads

iped-engine/src/main/java/iped/engine/core/Manager.java

@@ -25,6 +25,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.StandardOpenOption;
+import java.security.Security;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -47,6 +48,7 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
 
 import iped.data.ICaseData;
 import iped.data.IItem;
@@ -154,7 +156,13 @@ public class Manager {
     private final AtomicBoolean initSleuthkitServers = new AtomicBoolean(false);
 
     private static final String appWinExeFileName = "IPED-SearchApp.exe";
-
+
+    static {
+        if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) {
+            Security.addProvider(new BouncyCastleProvider());
+        }
+    }
+
     public static Manager getInstance() {
         return instance;
     }

iped-engine/src/main/java/iped/engine/task/HashTask.java

@@ -26,8 +26,11 @@
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
 
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -36,7 +39,6 @@
 import iped.engine.config.ConfigurationManager;
 import iped.engine.config.HashTaskConfig;
 import iped.parsers.whatsapp.WhatsAppParser;
-import iped.utils.IOUtil;
 
 /**
  * Classe para calcular e manipular hashes.
@@ -45,6 +47,10 @@ public class HashTask extends AbstractTask {
 
     private static Logger LOGGER = LoggerFactory.getLogger(HashTask.class);
 
+    private static final int HASH_BUFFER_LEN = 1024 * 1024;
+
+    private static final ExecutorService executorService = Executors.newCachedThreadPool();
+
     public enum HASH {
         MD5("md5"), //$NON-NLS-1$
         SHA1("sha-1"), //$NON-NLS-1$
@@ -87,7 +93,7 @@ public void init(ConfigurationManager configurationManager) throws Exception {
             if (!algorithm.equalsIgnoreCase(HASH.EDONKEY.toString())) {
                 digest = MessageDigest.getInstance(algorithm.toUpperCase());
             } else {
-                digest = MessageDigest.getInstance("MD4", new BouncyCastleProvider()); //$NON-NLS-1$
+                digest = MessageDigest.getInstance("MD4"); //$NON-NLS-1$
             }
             digestMap.put(algorithm, digest);
             if (HASH.SHA256.toString().equals(algorithm)) {
@@ -99,8 +105,9 @@ public void init(ConfigurationManager configurationManager) throws Exception {
 
     @Override
     public void finish() throws Exception {
-        // TODO Auto-generated method stub
-
+        if (!executorService.isShutdown()) {
+            executorService.shutdown();
+        }
     }
 
     public void process(IItem evidence) {
@@ -119,19 +126,54 @@ public void process(IItem evidence) {
             return;
         }
 
-        InputStream in = null;
-        try {
-            in = evidence.getBufferedInputStream();
-            byte[] buf = new byte[1024 * 1024];
+        try (InputStream in = evidence.getBufferedInputStream()) {
+
+            byte[] readBuf = new byte[HASH_BUFFER_LEN];
+            byte[] hashBuf = new byte[HASH_BUFFER_LEN];
+            byte[] tempBuf = null;
             int len;
-            while ((len = in.read(buf)) >= 0 && !Thread.currentThread().isInterrupted()) {
+
+            AtomicReference<CountDownLatch> countDown = new AtomicReference<>(null);
+            AtomicReference<Exception> ex = new AtomicReference<Exception>(null);
+
+            while ((len = in.read(readBuf)) >= 0 && !Thread.currentThread().isInterrupted()) {
+
+                if (countDown.get() != null) {
+                    countDown.get().await();
+                }
+
+                countDown.set(new CountDownLatch(digestMap.size()));
+
+                // swap hashBuf <-> readBuf
+                tempBuf = hashBuf;
+                hashBuf = readBuf;
+                readBuf = tempBuf;
+
+                final int currLen = len;
+                final byte[] currHashBuf = hashBuf;
                 for (String algo : digestMap.keySet()) {
-                    if (!algo.equals(HASH.EDONKEY.toString())) {
-                        digestMap.get(algo).update(buf, 0, len);
-                    } else {
-                        updateEd2k(buf, len);
-                    }
+                    executorService.execute(() -> {
+                        try {
+                            if (!algo.equals(HASH.EDONKEY.toString())) {
+                                digestMap.get(algo).update(currHashBuf, 0, currLen);
+                            } else {
+                                updateEd2k(currHashBuf, currLen);
+                            }
+                        } catch (Exception e) {
+                            ex.set(e);
+                        } finally {
+                            countDown.get().countDown();
+                        }
+                    });
                 }
+
+                if (ex.get() != null) {
+                    throw ex.get();
+                }
+            }
+
+            if (countDown.get() != null) {
+                countDown.get().await();
             }
 
             boolean defaultHash = true;
@@ -161,8 +203,6 @@ public void process(IItem evidence) {
                     e.toString());
             // e.printStackTrace();
 
-        } finally {
-            IOUtil.closeQuietly(in);
         }
 
     }

iped-engine/src/main/java/iped/engine/task/regex/validator/cripto/EthereumAddressValidatorService.java

@@ -7,7 +7,6 @@
 import java.util.List;
 
 import org.bouncycastle.jcajce.provider.digest.Keccak;
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
 
 import iped.engine.task.regex.BasicAbstractRegexValidatorService;
 
@@ -23,7 +22,6 @@ public class EthereumAddressValidatorService extends BasicAbstractRegexValidator
     private static final int[] MASKS = { 128, 8 };
 
     static {
-        Security.addProvider(new BouncyCastleProvider());
         digest = new Keccak.Digest256();
     }