Feature/parallelize file parsing (#28)

* Parallelize the file formatting * Formatter is no longeur a private variable * Improve parallelism usage based on https://stackoverflow.com/questions/33596618/how-can-i-get-a-parallel-stream-of-files-walk/33597291#comment54977780_33597291
spotify · Mar 19, 2018 · 97044ef · 97044ef
1 parent 0f0dd43
commit 97044ef
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 24 deletions.
diff --git a/pom.xml b/pom.xml
@@ -35,6 +35,11 @@
             <organization>Coveo</organization>
             <organizationUrl>http://github.com/coveo</organizationUrl>
         </developer>
+        <developer>
+            <name>Andy Emond</name>
+            <organization>Coveo</organization>
+            <organizationUrl>http://github.com/coveo</organizationUrl>
+        </developer>
     </developers>
 
     <scm>
@@ -148,6 +153,14 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

diff --git a/src/main/java/com/coveo/AbstractFMT.java b/src/main/java/com/coveo/AbstractFMT.java
@@ -2,15 +2,20 @@
 
 import com.google.common.base.Charsets;
 import com.google.common.io.CharSource;
-import com.google.common.io.Files;
 import com.google.googlejavaformat.java.*;
 import com.google.googlejavaformat.java.RemoveUnusedImports.JavadocOnlyImports;
+
 import java.io.File;
 import java.io.FileFilter;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
 import org.apache.maven.plugin.AbstractMojo;
 import org.apache.maven.plugin.MojoExecutionException;
 import org.apache.maven.plugin.MojoFailureException;
@@ -19,8 +24,6 @@
 
 public abstract class AbstractFMT extends AbstractMojo {
   private Log logger = getLog();
-  /** Lazily initialized based on style. Use formatter() for access. */
-  private Formatter formatter;
 
   @Parameter(
     defaultValue = "${project.build.sourceDirectory}",
@@ -87,8 +90,10 @@ public void execute() throws MojoExecutionException, MojoFailureException {
       }
     }
 
+    Formatter formatter = getFormatter();
+
     for (File directoryToFormat : directoriesToFormat) {
-      formatSourceFilesInDirectory(directoryToFormat);
+      formatSourceFilesInDirectory(directoryToFormat, formatter);
     }
 
     logNumberOfFilesProcessed();
@@ -115,19 +120,23 @@ public List<String> getFilesProcessed() {
     return filesProcessed;
   }
 
-  private void formatSourceFilesInDirectory(File directory) throws MojoFailureException {
+  public void formatSourceFilesInDirectory(File directory, Formatter formatter)
+      throws MojoFailureException {
     if (!directory.isDirectory()) {
       logger.info("Directory '" + directory + "' is not a directory. Skipping.");
       return;
     }
 
-    List<File> files = Arrays.asList(directory.listFiles(getFileFilter()));
-    for (File file : files) {
-      if (file.isDirectory()) {
-        formatSourceFilesInDirectory(file);
-      } else {
-        formatSourceFile(file);
-      }
+    try (Stream<Path> paths = Files.walk(Paths.get(directory.getPath()))) {
+      paths
+          .collect(Collectors.toList())
+          .parallelStream()
+          .filter(Files::isRegularFile)
+          .map(Path::toFile)
+          .filter((file) -> getFileFilter().accept(file))
+          .forEach(file -> formatSourceFile(file, formatter));
+    } catch (IOException exception) {
+      throw new MojoFailureException(exception.getMessage());
     }
   }
 
@@ -145,11 +154,8 @@ private JavaFormatterOptions.Style style() throws MojoFailureException {
     throw new MojoFailureException(message);
   }
 
-  private synchronized Formatter formatter() throws MojoFailureException {
-    if (formatter == null) {
-      formatter = new Formatter(JavaFormatterOptions.builder().style(style()).build());
-    }
-    return formatter;
+  private Formatter getFormatter() throws MojoFailureException {
+    return new Formatter(JavaFormatterOptions.builder().style(style()).build());
   }
 
   private FileFilter getFileFilter() {
@@ -164,7 +170,7 @@ public boolean accept(File pathname) {
     };
   }
 
-  private void formatSourceFile(File file) throws MojoFailureException {
+  private void formatSourceFile(File file, Formatter formatter) {
     if (file.isDirectory()) {
       logger.info("File '" + file + "' is a directory. Skipping.");
       return;
@@ -174,10 +180,10 @@ private void formatSourceFile(File file) throws MojoFailureException {
       logger.debug("Formatting '" + file + "'.");
     }
 
-    CharSource source = Files.asCharSource(file, Charsets.UTF_8);
+    CharSource source = com.google.common.io.Files.asCharSource(file, Charsets.UTF_8);
     try {
       String input = source.read();
-      String formatted = formatter().formatSource(input);
+      String formatted = formatter.formatSource(input);
       formatted = RemoveUnusedImports.removeUnusedImports(formatted, JavadocOnlyImports.KEEP);
       formatted = ImportOrderer.reorderImports(formatted);
       if (!input.equals(formatted)) {
@@ -188,9 +194,7 @@ private void formatSourceFile(File file) throws MojoFailureException {
       if (filesProcessed.size() % 100 == 0) {
         logNumberOfFilesProcessed();
       }
-    } catch (FormatterException e) {
-      logger.warn("Failed to format file '" + file + "'.", e);
-    } catch (IOException e) {
+    } catch (FormatterException | IOException e) {
       logger.warn("Failed to format file '" + file + "'.", e);
     }
   }