KAFKA-7844: Use regular subproject for generator to fix *All targets (a…

…pache#6182)

The presence of the buildSrc subproject is causing problems when we try
to run installAll, jarAll, and the other "all" targets. It's easier
just to make the generator code a regular subproject and use the
JavaExec gradle task to run the code. This also makes it more
straightforward to run the generator unit tests.

Reviewers: David Arthur <[email protected]>, Ismael Juma <[email protected]>

Co-authored-by: Colin P. Mccabe <[email protected]>
Co-authored-by: Stanislav Kozlovski <[email protected]>

build.gradle

@@ -249,10 +249,11 @@ subprojects {
       exceptionFormat = testExceptionFormat
     }
 
-    useJUnit {
-      excludeCategories 'org.apache.kafka.test.IntegrationTest'
+    if (it.project.name != 'generator') {
+      useJUnit {
+        excludeCategories 'org.apache.kafka.test.IntegrationTest'
+      }
     }
-
   }
 
   jar {
@@ -823,6 +824,23 @@ project(':examples') {
   }
 }
 
+project(':generator') {
+  dependencies {
+    compile libs.jacksonDatabind
+    compile libs.jacksonJDK8Datatypes
+    compile libs.jacksonJaxrsJsonProvider
+    testCompile libs.junit
+  }
+
+  integrationTest {
+    enabled = false
+  }
+
+  javadoc {
+    enabled = false
+  }
+}
+
 project(':clients') {
   archivesBaseName = "kafka-clients"
 
@@ -899,9 +917,12 @@ project(':clients') {
     delete "$buildDir/kafka/"
   }
 
-  task processMessages(type:org.apache.kafka.task.ProcessMessagesTask) {
-    inputDirectory = file("src/main/resources/common/message")
-    outputDirectory = file("src/generated/java/org/apache/kafka/common/message")
+  task processMessages(type:JavaExec) {
+    main = "org.apache.kafka.message.MessageGenerator"
+    classpath = project(':generator').sourceSets.main.runtimeClasspath
+    args = [ "src/generated/java/org/apache/kafka/common/message", "src/main/resources/common/message" ]
+    inputs.dir("src/main/resources/common/message")
+    outputs.dir("src/generated/java/org/apache/kafka/common/message")
   }
 
   sourceSets {
@@ -912,8 +933,7 @@ project(':clients') {
     }
     test {
       java {
-        srcDirs = ["src/generated/java", "src/test/java",
-            "$rootDir/buildSrc/src/main/java/org/apache/kafka/message/"]
+        srcDirs = ["src/generated/java", "src/test/java"]
       }
     }
   }

buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java → generator/src/main/java/org/apache/kafka/message/MessageGenerator.java

@@ -24,14 +24,12 @@
 import com.fasterxml.jackson.databind.SerializationFeature;
 
 import java.io.BufferedWriter;
-import java.io.File;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Locale;
-import java.util.Map;
 
 /**
  * The Kafka message generator.
@@ -94,40 +92,30 @@ public final class MessageGenerator {
         JSON_SERDE.setSerializationInclusion(JsonInclude.Include.NON_EMPTY);
     }
 
-    public static Map<String, File> getOutputFiles(String outputDir, String inputDir) throws Exception {
-        HashMap<String, File> outputFiles = new HashMap<>();
-        for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
-            String jsonName = inputPath.getFileName().toString();
-            String javaName = jsonName.substring(0, jsonName.length() - JSON_SUFFIX.length()) + "Data.java";
-            File outputFile = new File(outputDir, javaName);
-            outputFiles.put(outputFile.toString(), outputFile);
-        }
-        File factoryFile = new File(outputDir, API_MESSAGE_FACTORY_JAVA);
-        outputFiles.put(factoryFile.toString(), factoryFile);
-        return outputFiles;
-    }
-
     public static void processDirectories(String outputDir, String inputDir) throws Exception {
         Files.createDirectories(Paths.get(outputDir));
         int numProcessed = 0;
         ApiMessageFactoryGenerator messageFactoryGenerator = new ApiMessageFactoryGenerator();
         HashSet<String> outputFileNames = new HashSet<>();
-        for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
-            try {
-                MessageSpec spec = JSON_SERDE.
-                    readValue(inputPath.toFile(), MessageSpec.class);
-                String javaName = spec.generatedClassName() + JAVA_SUFFIX;
-                outputFileNames.add(javaName);
-                Path outputPath = Paths.get(outputDir, javaName);
-                try (BufferedWriter writer = Files.newBufferedWriter(outputPath)) {
-                    MessageDataGenerator generator = new MessageDataGenerator();
-                    generator.generate(spec);
-                    generator.write(writer);
+        try (DirectoryStream<Path> directoryStream = Files
+                .newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) {
+            for (Path inputPath : directoryStream) {
+                try {
+                    MessageSpec spec = JSON_SERDE.
+                        readValue(inputPath.toFile(), MessageSpec.class);
+                    String javaName = spec.generatedClassName() + JAVA_SUFFIX;
+                    outputFileNames.add(javaName);
+                    Path outputPath = Paths.get(outputDir, javaName);
+                    try (BufferedWriter writer = Files.newBufferedWriter(outputPath)) {
+                        MessageDataGenerator generator = new MessageDataGenerator();
+                        generator.generate(spec);
+                        generator.write(writer);
+                    }
+                    numProcessed++;
+                    messageFactoryGenerator.registerMessageType(spec);
+                } catch (Exception e) {
+                    throw new RuntimeException("Exception while processing " + inputPath.toString(), e);
                 }
-                numProcessed++;
-                messageFactoryGenerator.registerMessageType(spec);
-            } catch (Exception e) {
-                throw new RuntimeException("Exception while processing " + inputPath.toString(), e);
             }
         }
         Path factoryOutputPath = Paths.get(outputDir, API_MESSAGE_FACTORY_JAVA);
@@ -137,9 +125,15 @@ public static void processDirectories(String outputDir, String inputDir) throws
             messageFactoryGenerator.write(writer);
         }
         numProcessed++;
-        for (Path outputPath : Files.newDirectoryStream(Paths.get(outputDir))) {
-            if (!outputFileNames.contains(outputPath.getFileName().toString())) {
-                Files.delete(outputPath);
+        try (DirectoryStream<Path> directoryStream = Files.
+                newDirectoryStream(Paths.get(outputDir))) {
+            for (Path outputPath : directoryStream) {
+                Path fileName = outputPath.getFileName();
+                if (fileName != null) {
+                    if (!outputFileNames.contains(fileName.toString())) {
+                        Files.delete(outputPath);
+                    }
+                }
             }
         }
         System.out.printf("MessageGenerator: processed %d Kafka message JSON files(s).%n", numProcessed);

settings.gradle

@@ -17,4 +17,4 @@ include 'core', 'examples', 'clients', 'tools', 'streams', 'streams:streams-scal
         'streams:upgrade-system-tests-0100', 'streams:upgrade-system-tests-0101', 'streams:upgrade-system-tests-0102',
         'streams:upgrade-system-tests-0110', 'streams:upgrade-system-tests-10', 'streams:upgrade-system-tests-11', 'streams:upgrade-system-tests-20',
         'streams:upgrade-system-tests-21' , 'log4j-appender', 'connect:api', 'connect:transforms', 'connect:runtime', 'connect:json', 'connect:file',
-        'connect:basic-auth-extension', 'jmh-benchmarks'
+        'connect:basic-auth-extension', 'jmh-benchmarks', 'generator'