diff --git a/build.gradle b/build.gradle index 49955ca7c5c9d..58d554f968a53 100644 --- a/build.gradle +++ b/build.gradle @@ -249,10 +249,11 @@ subprojects { exceptionFormat = testExceptionFormat } - useJUnit { - excludeCategories 'org.apache.kafka.test.IntegrationTest' + if (it.project.name != 'generator') { + useJUnit { + excludeCategories 'org.apache.kafka.test.IntegrationTest' + } } - } jar { @@ -823,6 +824,23 @@ project(':examples') { } } +project(':generator') { + dependencies { + compile libs.jacksonDatabind + compile libs.jacksonJDK8Datatypes + compile libs.jacksonJaxrsJsonProvider + testCompile libs.junit + } + + integrationTest { + enabled = false + } + + javadoc { + enabled = false + } +} + project(':clients') { archivesBaseName = "kafka-clients" @@ -899,9 +917,12 @@ project(':clients') { delete "$buildDir/kafka/" } - task processMessages(type:org.apache.kafka.task.ProcessMessagesTask) { - inputDirectory = file("src/main/resources/common/message") - outputDirectory = file("src/generated/java/org/apache/kafka/common/message") + task processMessages(type:JavaExec) { + main = "org.apache.kafka.message.MessageGenerator" + classpath = project(':generator').sourceSets.main.runtimeClasspath + args = [ "src/generated/java/org/apache/kafka/common/message", "src/main/resources/common/message" ] + inputs.dir("src/main/resources/common/message") + outputs.dir("src/generated/java/org/apache/kafka/common/message") } sourceSets { @@ -912,8 +933,7 @@ project(':clients') { } test { java { - srcDirs = ["src/generated/java", "src/test/java", - "$rootDir/buildSrc/src/main/java/org/apache/kafka/message/"] + srcDirs = ["src/generated/java", "src/test/java"] } } } diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle deleted file mode 100644 index 23a1fc413f78b..0000000000000 --- a/buildSrc/build.gradle +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -repositories { - mavenCentral() -} - -dependencies { - compile "com.fasterxml.jackson.core:jackson-databind:2.9.6" - compile "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.6" - compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.9.6" -} - -test.enabled=false diff --git a/buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java b/buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java deleted file mode 100644 index dd85a5491c91c..0000000000000 --- a/buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kafka.task; - -import org.apache.kafka.message.MessageGenerator; -import org.gradle.api.DefaultTask; -import org.gradle.api.tasks.InputDirectory; -import org.gradle.api.tasks.OutputFiles; -import org.gradle.api.tasks.TaskAction; - -import java.io.File; -import java.util.Map; - -/** - * A gradle task which processes a directory full of JSON files into an output directory. - */ -public class ProcessMessagesTask extends DefaultTask { - /** - * The directory where we should read the input JSON from. - */ - public File inputDirectory; - - /** - * The directory that we should write output JSON to. - */ - public File outputDirectory; - - @InputDirectory - public File getInputDirectory() { - return inputDirectory; - } - - /** - * Define the task outputs. - * - * Gradle consults this to see if the task is up-to-date. - */ - @OutputFiles - public Map getOutputFiles() throws Exception { - return MessageGenerator.getOutputFiles( - outputDirectory.toString(), inputDirectory.toString()); - } - - @TaskAction - public void run() { - try { - MessageGenerator.processDirectories( - outputDirectory.toString(), inputDirectory.toString()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} diff --git a/buildSrc/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java b/generator/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java similarity index 100% rename from buildSrc/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java rename to generator/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java diff --git a/buildSrc/src/main/java/org/apache/kafka/message/CodeBuffer.java b/generator/src/main/java/org/apache/kafka/message/CodeBuffer.java similarity index 100% rename from buildSrc/src/main/java/org/apache/kafka/message/CodeBuffer.java rename to generator/src/main/java/org/apache/kafka/message/CodeBuffer.java diff --git a/buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java b/generator/src/main/java/org/apache/kafka/message/FieldSpec.java similarity index 100% rename from buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java rename to generator/src/main/java/org/apache/kafka/message/FieldSpec.java diff --git a/buildSrc/src/main/java/org/apache/kafka/message/FieldType.java b/generator/src/main/java/org/apache/kafka/message/FieldType.java similarity index 100% rename from buildSrc/src/main/java/org/apache/kafka/message/FieldType.java rename to generator/src/main/java/org/apache/kafka/message/FieldType.java diff --git a/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java b/generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java similarity index 100% rename from buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java rename to generator/src/main/java/org/apache/kafka/message/HeaderGenerator.java diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java similarity index 100% rename from buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java rename to generator/src/main/java/org/apache/kafka/message/MessageDataGenerator.java diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java similarity index 76% rename from buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java rename to generator/src/main/java/org/apache/kafka/message/MessageGenerator.java index 2abc7046f00da..74f9880c2c66c 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java +++ b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java @@ -24,14 +24,12 @@ import com.fasterxml.jackson.databind.SerializationFeature; import java.io.BufferedWriter; -import java.io.File; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.HashMap; import java.util.HashSet; import java.util.Locale; -import java.util.Map; /** * The Kafka message generator. @@ -94,40 +92,30 @@ public final class MessageGenerator { JSON_SERDE.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); } - public static Map getOutputFiles(String outputDir, String inputDir) throws Exception { - HashMap outputFiles = new HashMap<>(); - for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) { - String jsonName = inputPath.getFileName().toString(); - String javaName = jsonName.substring(0, jsonName.length() - JSON_SUFFIX.length()) + "Data.java"; - File outputFile = new File(outputDir, javaName); - outputFiles.put(outputFile.toString(), outputFile); - } - File factoryFile = new File(outputDir, API_MESSAGE_FACTORY_JAVA); - outputFiles.put(factoryFile.toString(), factoryFile); - return outputFiles; - } - public static void processDirectories(String outputDir, String inputDir) throws Exception { Files.createDirectories(Paths.get(outputDir)); int numProcessed = 0; ApiMessageFactoryGenerator messageFactoryGenerator = new ApiMessageFactoryGenerator(); HashSet outputFileNames = new HashSet<>(); - for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) { - try { - MessageSpec spec = JSON_SERDE. - readValue(inputPath.toFile(), MessageSpec.class); - String javaName = spec.generatedClassName() + JAVA_SUFFIX; - outputFileNames.add(javaName); - Path outputPath = Paths.get(outputDir, javaName); - try (BufferedWriter writer = Files.newBufferedWriter(outputPath)) { - MessageDataGenerator generator = new MessageDataGenerator(); - generator.generate(spec); - generator.write(writer); + try (DirectoryStream directoryStream = Files + .newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) { + for (Path inputPath : directoryStream) { + try { + MessageSpec spec = JSON_SERDE. + readValue(inputPath.toFile(), MessageSpec.class); + String javaName = spec.generatedClassName() + JAVA_SUFFIX; + outputFileNames.add(javaName); + Path outputPath = Paths.get(outputDir, javaName); + try (BufferedWriter writer = Files.newBufferedWriter(outputPath)) { + MessageDataGenerator generator = new MessageDataGenerator(); + generator.generate(spec); + generator.write(writer); + } + numProcessed++; + messageFactoryGenerator.registerMessageType(spec); + } catch (Exception e) { + throw new RuntimeException("Exception while processing " + inputPath.toString(), e); } - numProcessed++; - messageFactoryGenerator.registerMessageType(spec); - } catch (Exception e) { - throw new RuntimeException("Exception while processing " + inputPath.toString(), e); } } Path factoryOutputPath = Paths.get(outputDir, API_MESSAGE_FACTORY_JAVA); @@ -137,9 +125,15 @@ public static void processDirectories(String outputDir, String inputDir) throws messageFactoryGenerator.write(writer); } numProcessed++; - for (Path outputPath : Files.newDirectoryStream(Paths.get(outputDir))) { - if (!outputFileNames.contains(outputPath.getFileName().toString())) { - Files.delete(outputPath); + try (DirectoryStream directoryStream = Files. + newDirectoryStream(Paths.get(outputDir))) { + for (Path outputPath : directoryStream) { + Path fileName = outputPath.getFileName(); + if (fileName != null) { + if (!outputFileNames.contains(fileName.toString())) { + Files.delete(outputPath); + } + } } } System.out.printf("MessageGenerator: processed %d Kafka message JSON files(s).%n", numProcessed); diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java b/generator/src/main/java/org/apache/kafka/message/MessageSpec.java similarity index 100% rename from buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java rename to generator/src/main/java/org/apache/kafka/message/MessageSpec.java diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageSpecType.java b/generator/src/main/java/org/apache/kafka/message/MessageSpecType.java similarity index 100% rename from buildSrc/src/main/java/org/apache/kafka/message/MessageSpecType.java rename to generator/src/main/java/org/apache/kafka/message/MessageSpecType.java diff --git a/buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java b/generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java similarity index 100% rename from buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java rename to generator/src/main/java/org/apache/kafka/message/SchemaGenerator.java diff --git a/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java b/generator/src/main/java/org/apache/kafka/message/StructSpec.java similarity index 100% rename from buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java rename to generator/src/main/java/org/apache/kafka/message/StructSpec.java diff --git a/buildSrc/src/main/java/org/apache/kafka/message/Versions.java b/generator/src/main/java/org/apache/kafka/message/Versions.java similarity index 100% rename from buildSrc/src/main/java/org/apache/kafka/message/Versions.java rename to generator/src/main/java/org/apache/kafka/message/Versions.java diff --git a/clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java b/generator/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java similarity index 100% rename from clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java rename to generator/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java diff --git a/clients/src/test/java/org/apache/kafka/message/VersionsTest.java b/generator/src/test/java/org/apache/kafka/message/VersionsTest.java similarity index 100% rename from clients/src/test/java/org/apache/kafka/message/VersionsTest.java rename to generator/src/test/java/org/apache/kafka/message/VersionsTest.java diff --git a/settings.gradle b/settings.gradle index a74df21005e5e..5ae78e4036033 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,4 +17,4 @@ include 'core', 'examples', 'clients', 'tools', 'streams', 'streams:streams-scal 'streams:upgrade-system-tests-0100', 'streams:upgrade-system-tests-0101', 'streams:upgrade-system-tests-0102', 'streams:upgrade-system-tests-0110', 'streams:upgrade-system-tests-10', 'streams:upgrade-system-tests-11', 'streams:upgrade-system-tests-20', 'streams:upgrade-system-tests-21' , 'log4j-appender', 'connect:api', 'connect:transforms', 'connect:runtime', 'connect:json', 'connect:file', - 'connect:basic-auth-extension', 'jmh-benchmarks' + 'connect:basic-auth-extension', 'jmh-benchmarks', 'generator'