diff --git a/bom/application/pom.xml b/bom/application/pom.xml index ef88d66004f61..021caa256438c 100644 --- a/bom/application/pom.xml +++ b/bom/application/pom.xml @@ -3414,6 +3414,11 @@ avro ${avro.version} + + org.apache.avro + avro-compiler + ${avro.version} + org.apache.kafka kafka-streams diff --git a/core/deployment/src/main/java/io/quarkus/deployment/CodeGenerator.java b/core/deployment/src/main/java/io/quarkus/deployment/CodeGenerator.java index 25c2550b7bd20..a530e31c84267 100644 --- a/core/deployment/src/main/java/io/quarkus/deployment/CodeGenerator.java +++ b/core/deployment/src/main/java/io/quarkus/deployment/CodeGenerator.java @@ -45,7 +45,7 @@ public static List init(ClassLoader deploymentClassLoader, codeGenProviderClass = (Class) deploymentClassLoader .loadClass(CodeGenProvider.class.getName()); } catch (ClassNotFoundException e) { - throw new CodeGenException("Failde to load CodeGenProvider class from deployment classloader", e); + throw new CodeGenException("Failed to load CodeGenProvider class from deployment classloader", e); } for (CodeGenProvider provider : ServiceLoader.load(codeGenProviderClass, deploymentClassLoader)) { Path outputDir = codeGenOutDir(generatedSourcesDir, provider, sourceRegistrar); diff --git a/extensions/avro/deployment/pom.xml b/extensions/avro/deployment/pom.xml index 65d66aaf0177d..d322405cf1f7d 100644 --- a/extensions/avro/deployment/pom.xml +++ b/extensions/avro/deployment/pom.xml @@ -22,11 +22,20 @@ io.quarkus quarkus-avro + + org.apache.avro + avro-compiler + io.quarkus quarkus-junit5-internal test + + io.quarkus + quarkus-resteasy-reactive-deployment + test + diff --git a/extensions/avro/deployment/src/main/java/io/quarkus/avro/deployment/AvroCodeGenProviderBase.java b/extensions/avro/deployment/src/main/java/io/quarkus/avro/deployment/AvroCodeGenProviderBase.java new file mode 100644 index 0000000000000..cc83d36550757 --- /dev/null +++ b/extensions/avro/deployment/src/main/java/io/quarkus/avro/deployment/AvroCodeGenProviderBase.java @@ -0,0 +1,174 @@ +package io.quarkus.avro.deployment; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.avro.generic.GenericData; +import org.jboss.logging.Logger; + +import io.quarkus.bootstrap.prebuild.CodeGenException; +import io.quarkus.deployment.CodeGenContext; +import io.quarkus.deployment.CodeGenProvider; + +public abstract class AvroCodeGenProviderBase implements CodeGenProvider { + + private static final Logger log = Logger.getLogger(AvroCodeGenProviderBase.class); + + /** + * The directory (within the java classpath) that contains the velocity + * templates to use for code generation. + */ + static final String templateDirectory = "/org/apache/avro/compiler/specific/templates/java/classic/"; + + @Override + public String inputDirectory() { + return "avro"; + } + + @Override + public boolean trigger(CodeGenContext context) throws CodeGenException { + init(); + boolean filesGenerated = false; + + AvroOptions options = new AvroOptions(context.properties(), inputExtension()); + Path input = context.inputDir(); + Path outputDir = context.outDir(); + + Set importedPaths = new HashSet<>(); + + // compile the imports first + for (String imprt : options.imports) { + Path importPath = Paths.get(input.toAbsolutePath().toString(), imprt).toAbsolutePath(); + if (Files.isDirectory(importPath)) { + for (Path file : gatherAllFiles(importPath)) { + compileSingleFile(file, outputDir, options); + importedPaths.add(file); + filesGenerated = true; + } + } else { + compileSingleFile(importPath, outputDir, options); + importedPaths.add(importPath); + filesGenerated = true; + } + } + + // compile the rest of the files + for (Path file : gatherAllFiles(input)) { + if (!importedPaths.contains(file)) { + compileSingleFile(file, outputDir, options); + filesGenerated = true; + } + } + + return filesGenerated; + } + + abstract void init(); + + private Collection gatherAllFiles(Path importPath) throws CodeGenException { + try { + return Files.find(importPath, 20, + (path, ignored) -> Files.isRegularFile(path) && path.toString().endsWith("." + inputExtension())) + .map(Path::toAbsolutePath) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new CodeGenException("Failed to list matching files in " + importPath, e); + } + } + + abstract void compileSingleFile(Path importPath, Path outputDir, AvroOptions options) throws CodeGenException; + + public static class AvroOptions { + + public static final String[] EMPTY = new String[0]; + + private final Map properties; + + /** + * A list of files or directories that should be compiled first thus making them + * importable by subsequently compiled schemas. Note that imported files should + * not reference each other. + *

+ * All paths should be relative to the src/[main|test]/avro directory + *

+ * Passed as a comma-separated list. + */ + final String[] imports; + + /** + * The Java type to use for Avro strings. May be one of CharSequence, String or + * Utf8. CharSequence by default. + */ + final GenericData.StringType stringType; + + /** + * The createOptionalGetters parameter enables generating the getOptional... + * methods that return an Optional of the requested type. This works ONLY on + * Java 8+ + */ + final boolean createOptionalGetters; + + /** + * Determines whether or not to use Java classes for decimal types, defaults to false + */ + final boolean enableDecimalLogicalType; + + /** + * Determines whether or not to create setters for the fields of the record. The + * default is to create setters. + */ + final boolean createSetters; + + /** + * The gettersReturnOptional parameter enables generating get... methods that + * return an Optional of the requested type. This will replace the This works + * ONLY on Java 8+ + */ + final boolean gettersReturnOptional; + + /** + * The optionalGettersForNullableFieldsOnly parameter works in conjunction with + * gettersReturnOptional option. If it is set, Optional getters will be + * generated only for fields that are nullable. If the field is mandatory, + * regular getter will be generated. This works ONLY on Java 8+. + */ + final boolean optionalGettersForNullableFieldsOnly; + + AvroOptions(Map properties, String specificPropertyKey) { + this.properties = properties; + String imports = prop("avro.codegen." + specificPropertyKey + ".imports", ""); + this.imports = "".equals(imports) ? EMPTY : imports.split(","); + + stringType = GenericData.StringType.valueOf(prop("avro.codegen.stringType", "String")); + createOptionalGetters = getBooleanProperty("avro.codegen.createOptionalGetters", false); + enableDecimalLogicalType = getBooleanProperty("avro.codegen.enableDecimalLogicalType", false); + createSetters = getBooleanProperty("avro.codegen.createSetters", true); + gettersReturnOptional = getBooleanProperty("avro.codegen.gettersReturnOptional", false); + optionalGettersForNullableFieldsOnly = getBooleanProperty("avro.codegen.optionalGettersForNullableFieldsOnly", + false); + } + + private String prop(String propName, String defaultValue) { + return properties.getOrDefault(propName, defaultValue); + } + + private boolean getBooleanProperty(String propName, boolean defaultValue) { + String value = prop(propName, String.valueOf(defaultValue)).toLowerCase(Locale.ROOT); + if (Boolean.FALSE.toString().equals(value)) { + return false; + } + if (Boolean.TRUE.toString().equals(value)) { + return true; + } + return defaultValue; + } + } +} diff --git a/extensions/avro/deployment/src/main/java/io/quarkus/avro/deployment/AvroProtocolCodeGenProvider.java b/extensions/avro/deployment/src/main/java/io/quarkus/avro/deployment/AvroProtocolCodeGenProvider.java new file mode 100644 index 0000000000000..fec0652e58af3 --- /dev/null +++ b/extensions/avro/deployment/src/main/java/io/quarkus/avro/deployment/AvroProtocolCodeGenProvider.java @@ -0,0 +1,54 @@ +package io.quarkus.avro.deployment; + +import java.io.IOException; +import java.nio.file.Path; + +import org.apache.avro.Protocol; +import org.apache.avro.compiler.specific.SpecificCompiler; + +import io.quarkus.bootstrap.prebuild.CodeGenException; +import io.quarkus.deployment.CodeGenProvider; + +/** + * Avro code generator for Avro Protocol, based on the avro-maven-plugin + * + * @see AvroCodeGenProviderBase + */ +public class AvroProtocolCodeGenProvider extends AvroCodeGenProviderBase implements CodeGenProvider { + + @Override + public String providerId() { + return "avpr"; + } + + @Override + public String inputExtension() { + return "avpr"; + } + + @Override + void init() { + } + + void compileSingleFile(Path filePath, + Path outputDirectory, + AvroOptions options) throws CodeGenException { + try { + final Protocol protocol = Protocol.parse(filePath.toFile()); + final SpecificCompiler compiler = new SpecificCompiler(protocol); + compiler.setTemplateDir(templateDirectory); + compiler.setStringType(options.stringType); + compiler.setFieldVisibility(SpecificCompiler.FieldVisibility.PRIVATE); + compiler.setCreateOptionalGetters(options.createOptionalGetters); + compiler.setGettersReturnOptional(options.gettersReturnOptional); + compiler.setOptionalGettersForNullableFieldsOnly(options.optionalGettersForNullableFieldsOnly); + compiler.setCreateSetters(options.createSetters); + compiler.setEnableDecimalLogicalType(options.enableDecimalLogicalType); + + compiler.setOutputCharacterEncoding("UTF-8"); + compiler.compileToDestination(filePath.toFile(), outputDirectory.toFile()); + } catch (IOException e) { + new CodeGenException("Failed to compile avro protocole file: " + filePath.toString() + " to Java", e); + } + } +} diff --git a/extensions/avro/deployment/src/main/java/io/quarkus/avro/deployment/AvroSchemaCodeGenProvider.java b/extensions/avro/deployment/src/main/java/io/quarkus/avro/deployment/AvroSchemaCodeGenProvider.java new file mode 100644 index 0000000000000..a274ca77a7fdb --- /dev/null +++ b/extensions/avro/deployment/src/main/java/io/quarkus/avro/deployment/AvroSchemaCodeGenProvider.java @@ -0,0 +1,75 @@ +package io.quarkus.avro.deployment; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; + +import org.apache.avro.Schema; +import org.apache.avro.compiler.specific.SpecificCompiler; + +import io.quarkus.bootstrap.prebuild.CodeGenException; +import io.quarkus.deployment.CodeGenProvider; + +/** + * Avro code generator for Avro Schema, based on the avro-maven-plugin + * + * @see AvroCodeGenProviderBase + */ +public class AvroSchemaCodeGenProvider extends AvroCodeGenProviderBase implements CodeGenProvider { + + Schema.Parser schemaParser; + + @Override + public String providerId() { + return "avsc"; + } + + @Override + public String inputExtension() { + return "avsc"; + } + + void init() { + schemaParser = new Schema.Parser(); + } + + @Override + void compileSingleFile(Path filePath, + Path outputDirectory, + AvroOptions options) throws CodeGenException { + final Schema schema; + + File file = filePath.toFile(); + + // This is necessary to maintain backward-compatibility. If there are + // no imported files then isolate the schemas from each other, otherwise + // allow them to share a single schema so reuse and sharing of schema + // is possible. + try { + if (options.imports == null) { + schema = new Schema.Parser().parse(file); + } else { + schema = schemaParser.parse(file); + } + } catch (IOException e) { + throw new CodeGenException("", e); + } + + final SpecificCompiler compiler = new SpecificCompiler(schema); + compiler.setTemplateDir(templateDirectory); + compiler.setStringType(options.stringType); + compiler.setFieldVisibility(SpecificCompiler.FieldVisibility.PRIVATE); + compiler.setCreateOptionalGetters(options.createOptionalGetters); + compiler.setGettersReturnOptional(options.gettersReturnOptional); + compiler.setOptionalGettersForNullableFieldsOnly(options.optionalGettersForNullableFieldsOnly); + compiler.setCreateSetters(options.createSetters); + compiler.setEnableDecimalLogicalType(options.enableDecimalLogicalType); + compiler.setOutputCharacterEncoding("UTF-8"); + try { + compiler.compileToDestination(file, outputDirectory.toFile()); + } catch (IOException e) { + throw new CodeGenException("Failed to copy compiled files to output directory " + + outputDirectory.toAbsolutePath(), e); + } + } +} diff --git a/extensions/avro/deployment/src/main/resources/META-INF/services/io.quarkus.deployment.CodeGenProvider b/extensions/avro/deployment/src/main/resources/META-INF/services/io.quarkus.deployment.CodeGenProvider new file mode 100644 index 0000000000000..d27d31cd4e5f8 --- /dev/null +++ b/extensions/avro/deployment/src/main/resources/META-INF/services/io.quarkus.deployment.CodeGenProvider @@ -0,0 +1,2 @@ +io.quarkus.avro.deployment.AvroSchemaCodeGenProvider +io.quarkus.avro.deployment.AvroProtocolCodeGenProvider \ No newline at end of file diff --git a/integration-tests/avro-reload/pom.xml b/integration-tests/avro-reload/pom.xml new file mode 100644 index 0000000000000..ab2490f872f22 --- /dev/null +++ b/integration-tests/avro-reload/pom.xml @@ -0,0 +1,81 @@ + + + 4.0.0 + + + io.quarkus + quarkus-integration-tests-parent + 999-SNAPSHOT + + + quarkus-avro-reload-test + Quarkus - Integration Test - Avro code generation and reload + + + imports,directImport/PrivacyDirectImport.avsc + + + + + io.quarkus + quarkus-avro + + + io.quarkus + quarkus-resteasy-reactive-deployment + + + io.quarkus + quarkus-junit5-internal + test + + + org.assertj + assertj-core + test + + + io.rest-assured + rest-assured + test + + + + + io.quarkus + quarkus-avro-deployment + ${project.version} + pom + test + + + * + * + + + + + + + + + io.quarkus + quarkus-maven-plugin + + + + generate-code + + generate-code + generate-code-tests + build + + + + + + + + diff --git a/integration-tests/avro-reload/src/test/avro/User.avpr b/integration-tests/avro-reload/src/test/avro/User.avpr new file mode 100644 index 0000000000000..6dd8b9b890031 --- /dev/null +++ b/integration-tests/avro-reload/src/test/avro/User.avpr @@ -0,0 +1,41 @@ +{ + "protocol" : "ProtocolTest", + "namespace" : "test", + "types" : [ + { + "type" : "enum", + "name" : "ProtocolPrivacy", + "symbols" : [ "Public", "Private"] + }, + { + "type": "record", + "namespace": "test", + "name": "ProtocolUser", + "doc": "User Test Bean", + "fields": [ + { + "name": "id", + "type": ["null", "string"], + "default": null + }, + { + "name": "createdOn", + "type": ["null", "long"], + "default": null + }, + { + "name": "privacy", + "type": ["null", "ProtocolPrivacy"], + "default": null + }, + { + "name": "modifiedOn", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + } + } + ] + } + ] +} diff --git a/integration-tests/avro-reload/src/test/avro/User.avsc b/integration-tests/avro-reload/src/test/avro/User.avsc new file mode 100644 index 0000000000000..11c4aa7d51ab8 --- /dev/null +++ b/integration-tests/avro-reload/src/test/avro/User.avsc @@ -0,0 +1,45 @@ +{ + "type": "record", + "namespace": "test", + "name": "SchemaUser", + "doc": "User Test Bean", + "fields": [ + { + "name": "id", + "type": ["null", "string"], + "default": null + }, + { + "name": "createdOn", + "type": ["null", "long"], + "default": null + }, + { + "name": "privacy", + "type": ["null", { + "type": "enum", + "name": "SchemaPrivacy", + "namespace": "test", + "symbols" : ["Public","Private"] + }], + "default": null + }, + { + "name": "privacyImported", + "type": ["null", "test.PrivacyImport"], + "default": null + }, + { + "name": "privacyDirectImport", + "type": ["null", "test.PrivacyDirectImport"], + "default": null + }, + { + "name": "time", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + } + } + ] +} diff --git a/integration-tests/avro-reload/src/test/avro/directImport/PrivacyDirectImport.avsc b/integration-tests/avro-reload/src/test/avro/directImport/PrivacyDirectImport.avsc new file mode 100644 index 0000000000000..a5b629592068a --- /dev/null +++ b/integration-tests/avro-reload/src/test/avro/directImport/PrivacyDirectImport.avsc @@ -0,0 +1,7 @@ +{ + "type": "enum", + "namespace": "test", + "name": "PrivacyDirectImport", + "doc": "Privacy Test Enum", + "symbols" : ["Public","Private"] +} diff --git a/integration-tests/avro-reload/src/test/avro/imports/PrivacyImport.avsc b/integration-tests/avro-reload/src/test/avro/imports/PrivacyImport.avsc new file mode 100644 index 0000000000000..d697a35c47e67 --- /dev/null +++ b/integration-tests/avro-reload/src/test/avro/imports/PrivacyImport.avsc @@ -0,0 +1,8 @@ +/* copied from avro-maven-plugin, licensed under Apache Software License */ +{ + "type": "enum", + "namespace": "test", + "name": "PrivacyImport", + "doc": "Privacy Test Enum", + "symbols" : ["Public","Private"] +} diff --git a/integration-tests/avro-reload/src/test/java/io/quarkus/avro/deployment/AvroCodeReloadTest.java b/integration-tests/avro-reload/src/test/java/io/quarkus/avro/deployment/AvroCodeReloadTest.java new file mode 100644 index 0000000000000..c4dcc9a90c4df --- /dev/null +++ b/integration-tests/avro-reload/src/test/java/io/quarkus/avro/deployment/AvroCodeReloadTest.java @@ -0,0 +1,46 @@ +package io.quarkus.avro.deployment; + +import static io.restassured.RestAssured.when; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.regex.Pattern; + +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.spec.JavaArchive; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import io.quarkus.test.QuarkusDevModeTest; + +public class AvroCodeReloadTest { + @RegisterExtension + public static final QuarkusDevModeTest test = new QuarkusDevModeTest() + .setArchiveProducer( + () -> ShrinkWrap.create(JavaArchive.class) + .addClasses(AvroReloadResource.class)) + .setCodeGenSources("avro") + .setBuildSystemProperty("avro.codegen.avsc.imports", "imports"); + + @Test + void shouldAlterSchema() throws InterruptedException { + assertThat(when().get().body().print().split(",")).containsExactlyInAnyOrder("Public", "Private"); + + test.modifyFile("avro/imports/PrivacyImport.avsc", + text -> text.replaceAll(Pattern.quote("\"symbols\" : [\"Public\",\"Private\"]"), + "\"symbols\" : [\"Public\",\"Private\",\"Default\"]")); + Thread.sleep(5000); // to wait for eager reload for code gen sources to happen + assertThat(when().get().body().print().split(",")).containsExactlyInAnyOrder("Public", "Private", "Default"); + } + + @Test + void shouldAlterProtocol() throws InterruptedException { + assertThat(when().get("/protocol").body().print().split(",")).containsExactlyInAnyOrder("Public", "Private"); + + test.modifyFile("avro/User.avpr", + text -> text.replaceAll(Pattern.quote("\"symbols\" : [ \"Public\", \"Private\"]"), + "\"symbols\" : [ \"Public\", \"Private\", \"Default\"]")); + Thread.sleep(5000); // to wait for eager reload for code gen sources to happen + assertThat(when().get("/protocol").body().print().split(",")).containsExactlyInAnyOrder("Public", "Private", "Default"); + } + +} diff --git a/integration-tests/avro-reload/src/test/java/io/quarkus/avro/deployment/AvroReloadResource.java b/integration-tests/avro-reload/src/test/java/io/quarkus/avro/deployment/AvroReloadResource.java new file mode 100644 index 0000000000000..f93e7744b0ef1 --- /dev/null +++ b/integration-tests/avro-reload/src/test/java/io/quarkus/avro/deployment/AvroReloadResource.java @@ -0,0 +1,26 @@ +package io.quarkus.avro.deployment; + +import static java.util.stream.Collectors.joining; + +import java.util.Arrays; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; + +import test.PrivacyImport; +import test.ProtocolPrivacy; + +@Path("/") +public class AvroReloadResource { + @GET + public String getAvailablePrivacyImports() { + return Arrays.stream(PrivacyImport.values()).map(String::valueOf).collect(joining(",")); + } + + @GET + @Path("/protocol") + public String getAvailableProtocolPrivacies() { + return Arrays.stream(ProtocolPrivacy.values()).map(String::valueOf).collect(joining(",")); + } + +} diff --git a/integration-tests/kafka-avro/pom.xml b/integration-tests/kafka-avro/pom.xml index b47bbccf3f8b2..f96cd5599ce4f 100644 --- a/integration-tests/kafka-avro/pom.xml +++ b/integration-tests/kafka-avro/pom.xml @@ -250,30 +250,13 @@ - - org.apache.avro - avro-maven-plugin - 1.10.2 - - - generate-sources - - schema - - - src/main/avro - ${project.build.directory}/generated-sources - String - - - - io.quarkus quarkus-maven-plugin + generate-code build diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 952a973d1532c..c932b1d249dd7 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -23,6 +23,7 @@ true + avro-reload bouncycastle bouncycastle-fips bouncycastle-fips-jsse diff --git a/test-framework/junit5-internal/src/main/java/io/quarkus/test/QuarkusDevModeTest.java b/test-framework/junit5-internal/src/main/java/io/quarkus/test/QuarkusDevModeTest.java index bb291e63bd2d7..9af9784a2acfe 100644 --- a/test-framework/junit5-internal/src/main/java/io/quarkus/test/QuarkusDevModeTest.java +++ b/test-framework/junit5-internal/src/main/java/io/quarkus/test/QuarkusDevModeTest.java @@ -16,7 +16,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.Enumeration; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.ServiceLoader; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; @@ -98,6 +100,8 @@ public class QuarkusDevModeTest private Path testLocation; private String[] commandLineArgs = new String[0]; + private final Map buildSystemProperties = new HashMap<>(); + private static final List compilationProviders; static { @@ -136,6 +140,11 @@ public List getLogRecords() { return inMemoryLogHandler.records; } + public QuarkusDevModeTest setBuildSystemProperty(String name, String value) { + buildSystemProperties.put(name, value); + return this; + } + public Object createTestInstance(TestInstanceFactoryContext factoryContext, ExtensionContext extensionContext) throws TestInstantiationException { try { @@ -202,6 +211,7 @@ public void close() throws Throwable { context.setTest(true); context.setAbortOnFailedStart(true); context.getBuildSystemProperties().put("quarkus.banner.enabled", "false"); + context.getBuildSystemProperties().putAll(buildSystemProperties); devModeMain = new DevModeMain(context); devModeMain.start(); ApplicationStateNotification.waitForApplicationStart();