Skip to content

Commit

Permalink
Experimental support for Avro in native mode.
Browse files Browse the repository at this point in the history
  • Loading branch information
cescoffier committed Jul 19, 2020
1 parent 1c416eb commit 5c68f82
Show file tree
Hide file tree
Showing 12 changed files with 413 additions and 6 deletions.
16 changes: 16 additions & 0 deletions bom/application/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@
<gson.version>2.8.6</gson.version>
<webjars-locator-core.version>0.46</webjars-locator-core.version>
<log4j2-jboss-logmanager.version>1.0.0.Beta1</log4j2-jboss-logmanager.version>
<avro.version>1.10.0</avro.version>
</properties>

<dependencyManagement>
Expand Down Expand Up @@ -804,6 +805,16 @@
<artifactId>quarkus-kafka-streams-deployment</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-avro</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-avro-deployment</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-smallrye-health</artifactId>
Expand Down Expand Up @@ -2651,6 +2662,11 @@
<artifactId>kafka-clients</artifactId>
<version>${kafka2.version}</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>${avro.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams</artifactId>
Expand Down
49 changes: 49 additions & 0 deletions extensions/avro/deployment/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-avro-parent</artifactId>
<version>999-SNAPSHOT</version>
</parent>

<artifactId>quarkus-avro-deployment</artifactId>
<name>Quarkus - Avro - Deployment</name>

<dependencies>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-core-deployment</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-avro</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-junit5-internal</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<annotationProcessorPaths>
<path>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-extension-processor</artifactId>
<version>${project.version}</version>
</path>
</annotationProcessorPaths>
</configuration>
</plugin>
</plugins>
</build>

</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package io.quarkus.avro.deployment;

import java.util.Collection;

import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.specific.AvroGenerated;
import org.jboss.jandex.AnnotationInstance;
import org.jboss.jandex.AnnotationTarget;
import org.jboss.jandex.DotName;

import io.quarkus.deployment.annotations.BuildProducer;
import io.quarkus.deployment.annotations.BuildStep;
import io.quarkus.deployment.builditem.CombinedIndexBuildItem;
import io.quarkus.deployment.builditem.nativeimage.NativeImageConfigBuildItem;
import io.quarkus.deployment.builditem.nativeimage.NativeImageSystemPropertyBuildItem;
import io.quarkus.deployment.builditem.nativeimage.ReflectiveClassBuildItem;

public class AvroProcessor {

@BuildStep
public void build(CombinedIndexBuildItem indexBuildItem,
BuildProducer<ReflectiveClassBuildItem> reflectiveClass,
BuildProducer<NativeImageSystemPropertyBuildItem> sys,
BuildProducer<NativeImageConfigBuildItem> conf) {

NativeImageConfigBuildItem.Builder builder = NativeImageConfigBuildItem.builder();
builder.addRuntimeInitializedClass(GenericDatumReader.class.getName());

Collection<AnnotationInstance> annotations = indexBuildItem.getIndex()
.getAnnotations(DotName.createSimple(AvroGenerated.class.getName()));
for (AnnotationInstance annotation : annotations) {
if (annotation.target().kind() == AnnotationTarget.Kind.CLASS) {
String className = annotation.target().asClass().name().toString();
builder.addRuntimeInitializedClass(className);
reflectiveClass.produce(new ReflectiveClassBuildItem(true, true, true, className));
}
}

builder.addRuntimeInitializedClass("org.apache.avro.reflect.ReflectData");
conf.produce(builder.build());
sys.produce(new NativeImageSystemPropertyBuildItem("avro.disable.unsafe", "true"));
}
}
21 changes: 21 additions & 0 deletions extensions/avro/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>quarkus-build-parent</artifactId>
<groupId>io.quarkus</groupId>
<version>999-SNAPSHOT</version>
<relativePath>../../build-parent/pom.xml</relativePath>
</parent>

<modelVersion>4.0.0</modelVersion>
<artifactId>quarkus-avro-parent</artifactId>
<name>Quarkus - Avro</name>
<packaging>pom</packaging>

<modules>
<module>deployment</module>
<module>runtime</module>
</modules>
</project>
56 changes: 56 additions & 0 deletions extensions/avro/runtime/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-avro-parent</artifactId>
<version>999-SNAPSHOT</version>
</parent>

<artifactId>quarkus-avro</artifactId>
<name>Quarkus - Avro - Runtime</name>
<description>Provide support for the Avro data serialization system</description>
<dependencies>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-core</artifactId>
</dependency>

<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</dependency>

<dependency>
<groupId>org.graalvm.nativeimage</groupId>
<artifactId>svm</artifactId>
</dependency>
</dependencies>

<build>
<!-- Mark this as a runtime dependency, so to make sure it's included on the final classpath during native-image -->
<plugins>
<plugin>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-bootstrap-maven-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<annotationProcessorPaths>
<path>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-extension-processor</artifactId>
<version>${project.version}</version>
</path>
</annotationProcessorPaths>
</configuration>
</plugin>
</plugins>
</build>


</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package io.quarkus.avro.graal;

import java.lang.reflect.Constructor;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Function;

import org.apache.avro.generic.IndexedRecord;

import com.oracle.svm.core.annotate.Alias;
import com.oracle.svm.core.annotate.Inject;
import com.oracle.svm.core.annotate.RecomputeFieldValue;
import com.oracle.svm.core.annotate.Substitute;
import com.oracle.svm.core.annotate.TargetClass;

@TargetClass(className = "org.apache.avro.reflect.ReflectionUtil")
final class Target_org_apache_avro_reflect_ReflectionUtil {

/**
* Use reflection instead of method handles
*/
@Substitute
public static <V, R> Function<V, R> getConstructorAsFunction(Class<V> parameterClass, Class<R> clazz) {
try {
Constructor<R> constructor = clazz.getConstructor(parameterClass);
return new Function<V, R>() {
@Override
public R apply(V v) {
try {
return constructor.newInstance(v);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
} catch (Throwable t) {
// if something goes wrong, do not provide a Function instance
return null;
}
}

}

@TargetClass(className = "org.apache.avro.reflect.ReflectData")
final class Target_org_apache_avro_reflect_ReflectData {

@Inject
@RecomputeFieldValue(kind = RecomputeFieldValue.Kind.None)
Map<Class<?>, Target_org_apache_avro_reflect_ReflectData_ClassAccessorData> ACCESSORS;

@Substitute
private Target_org_apache_avro_reflect_ReflectData_ClassAccessorData getClassAccessorData(Class<?> c) {
if (ACCESSORS == null) {
ACCESSORS = new HashMap<>();
}

Map<Class<?>, Target_org_apache_avro_reflect_ReflectData_ClassAccessorData> map = ACCESSORS;
Target_org_apache_avro_reflect_ReflectData_ClassAccessorData o = map.get(c);
if (o == null) {
if (!IndexedRecord.class.isAssignableFrom(c)) {
Target_org_apache_avro_reflect_ReflectData_ClassAccessorData d = new Target_org_apache_avro_reflect_ReflectData_ClassAccessorData(
c);
map.put(c, d);
}
return null;
}
return o;
}
}

@TargetClass(className = "org.apache.avro.reflect.ReflectData", innerClass = "ClassAccessorData")
final class Target_org_apache_avro_reflect_ReflectData_ClassAccessorData<T> {
// Just provide access to "ReflectData.ClassAccessorData"

@Alias
public Target_org_apache_avro_reflect_ReflectData_ClassAccessorData(Class<?> c) {

}

}

class AvroSubstitutions {
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
name: "Apache Avro"
metadata:
keywords:
- "avro"
guide: "https://quarkus.io/guides/kafka"
categories:
- "serialization"
status: "experimental"
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import io.quarkus.deployment.builditem.AdditionalIndexedClassesBuildItem;
import io.quarkus.deployment.builditem.CombinedIndexBuildItem;
import io.quarkus.deployment.builditem.IndexDependencyBuildItem;
import io.quarkus.deployment.builditem.nativeimage.NativeImageProxyDefinitionBuildItem;
import io.quarkus.deployment.builditem.nativeimage.ReflectiveClassBuildItem;
import io.quarkus.deployment.builditem.nativeimage.ReflectiveHierarchyBuildItem;
import io.quarkus.kafka.client.runtime.KafkaRuntimeConfigProducer;
Expand Down Expand Up @@ -100,6 +101,7 @@ void contributeClassesToIndex(BuildProducer<AdditionalIndexedClassesBuildItem> a

@BuildStep
public void build(CombinedIndexBuildItem indexBuildItem, BuildProducer<ReflectiveClassBuildItem> reflectiveClass,
BuildProducer<NativeImageProxyDefinitionBuildItem> proxies,
Capabilities capabilities) {
final Set<DotName> toRegister = new HashSet<>();

Expand Down Expand Up @@ -139,6 +141,71 @@ public void build(CombinedIndexBuildItem indexBuildItem, BuildProducer<Reflectiv
// classes needed to perform reflection on DirectByteBuffer - only really needed for Java 8
reflectiveClass.produce(new ReflectiveClassBuildItem(true, false, "java.nio.DirectByteBuffer"));
reflectiveClass.produce(new ReflectiveClassBuildItem(true, false, "sun.misc.Cleaner"));

// Avro - for both Confluent and Apicurio
try {
Class.forName("io.confluent.kafka.serializers.KafkaAvroDeserializer");
reflectiveClass
.produce(new ReflectiveClassBuildItem(true, false,
"io.confluent.kafka.serializers.KafkaAvroDeserializer",
"io.confluent.kafka.serializers.KafkaAvroSerializer"));

reflectiveClass
.produce(new ReflectiveClassBuildItem(true, true, false,
"io.confluent.kafka.serializers.subject.TopicNameStrategy",
"io.confluent.kafka.serializers.subject.TopicRecordNameStrategy",
"io.confluent.kafka.serializers.subject.RecordNameStrategy"));

reflectiveClass
.produce(new ReflectiveClassBuildItem(true, true, false,
"io.confluent.kafka.schemaregistry.client.rest.entities.ErrorMessage",
"io.confluent.kafka.schemaregistry.client.rest.entities.Schema",
"io.confluent.kafka.schemaregistry.client.rest.entities.Config",
"io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference",
"io.confluent.kafka.schemaregistry.client.rest.entities.SchemaString",
"io.confluent.kafka.schemaregistry.client.rest.entities.SchemaTypeConverter",
"io.confluent.kafka.schemaregistry.client.rest.entities.ServerClusterId",
"io.confluent.kafka.schemaregistry.client.rest.entities.SujectVersion"));

reflectiveClass
.produce(new ReflectiveClassBuildItem(true, true, false,
"io.confluent.kafka.schemaregistry.client.rest.entities.requests.CompatibilityCheckResponse",
"io.confluent.kafka.schemaregistry.client.rest.entities.requests.ConfigUpdateRequest",
"io.confluent.kafka.schemaregistry.client.rest.entities.requests.ModeGetResponse",
"io.confluent.kafka.schemaregistry.client.rest.entities.requests.ModeUpdateRequest",
"io.confluent.kafka.schemaregistry.client.rest.entities.requests.RegisterSchemaRequest",
"io.confluent.kafka.schemaregistry.client.rest.entities.requests.RegisterSchemaResponse"));
} catch (ClassNotFoundException e) {
//ignore, Confluent Avro is not in the classpath
}

try {
Class.forName("io.apicurio.registry.utils.serde.AvroKafkaDeserializer");
reflectiveClass.produce(
new ReflectiveClassBuildItem(true, true, false,
"io.apicurio.registry.utils.serde.AvroKafkaDeserializer",
"io.apicurio.registry.utils.serde.AvroKafkaSerializer"));

reflectiveClass.produce(new ReflectiveClassBuildItem(true, true, false,
"io.apicurio.registry.utils.serde.avro.ReflectAvroDatumProvider",
"io.apicurio.registry.utils.serde.strategy.AutoRegisterIdStrategy",
"io.apicurio.registry.utils.serde.strategy.CachedSchemaIdStrategy",
"io.apicurio.registry.utils.serde.strategy.FindBySchemaIdStrategy",
"io.apicurio.registry.utils.serde.strategy.FindLatestIdStrategy",
"io.apicurio.registry.utils.serde.strategy.GetOrCreateIdStrategy",
"io.apicurio.registry.utils.serde.strategy.RecordIdStrategy",
"io.apicurio.registry.utils.serde.strategy.SimpleTopicIdStrategy",
"io.apicurio.registry.utils.serde.strategy.TopicIdStrategy",
"io.apicurio.registry.utils.serde.strategy.TopicRecordIdStrategy"));

// Apicurio uses dynamic proxies, register them
proxies.produce(new NativeImageProxyDefinitionBuildItem("io.apicurio.registry.client.RegistryService",
"java.lang.AutoCloseable"));

} catch (ClassNotFoundException e) {
//ignore, Apicurio Avro is not in the classpath
}

}

@BuildStep
Expand Down
1 change: 1 addition & 0 deletions extensions/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
<module>mongodb-client</module>
<module>artemis-core</module>
<module>artemis-jms</module>
<module>avro</module>

<!-- Spring -->
<module>spring-di</module>
Expand Down
Loading

0 comments on commit 5c68f82

Please sign in to comment.