diff --git a/api/src/main/java/com/github/sbt/avro/AvroCompiler.java b/api/src/main/java/com/github/sbt/avro/AvroCompiler.java index 51e8bd9..d68c815 100644 --- a/api/src/main/java/com/github/sbt/avro/AvroCompiler.java +++ b/api/src/main/java/com/github/sbt/avro/AvroCompiler.java @@ -1,20 +1,10 @@ package com.github.sbt.avro; import java.io.File; -import java.io.IOException; public interface AvroCompiler { - void compileAvroSchema( - Class[] records, - File[] avdls, - AvroFileRef[] avscs, - File[] avprs, - File target, - String stringType, - String fieldVisibility, - Boolean enableDecimalLogicalType, - Boolean useNamespace, - Boolean optionalGetters, - Boolean createSetters - ) throws IOException; + void recompile(Class[] records, File target) throws Exception; + void compileIdls(File[] idls, File target) throws Exception; + void compileAvscs(AvroFileRef[] avscs, File target) throws Exception; + void compileAvprs(File[] avprs, File target) throws Exception; } diff --git a/bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java b/bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java index ea43d31..f5d01a0 100644 --- a/bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java +++ b/bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java @@ -1,30 +1,53 @@ package com.github.sbt.avro; +import org.apache.avro.Schema; +import org.apache.avro.specific.SpecificRecord; +import xsbti.Logger; + import org.apache.avro.Protocol; import org.apache.avro.compiler.idl.Idl; -import org.apache.avro.compiler.idl.ParseException; import org.apache.avro.compiler.specific.SpecificCompiler; import org.apache.avro.compiler.specific.SpecificCompiler.FieldVisibility; import org.apache.avro.generic.GenericData.StringType; -import org.apache.avro.specific.SpecificRecord; import java.io.File; -import java.io.IOException; +import java.util.HashSet; import java.util.Set; public class AvroCompilerBridge implements AvroCompiler { - void recompile( - Class[] records, - File target, - StringType stringType, - FieldVisibility fieldVisibility, - Boolean enableDecimalLogicalType, - Boolean useNamespace, - Boolean optionalGetters, - Boolean createSetters - // builder: SchemaParserBuilder + private final Logger logger; + private final StringType stringType; + private final FieldVisibility fieldVisibility; + private final boolean useNamespace; + private final boolean enableDecimalLogicalType; + private final boolean createSetters; + private final boolean optionalGetters; + + public AvroCompilerBridge( + Logger logger, + String stringType, + String fieldVisibility, + boolean useNamespace, + boolean enableDecimalLogicalType, + boolean createSetters, + boolean optionalGetters ) { + this.logger = logger; + this.stringType = StringType.valueOf(stringType); + this.fieldVisibility = FieldVisibility.valueOf(fieldVisibility); + this.useNamespace = useNamespace; + this.enableDecimalLogicalType = enableDecimalLogicalType; + this.createSetters = createSetters; + this.optionalGetters = optionalGetters; + } + + protected Schema.Parser createParser() { + return new Schema.Parser(); + } + + @Override + public void recompile(Class[] records, File target) throws Exception { AvscFilesCompiler compiler = new AvscFilesCompiler(); compiler.setStringType(stringType); compiler.setFieldVisibility(fieldVisibility); @@ -37,22 +60,20 @@ void recompile( if (AvroVersion.getMinor() > 9) { compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters); } - compiler.setLogCompileExceptions(true); compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/"); - compiler.compileClasses(Set.of(records), target); + + Set> classes = new HashSet<>(); + for (Class record : records) { + logger.info(() -> "Recompiling Avro record: " + record.getName()); + classes.add((Class) record); + } + compiler.compileClasses(classes, target); } - void compileIdls( - File[] idls, - File target, - StringType stringType, - FieldVisibility fieldVisibility, - Boolean enableDecimalLogicalType, - Boolean optionalGetters, - Boolean createSetters - ) throws IOException, ParseException { + @Override + public void compileIdls(File[] idls, File target) throws Exception { for (File idl : idls) { -// log.info(s"Compiling Avro IDL $idl") + logger.info(() -> "Compiling Avro IDL " + idl); Idl parser = new Idl(idl); Protocol protocol = parser.CompilationUnit(); SpecificCompiler compiler = new SpecificCompiler(protocol); @@ -70,17 +91,8 @@ void compileIdls( } } - void compileAvscs( - AvroFileRef[] avscs, - File target, - StringType stringType, - FieldVisibility fieldVisibility, - Boolean enableDecimalLogicalType, - Boolean useNamespace, - Boolean optionalGetters, - Boolean createSetters - // builder: SchemaParserBuilder - ) { + @Override + public void compileAvscs(AvroFileRef[] avscs, File target) throws Exception { AvscFilesCompiler compiler = new AvscFilesCompiler(); compiler.setStringType(stringType); compiler.setFieldVisibility(fieldVisibility); @@ -93,23 +105,20 @@ void compileAvscs( if (AvroVersion.getMinor() > 9) { compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters); } - compiler.setLogCompileExceptions(true); compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/"); + Set files = new HashSet<>(); + for (AvroFileRef ref: avscs) { + logger.info(() -> "Compiling Avro schema: " + ref.getFile()); + files.add(ref); + } compiler.compileFiles(Set.of(avscs), target); } - void compileAvprs( - File[] avprs, - File target, - StringType stringType, - FieldVisibility fieldVisibility, - Boolean enableDecimalLogicalType, - Boolean optionalGetters, - Boolean createSetters - ) throws IOException { + @Override + public void compileAvprs(File[] avprs, File target) throws Exception { for (File avpr : avprs) { -// log.info(s"Compiling Avro protocol $avpr") + logger.info(() -> "Compiling Avro protocol " + avpr); Protocol protocol = Protocol.parse(avpr); SpecificCompiler compiler = new SpecificCompiler(protocol); compiler.setStringType(stringType); @@ -125,32 +134,4 @@ void compileAvprs( compiler.compileToDestination(null, target); } } - - - @Override - public void compileAvroSchema( - Class[] records, - File[] avdls, - AvroFileRef[] avscs, - File[] avprs, - File target, - String stringType, - String fieldVisibility, - Boolean enableDecimalLogicalType, - Boolean useNamespace, - Boolean optionalGetters, - Boolean createSetters) { - StringType stringTypeEnum = StringType.valueOf(stringType); - FieldVisibility fieldVisibilityEnum = FieldVisibility.valueOf(fieldVisibility); - - try { - recompile((Class[]) records, target, stringTypeEnum, fieldVisibilityEnum, enableDecimalLogicalType, useNamespace, optionalGetters, createSetters); - - compileIdls(avdls, target, stringTypeEnum, fieldVisibilityEnum, enableDecimalLogicalType, optionalGetters, createSetters); - compileAvscs(avscs, target,stringTypeEnum, fieldVisibilityEnum, enableDecimalLogicalType, useNamespace, optionalGetters, createSetters); - compileAvprs(avprs, target, stringTypeEnum, fieldVisibilityEnum, enableDecimalLogicalType, optionalGetters, createSetters); - } catch (Exception e) { - throw new RuntimeException("Avro schema compilation failed: " + e.getMessage(), e); - } - } } diff --git a/bridge/src/main/java/com/github/sbt/avro/AvscFilesCompiler.java b/bridge/src/main/java/com/github/sbt/avro/AvscFilesCompiler.java index f515668..46cefe2 100644 --- a/bridge/src/main/java/com/github/sbt/avro/AvscFilesCompiler.java +++ b/bridge/src/main/java/com/github/sbt/avro/AvscFilesCompiler.java @@ -6,19 +6,15 @@ import org.apache.avro.generic.GenericData; import org.apache.avro.specific.SpecificData; import org.apache.avro.specific.SpecificRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.util.*; +import java.util.function.Supplier; import java.util.stream.Collectors; public class AvscFilesCompiler { - private static final Logger LOG = LoggerFactory.getLogger(AvscFilesCompiler.class); - - // private final SchemaParserBuilder builder; private Schema.Parser schemaParser; private String templateDirectory; private GenericData.StringType stringType; @@ -29,7 +25,6 @@ public class AvscFilesCompiler { private Boolean gettersReturnOptional; private Boolean optionalGettersForNullableFieldsOnly; private Map compileExceptions; - private boolean logCompileExceptions; public AvscFilesCompiler() { // this.builder = builder; @@ -66,9 +61,6 @@ public void compileFiles(Set files, File outputDirectory) { for (AvroFileRef file : uncompiledFiles) { Exception e = compileExceptions.get(file); if (e != null) { - if (logCompileExceptions) { - LOG.error(file.toString(), e); - } ex.addSuppressed(e); } } @@ -107,9 +99,6 @@ public void compileClasses(Set> classes, File ou for (Class clazz : uncompiledClasses) { Exception e = compileExceptions.get(clazz); if (e != null) { - if (logCompileExceptions) { - LOG.error(clazz.toString(), e); - } ex.addSuppressed(e); } } @@ -214,10 +203,6 @@ public void setCreateSetters(boolean createSetters) { this.createSetters = createSetters; } - public void setLogCompileExceptions(final boolean logCompileExceptions) { - this.logCompileExceptions = logCompileExceptions; - } - public void setGettersReturnOptional(final boolean gettersReturnOptional) { this.gettersReturnOptional = gettersReturnOptional; } diff --git a/bridge/src/test/scala/com/github/sbt/avro/AvroCompilerBridgeSpec.scala b/bridge/src/test/scala/com/github/sbt/avro/AvscFilesCompilerSpec.scala similarity index 79% rename from bridge/src/test/scala/com/github/sbt/avro/AvroCompilerBridgeSpec.scala rename to bridge/src/test/scala/com/github/sbt/avro/AvscFilesCompilerSpec.scala index 22fd712..c8a2aad 100644 --- a/bridge/src/test/scala/com/github/sbt/avro/AvroCompilerBridgeSpec.scala +++ b/bridge/src/test/scala/com/github/sbt/avro/AvscFilesCompilerSpec.scala @@ -3,18 +3,29 @@ package com.github.sbt.avro import com.github.sbt.avro.test.{TestSpecificRecord, TestSpecificRecordParent} import org.apache.avro.compiler.specific.SpecificCompiler.FieldVisibility import org.apache.avro.generic.GenericData.StringType +import org.apache.avro.specific.SpecificRecord import org.specs2.mutable.Specification import java.io.File import java.nio.file.Files +import scala.collection.JavaConverters._ -class AvroCompilerBridgeSpec extends Specification { +class AvscFilesCompilerSpec extends Specification { val sourceDir = new File(getClass.getClassLoader.getResource("avro").toURI) - val targetDir = Files.createTempDirectory("sbt-avro-compiler-bridge").toFile val packageDir = new File(targetDir, "com/github/sbt/avro/test") + val compiler = new AvscFilesCompiler() + compiler.setUseNamespace(false) + compiler.setStringType(StringType.CharSequence) + compiler.setFieldVisibility(FieldVisibility.PRIVATE) + compiler.setEnableDecimalLogicalType(true) + compiler.setGettersReturnOptional(true) + compiler.setOptionalGettersForNullableFieldsOnly(true) + compiler.setCreateSetters(true) + compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/") + "It should be possible to compile types depending on others if source files are provided in right order" >> { val fullyQualifiedNames = Seq( new File(sourceDir, "a.avsc"), @@ -57,17 +68,7 @@ class AvroCompilerBridgeSpec extends Specification { _eJavaFile.delete() val refs = sourceFiles.map(s => new AvroFileRef(sourceDir, s.getName)) - val compiler = new AvroCompilerBridge - compiler.compileAvscs( - refs.toArray, - targetDir, - StringType.CharSequence, - FieldVisibility.PRIVATE, - true, - false, - false, - true, - ) + compiler.compileFiles(refs.toSet.asJava, targetDir) aJavaFile.isFile must beTrue bJavaFile.isFile must beTrue @@ -84,20 +85,13 @@ class AvroCompilerBridgeSpec extends Specification { "It should be possible to compile types depending on others if classes are provided in right order" >> { // TestSpecificRecordParent and TestSpecificRecord were previously generated from test_records.avsc - val compiler = new AvroCompilerBridge - compiler.recompile( - Array( + compiler.compileClasses( + Set[Class[_ <: SpecificRecord]]( // put parent 1st classOf[TestSpecificRecordParent], classOf[TestSpecificRecord] - ), - targetDir, - StringType.CharSequence, - FieldVisibility.PRIVATE, - true, - false, - false, - true + ).asJava, + targetDir ) val record = new File(packageDir, "TestSpecificRecord.java") diff --git a/build.sbt b/build.sbt index bcaa47c..96863ee 100644 --- a/build.sbt +++ b/build.sbt @@ -61,10 +61,7 @@ lazy val `sbt-avro-compiler-api`: Project = project .in(file("api")) .settings( crossPaths := false, - autoScalaLibrary := false, - libraryDependencies ++= Seq( - Dependencies.Provided.AvroCompiler - ) + autoScalaLibrary := false ) lazy val `sbt-avro-compiler-bridge`: Project = project @@ -75,6 +72,7 @@ lazy val `sbt-avro-compiler-bridge`: Project = project autoScalaLibrary := false, libraryDependencies ++= Seq( Dependencies.Provided.AvroCompiler, + Dependencies.Provided.SbtUtilInterface, Dependencies.Test.Specs2Core, ) ) diff --git a/plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala b/plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala index 7920fe7..768d611 100644 --- a/plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala +++ b/plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala @@ -4,6 +4,7 @@ import sbt.Keys.* import sbt.{Def, *} import Path.relativeTo import sbt.librarymanagement.DependencyFilter +import xsbti.Logger import java.io.File import java.net.URLClassLoader @@ -35,7 +36,6 @@ object SbtAvro extends AutoPlugin { val avroIncludes = settingKey[Seq[File]]("Avro schema includes.") val avroOptionalGetters = settingKey[Boolean]("Generate getters that return Optional for nullable fields. Default: false.") val avroSpecificRecords = settingKey[Seq[String]]("List of avro records to recompile with current avro version and settings. Classes must be part of the Avro library dependencies.") - // val avroSchemaParserBuilder = settingKey[SchemaParserBuilder](".avsc schema parser builder") val avroSource = settingKey[File]("Default Avro source directory.") val avroStringType = settingKey[String]("Type for representing strings. Possible values: CharSequence, String, Utf8. Default: CharSequence.") val avroUnpackDependencies = taskKey[Seq[File]]("Unpack avro dependencies.") @@ -114,8 +114,7 @@ object SbtAvro extends AutoPlugin { avroEnableDecimalLogicalType := true, avroUseNamespace := false, avroOptionalGetters := false, - avroCreateSetters := true, - // avroSchemaParserBuilder := DefaultSchemaParserBuilder.default() + avroCreateSetters := true ) override lazy val projectSettings: Seq[Setting[_]] = defaultSettings ++ @@ -175,6 +174,7 @@ object SbtAvro extends AutoPlugin { private def sourceGeneratorTask(key: TaskKey[Seq[File]]) = Def.task { val out = (key / streams).value + val version = avroVersion.value val srcDir = avroSource.value val externalSrcDir = (avroUnpackDependencies / target).value val includes = avroIncludes.value @@ -219,35 +219,43 @@ object SbtAvro extends AutoPlugin { val compiler = avroClassLoader .loadClass("com.github.sbt.avro.AvroCompilerBridge") - .getDeclaredConstructor() - .newInstance() - .asInstanceOf[AvroCompiler] - - val recs = records.map(avroClassLoader.loadClass) - val avdls = srcDirs.flatMap(d => (d ** AvroAvdlFilter).get) - val avscs = srcDirs.flatMap(d => (d ** AvroAvscFilter).get.map(avsc => new AvroFileRef(d, avsc.relativeTo(d).get.toString))) - val avprs = srcDirs.flatMap(d => (d ** AvroAvrpFilter).get) - - out.log - - compiler - .compileAvroSchema( - recs.toArray, - avdls.toArray, - avscs.toArray, - avprs.toArray, - outDir, + .getDeclaredConstructor( + classOf[Logger], + classOf[String], + classOf[String], + classOf[Boolean], + classOf[Boolean], + classOf[Boolean], + classOf[Boolean], + ) + .newInstance( + out.log, strType, fieldVis, - enbDecimal, - useNs, - createSetters, - optionalGetters + Boolean.box(useNs), + Boolean.box(enbDecimal), + Boolean.box(createSetters), + Boolean.box(optionalGetters) ) + .asInstanceOf[AvroCompiler] + + try { + val recs = records.map(avroClassLoader.loadClass) + val avdls = srcDirs.flatMap(d => (d ** AvroAvdlFilter).get) + val avscs = srcDirs.flatMap(d => (d ** AvroAvscFilter).get.map(avsc => new AvroFileRef(d, avsc.relativeTo(d).get.toString))) + val avprs = srcDirs.flatMap(d => (d ** AvroAvrpFilter).get) + + out.log.info(s"Avro compiler $version using stringType=$strType") - avroClassLoader.close() + compiler.recompile(recs.toArray, outDir) + compiler.compileIdls(avdls.toArray, outDir) + compiler.compileAvscs(avscs.toArray, outDir) + compiler.compileAvprs(avprs.toArray, outDir) - (outDir ** SbtAvro.JavaFileFilter).get.toSet + (outDir ** SbtAvro.JavaFileFilter).get.toSet + } finally { + avroClassLoader.close() + } } else { outReport.checked } diff --git a/plugin/src/sbt-test/sbt-avro/basic/project/Avro.scala b/plugin/src/sbt-test/sbt-avro/basic/project/Avro.scala index 9a8c266..a06e4d7 100644 --- a/plugin/src/sbt-test/sbt-avro/basic/project/Avro.scala +++ b/plugin/src/sbt-test/sbt-avro/basic/project/Avro.scala @@ -24,4 +24,4 @@ case class Avro(version: String) extends Platform { case None => (Compile / avroSource).value }) ) -} \ No newline at end of file +} diff --git a/plugin/src/sbt-test/sbt-avro/basic/project/AvroCrossPlugin.scala b/plugin/src/sbt-test/sbt-avro/basic/project/AvroCrossPlugin.scala index 14d0536..dbe99c1 100644 --- a/plugin/src/sbt-test/sbt-avro/basic/project/AvroCrossPlugin.scala +++ b/plugin/src/sbt-test/sbt-avro/basic/project/AvroCrossPlugin.scala @@ -22,4 +22,4 @@ object AvroCrossPlugin extends sbt.AutoPlugin { project.configurePlatform(Avro(version))(transformer) } } -} \ No newline at end of file +} diff --git a/plugin/src/sbt-test/sbt-avro/basic/test b/plugin/src/sbt-test/sbt-avro/basic/test index 85c19dd..d23c116 100644 --- a/plugin/src/sbt-test/sbt-avro/basic/test +++ b/plugin/src/sbt-test/sbt-avro/basic/test @@ -1,5 +1,3 @@ -#> set avroSchemaParserBuilder := com.github.sbt.avro.NameValidatorSchemaParserBuilder(validateDefaults = false) - > avroGenerate $ exists .avro-12/target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/A.java @@ -39,10 +37,3 @@ $ exists .avro-12/target/scala-2.13/src_managed/compiled_avro/test/com/github/sb $ exists .avro-12/target/scala-2.13/test-classes/com/github/sbt/avro/test/X.class $ exists .avro-12/target/scala-2.13/test-classes/com/github/sbt/avro/test/Y.class $ exists .avro-12/target/scala-2.13/test-classes/com/github/sbt/avro/test/Z.class - -> clean - -#> set avroSchemaParserBuilder := com.github.sbt.avro.NameValidatorSchemaParserBuilder(validateDefaults = true) - -# should fail because f.avsc has invalid default value -#-> avroGenerate diff --git a/plugin/src/sbt-test/sbt-avro/sbt_1.3/test b/plugin/src/sbt-test/sbt-avro/sbt_1.3/test index f7e3997..f410585 100644 --- a/plugin/src/sbt-test/sbt-avro/sbt_1.3/test +++ b/plugin/src/sbt-test/sbt-avro/sbt_1.3/test @@ -32,5 +32,3 @@ $ exists target/scala-2.13/src_managed/compiled_avro/test/com/github/sbt/avro/te $ exists target/scala-2.13/test-classes/com/github/sbt/avro/test/X.class $ exists target/scala-2.13/test-classes/com/github/sbt/avro/test/Y.class $ exists target/scala-2.13/test-classes/com/github/sbt/avro/test/Z.class - -> clean diff --git a/plugin/src/sbt-test/sbt-avro/settings/build.sbt b/plugin/src/sbt-test/sbt-avro/settings/build.sbt index 242a8f8..4442f2c 100644 --- a/plugin/src/sbt-test/sbt-avro/settings/build.sbt +++ b/plugin/src/sbt-test/sbt-avro/settings/build.sbt @@ -1,10 +1,10 @@ -addSbtPlugins(SbtAvro) +enablePlugins(SbtAvro) name := "settings-test" scalaVersion := "2.13.11" libraryDependencies ++= Seq( // depend on test jar to get some generated records in the build - "org.apache.avro" % "avro" % avroVersion.value % Avro classifier "tests", + "org.apache.avro" % "avro" % avroVersion.value % "avro" classifier "tests", "org.specs2" %% "specs2-core" % "4.20.8" % Test ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 1606a55..8946369 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -5,10 +5,12 @@ object Dependencies { object Versions { val Avro = "1.12.0" val Specs2 = "4.20.8" + val SbtUtilInterface = "1.10.1" } object Provided { val AvroCompiler = "org.apache.avro" % "avro-compiler" % Versions.Avro % "provided" + val SbtUtilInterface = "org.scala-sbt" % "util-interface" % Versions.SbtUtilInterface % "provided" } object Test {