Handle multiple avro compiler versions in the same build

sbt · Sep 11, 2024 · 11a406d · 11a406d
1 parent 336688e
commit 11a406d
Show file tree

Hide file tree

Showing 106 changed files with 429 additions and 469 deletions.
diff --git a/api/src/main/java/com/github/sbt/avro/AvroCompiler.java b/api/src/main/java/com/github/sbt/avro/AvroCompiler.java
@@ -0,0 +1,20 @@
+package com.github.sbt.avro;
+
+import java.io.File;
+import java.io.IOException;
+
+public interface AvroCompiler {
+    void compileAvroSchema(
+            Class<?>[] records,
+            File[] avdls,
+            AvroFileRef[] avscs,
+            File[] avprs,
+            File target,
+            String stringType,
+            String fieldVisibility,
+            Boolean enableDecimalLogicalType,
+            Boolean useNamespace,
+            Boolean optionalGetters,
+            Boolean createSetters
+    ) throws IOException;
+}
diff --git a/...com/github/sbt/avro/mojo/AvroFileRef.java → ...java/com/github/sbt/avro/AvroFileRef.java b/...com/github/sbt/avro/mojo/AvroFileRef.java → ...java/com/github/sbt/avro/AvroFileRef.java
@@ -1,4 +1,4 @@
-package com.github.sbt.avro.mojo;
+package com.github.sbt.avro;
 
 import java.io.File;
 import java.util.Objects;

diff --git a/bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java b/bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java
@@ -0,0 +1,156 @@
+package com.github.sbt.avro;
+
+import org.apache.avro.Protocol;
+import org.apache.avro.compiler.idl.Idl;
+import org.apache.avro.compiler.idl.ParseException;
+import org.apache.avro.compiler.specific.SpecificCompiler;
+import org.apache.avro.compiler.specific.SpecificCompiler.FieldVisibility;
+import org.apache.avro.generic.GenericData.StringType;
+import org.apache.avro.specific.SpecificRecord;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+
+public class AvroCompilerBridge implements AvroCompiler {
+
+    void recompile(
+            Class<? extends SpecificRecord>[] records,
+            File target,
+            StringType stringType,
+            FieldVisibility fieldVisibility,
+            Boolean enableDecimalLogicalType,
+            Boolean useNamespace,
+            Boolean optionalGetters,
+            Boolean createSetters
+            // builder: SchemaParserBuilder
+    ) {
+        AvscFilesCompiler compiler = new AvscFilesCompiler();
+        compiler.setStringType(stringType);
+        compiler.setFieldVisibility(fieldVisibility);
+        compiler.setUseNamespace(useNamespace);
+        compiler.setEnableDecimalLogicalType(enableDecimalLogicalType);
+        compiler.setCreateSetters(createSetters);
+        if (AvroVersion.getMinor() > 8) {
+            compiler.setGettersReturnOptional(optionalGetters);
+        }
+        if (AvroVersion.getMinor() > 9) {
+            compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters);
+        }
+        compiler.setLogCompileExceptions(true);
+        compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/");
+        compiler.compileClasses(Set.of(records), target);
+    }
+
+    void compileIdls(
+        File[] idls,
+        File target,
+        StringType stringType,
+        FieldVisibility fieldVisibility,
+        Boolean enableDecimalLogicalType,
+        Boolean optionalGetters,
+        Boolean createSetters
+    ) throws IOException, ParseException {
+        for (File idl : idls) {
+//            log.info(s"Compiling Avro IDL $idl")
+            Idl parser = new Idl(idl);
+            Protocol protocol = parser.CompilationUnit();
+            SpecificCompiler compiler = new SpecificCompiler(protocol);
+            compiler.setStringType(stringType);
+            compiler.setFieldVisibility(fieldVisibility);
+            compiler.setEnableDecimalLogicalType(enableDecimalLogicalType);
+            compiler.setCreateSetters(createSetters);
+            if (AvroVersion.getMinor() > 8) {
+                compiler.setGettersReturnOptional(optionalGetters);
+            }
+            if (AvroVersion.getMinor() > 9) {
+                compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters);
+            }
+            compiler.compileToDestination(null, target);
+        }
+    }
+
+    void compileAvscs(
+            AvroFileRef[] avscs,
+            File target,
+            StringType stringType,
+            FieldVisibility fieldVisibility,
+            Boolean enableDecimalLogicalType,
+            Boolean useNamespace,
+            Boolean optionalGetters,
+            Boolean createSetters
+            // builder: SchemaParserBuilder
+    ) {
+        AvscFilesCompiler compiler = new AvscFilesCompiler();
+        compiler.setStringType(stringType);
+        compiler.setFieldVisibility(fieldVisibility);
+        compiler.setUseNamespace(useNamespace);
+        compiler.setEnableDecimalLogicalType(enableDecimalLogicalType);
+        compiler.setCreateSetters(createSetters);
+        if (AvroVersion.getMinor() > 8) {
+            compiler.setGettersReturnOptional(optionalGetters);
+        }
+        if (AvroVersion.getMinor() > 9) {
+            compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters);
+        }
+        compiler.setLogCompileExceptions(true);
+        compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/");
+
+        compiler.compileFiles(Set.of(avscs), target);
+    }
+
+    void compileAvprs(
+            File[] avprs,
+            File target,
+            StringType stringType,
+            FieldVisibility fieldVisibility,
+            Boolean enableDecimalLogicalType,
+            Boolean optionalGetters,
+            Boolean createSetters
+    ) throws IOException {
+        for (File avpr : avprs) {
+//            log.info(s"Compiling Avro protocol $avpr")
+            Protocol protocol = Protocol.parse(avpr);
+            SpecificCompiler compiler = new SpecificCompiler(protocol);
+            compiler.setStringType(stringType);
+            compiler.setFieldVisibility(fieldVisibility);
+            compiler.setEnableDecimalLogicalType(enableDecimalLogicalType);
+            compiler.setCreateSetters(createSetters);
+            if (AvroVersion.getMinor() > 8) {
+                compiler.setGettersReturnOptional(optionalGetters);
+            }
+            if (AvroVersion.getMinor() > 9) {
+                compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters);
+            }
+            compiler.compileToDestination(null, target);
+        }
+    }
+
+
+    @Override
+    public void compileAvroSchema(
+            Class<?>[] records,
+            File[] avdls,
+            AvroFileRef[] avscs,
+            File[] avprs,
+            File target,
+            String stringType,
+            String fieldVisibility,
+            Boolean enableDecimalLogicalType,
+            Boolean useNamespace,
+            Boolean optionalGetters,
+            Boolean createSetters) {
+        StringType stringTypeEnum = StringType.valueOf(stringType);
+        FieldVisibility fieldVisibilityEnum = FieldVisibility.valueOf(fieldVisibility);
+
+        try {
+            recompile((Class<? extends SpecificRecord>[]) records, target, stringTypeEnum, fieldVisibilityEnum, enableDecimalLogicalType, useNamespace, optionalGetters, createSetters);
+
+            compileIdls(avdls, target, stringTypeEnum, fieldVisibilityEnum, enableDecimalLogicalType, optionalGetters, createSetters);
+            compileAvscs(avscs, target,stringTypeEnum, fieldVisibilityEnum, enableDecimalLogicalType, useNamespace, optionalGetters, createSetters);
+            compileAvprs(avprs, target, stringTypeEnum, fieldVisibilityEnum, enableDecimalLogicalType, optionalGetters, createSetters);
+        } catch (Exception e) {
+            throw new RuntimeException("Avro schema compilation failed: " + e.getMessage(), e);
+        }
+    }
+}
diff --git a/bridge/src/main/java/com/github/sbt/avro/AvroVersion.java b/bridge/src/main/java/com/github/sbt/avro/AvroVersion.java
@@ -0,0 +1,17 @@
+package com.github.sbt.avro;
+
+import org.apache.avro.Schema;
+
+public class AvroVersion {
+
+    private static String[] AVRO_VERSION_PARTS =
+            Schema.class.getPackage().getImplementationVersion().split("\\.", 3);
+
+    static int getMajor() {
+        return Integer.parseInt(AVRO_VERSION_PARTS[0]);
+    }
+
+    static int getMinor() {
+        return Integer.parseInt(AVRO_VERSION_PARTS[1]);
+    }
+}
diff --git a/...thub/sbt/avro/mojo/AvscFilesCompiler.java → ...om/github/sbt/avro/AvscFilesCompiler.java b/...thub/sbt/avro/mojo/AvscFilesCompiler.java → ...om/github/sbt/avro/AvscFilesCompiler.java
@@ -1,9 +1,7 @@
-package com.github.sbt.avro.mojo;
+package com.github.sbt.avro;
 
 import org.apache.avro.AvroRuntimeException;
-import org.apache.avro.AvroTypeException;
 import org.apache.avro.Schema;
-import org.apache.avro.SchemaParseException;
 import org.apache.avro.compiler.specific.SpecificCompiler;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.specific.SpecificData;
@@ -20,21 +18,22 @@ public class AvscFilesCompiler {
 
   private static final Logger LOG = LoggerFactory.getLogger(AvscFilesCompiler.class);
 
-  private final SchemaParserBuilder builder;
+  // private final SchemaParserBuilder builder;
   private Schema.Parser schemaParser;
   private String templateDirectory;
   private GenericData.StringType stringType;
   private SpecificCompiler.FieldVisibility fieldVisibility;
   private boolean useNamespace;
   private boolean enableDecimalLogicalType;
   private boolean createSetters;
-  private Optional<Boolean> optionalGetters = Optional.empty();
+  private Boolean gettersReturnOptional;
+  private Boolean optionalGettersForNullableFieldsOnly;
   private Map<AvroFileRef, Exception> compileExceptions;
   private boolean logCompileExceptions;
 
-  public AvscFilesCompiler(SchemaParserBuilder builder) {
-    this.builder = builder;
-    this.schemaParser = builder.build();
+  public AvscFilesCompiler() {
+    // this.builder = builder;
+    this.schemaParser = new Schema.Parser(); //builder.build();
   }
 
   public void compileFiles(Set<AvroFileRef> files, File outputDirectory) {
@@ -142,9 +141,12 @@ private boolean tryCompile(File src, Schema schema, File outputDirectory) {
     compiler.setFieldVisibility(fieldVisibility);
     compiler.setEnableDecimalLogicalType(enableDecimalLogicalType);
     compiler.setCreateSetters(createSetters);
-    if (optionalGetters.isPresent()) {
-      compiler.setGettersReturnOptional(optionalGetters.get());
-      compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters.get());
+
+    if (gettersReturnOptional != null) {
+      compiler.setGettersReturnOptional(gettersReturnOptional);
+    }
+    if (optionalGettersForNullableFieldsOnly != null) {
+      compiler.setOptionalGettersForNullableFieldsOnly(optionalGettersForNullableFieldsOnly);
     }
 
     try {
@@ -160,7 +162,7 @@ private boolean tryCompile(File src, Schema schema, File outputDirectory) {
   private Schema.Parser stashParser() {
     // on failure Schema.Parser changes cache state.
     // We want last successful state.
-    Schema.Parser parser = builder.build();
+    Schema.Parser parser = new Schema.Parser(); // builder.build();
     Set<String> predefinedTypes = parser.getTypes().keySet();
     Map<String, Schema> compiledTypes = schemaParser.getTypes();
     compiledTypes.keySet().removeAll(predefinedTypes);
@@ -216,7 +218,11 @@ public void setLogCompileExceptions(final boolean logCompileExceptions) {
     this.logCompileExceptions = logCompileExceptions;
   }
 
-  public void setOptionalGetters(final boolean optionalGetters) {
-    this.optionalGetters = Optional.of(optionalGetters);
+  public void setGettersReturnOptional(final boolean gettersReturnOptional) {
+    this.gettersReturnOptional = gettersReturnOptional;
+  }
+
+  public void setOptionalGettersForNullableFieldsOnly(final boolean optionalGettersForNullableFieldsOnly) {
+    this.optionalGettersForNullableFieldsOnly = optionalGettersForNullableFieldsOnly;
   }
 }
diff --git a/.../avro/mojo/SchemaGenerationException.java → ...b/sbt/avro/SchemaGenerationException.java b/.../avro/mojo/SchemaGenerationException.java → ...b/sbt/avro/SchemaGenerationException.java
@@ -1,4 +1,4 @@
-package com.github.sbt.avro.mojo;
+package com.github.sbt.avro;
 
 public class SchemaGenerationException extends RuntimeException {
 

diff --git a/build.sbt b/build.sbt
@@ -4,9 +4,31 @@ ThisBuild / version := {
   val orig = (ThisBuild / version).value
   if (orig.endsWith("-SNAPSHOT")) "3.5.1-SNAPSHOT" else orig
 }
-ThisBuild / scalaVersion := "2.12.19"
+
+// metadata
+ThisBuild / organization := "com.github.sbt"
+ThisBuild / organizationName := "sbt"
+ThisBuild / organizationHomepage := Some(url("https://www.scala-sbt.org/"))
+ThisBuild / homepage := Some(url("https://github.com/sbt/sbt-avro"))
+ThisBuild / licenses += ("BSD 3-Clause", url("https://github.com/sbt/sbt-avro/blob/main/LICENSE"))
+ThisBuild / scmInfo := Some(ScmInfo(url("https://github.com/sbt/sbt-avro"), "scm:git:[email protected]:sbt/sbt-avro.git"))
+ThisBuild / developers := List(
+  Developer(
+    id = "nevillelyh",
+    name = "Neville Li",
+    email = "@nevillelyh",
+    url = url("https://www.lyh.me/")
+  ),
+  Developer(
+    id = "RustedBones",
+    name = "Michel Davit",
+    email = "[email protected]",
+    url = url("https://michel.davit.fr")
+  )
+)
 
 // sbt-github-actions
+ThisBuild / scalaVersion := "2.12.19"
 ThisBuild / githubWorkflowBuild := Seq(
   WorkflowStep.Sbt(name = Some("Build project"), commands = List("compile", "test", "scripted"))
 )
@@ -24,36 +46,41 @@ ThisBuild / githubWorkflowPublish := Seq(WorkflowStep.Sbt(
   )
 ))
 
+lazy val `avro-compiler-api`: Project = project
+  .in(file("api"))
+  .settings(
+    crossPaths := false,
+    autoScalaLibrary := false,
+    libraryDependencies ++= Seq(
+      Dependencies.Provided.AvroCompiler
+    )
+  )
+
+lazy val `avro-compiler-bridge`: Project = project
+  .in(file("bridge"))
+  .dependsOn(`avro-compiler-api`)
+  .settings(
+    crossPaths := false,
+    autoScalaLibrary := false,
+    libraryDependencies ++= Seq(
+      Dependencies.Provided.AvroCompiler
+    )
+  )
+
 lazy val `sbt-avro`: Project = project
-    .in(file("."))
+    .in(file("plugin"))
+    .dependsOn(
+      `avro-compiler-api`,
+      `avro-compiler-bridge` % "test"
+    )
     .enablePlugins(SbtPlugin)
     .settings(
-      organization := "com.github.sbt",
-      organizationName := "sbt",
-      organizationHomepage := Some(url("https://www.scala-sbt.org/")),
-      homepage := Some(url("https://github.com/sbt/sbt-avro")),
-      licenses += ("BSD 3-Clause", url("https://github.com/sbt/sbt-avro/blob/main/LICENSE")),
       description := "Sbt plugin for compiling Avro sources",
-      scmInfo := Some(ScmInfo(url("https://github.com/sbt/sbt-avro"), "scm:git:[email protected]:sbt/sbt-avro.git")),
-      developers := List(
-        Developer(
-          id = "nevillelyh",
-          name = "Neville Li",
-          email = "@nevillelyh",
-          url = url("https://www.lyh.me/")
-        ),
-        Developer(
-          id = "RustedBones",
-          name = "Michel Davit",
-          email = "[email protected]",
-          url = url("https://michel.davit.fr")
-        )
-      ),
       pluginCrossBuild / sbtVersion := "1.3.0",
       Compile / scalacOptions ++= Seq("-deprecation"),
       libraryDependencies ++= Seq(
-        Dependencies.Provided.AvroCompiler,
-        Dependencies.Test.Specs2Core
+        Dependencies.Test.Specs2Core,
+        Dependencies.Test.AvroCompiler
       ),
       scriptedLaunchOpts ++= Seq(
         "-Xmx1024M",