Modulable avro plugin

.github/workflows/ci.yml

@@ -53,7 +53,7 @@ jobs:
         run: sbt '++ ${{ matrix.scala }}' compile test scripted
 
       - name: Compress target directories
-        run: tar cf targets.tar target project/target
+        run: tar cf targets.tar target api/target bridge/target plugin/target project/target
 
       - name: Upload target directories
         uses: actions/upload-artifact@v4

README.md

@@ -18,46 +18,58 @@ For instance, add the following lines to `project/plugins.sbt`:
 
 ```
 addSbtPlugin("com.github.sbt" % "sbt-avro" % "3.5.0")
-
-// Java sources compiled with one version of Avro might be incompatible with a
-// different version of the Avro library. Therefore we specify the compiler
-// version here explicitly.
-libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.12.0"
 ```
 
-Add the library dependency to `build.sbt`:
+Enable the plugin in your `build.sbt` and select the desired avro version to use:
 
 ```
-libraryDependencies += "org.apache.avro" % "avro" % avroCompilerVersion
+enablePlugins(SbtAvro)
+avroVersion := "1.12.0"
 ```
 
+## Config
+
+An `avro` configuration will be added to the project. Libraries defined with this scope will be loaded by the sbt plugin
+to generate the avro classes.
+
 ## Settings
 
-| Name                                       | Default                                       | Description                                                                             |
-|:-------------------------------------------|:----------------------------------------------|:----------------------------------------------------------------------------------------|
-| `avroSource`                               | `sourceDirectory` / `avro`                    | Source directory with `*.avsc`, `*.avdl` and `*.avpr` files.                            |
-| `avroSpecificRecords`                      | `Seq.empty`                                   | List of avro generated classes to recompile with current avro version and settings.     |
-| `avroSchemaParserBuilder`                  | `DefaultSchemaParserBuilder.default()`        | `.avsc` schema parser builder                                                           |
-| `avroUnpackDependencies` / `includeFilter` | All avro specifications                       | Avro specification files from dependencies to unpack                                    |
-| `avroUnpackDependencies` / `excludeFilter` | Hidden files                                  | Avro specification files from dependencies to exclude from unpacking                    |
-| `avroUnpackDependencies` / `target`        | `sourceManaged` / `avro` / `$config`          | Target directory for schemas packaged in the dependencies                               |
-| `avroGenerate` / `target`                  | `sourceManaged` / `compiled_avro` / `$config` | Source directory for generated `.java` files.                                           |
-| `avroDependencyIncludeFilter`              | `source` typed `avro` classifier artifacts    | Dependencies containing avro schema to be unpacked for generation                       |
-| `avroIncludes`                             | `Seq()`                                       | Paths with extra `*.avsc` files to be included in compilation.                          |
-| `packageAvro` / `artifactClassifier`       | `Some("avro")`                                | Classifier for avro artifact                                                            |
-| `packageAvro` / `publishArtifact`          | `false`                                       | Enable / Disable avro artifact publishing                                               |
-| `avroStringType`                           | `CharSequence`                                | Type for representing strings. Possible values: `CharSequence`, `String`, `Utf8`.       |
-| `avroUseNamespace`                         | `false`                                       | Validate that directory layout reflects namespaces, i.e. `com/myorg/MyRecord.avsc`.     |
-| `avroFieldVisibility`                      | `public`                                      | Field Visibility for the properties. Possible values: `private`, `public`.              |
-| `avroEnableDecimalLogicalType`             | `true`                                        | Use `java.math.BigDecimal` instead of `java.nio.ByteBuffer` for logical type `decimal`. |
-| `avroOptionalGetters`                      | `false` (requires avro `1.10+`)               | Generate getters that return `Optional` for nullable fields.                            |
-
-## Tasks
+### Project settings
+
+| Name                           | Default                                                                  | Description                                                                             |
+|:-------------------------------|:-------------------------------------------------------------------------|:----------------------------------------------------------------------------------------|
+| `avroAdditionalDependencies`   | `avro-compiler % avroVersion % "avro"`, `avro % avroVersion % "compile"` | Additional dependencies to be added to library dependencies.                            | 
+| `avroCompiler`                 | `com.github.sbt.avro.AvroCompilerBridge`                                 | Sbt avro compiler class.                                                                |
+| `avroCreateSetters`            | `true`                                                                   | Generate setters.                                                                       |
+| `avroDependencyIncludeFilter`  | `source` typed `avro` classifier artifacts                               | Filter for including modules containing avro dependencies.                              |
+| `avroEnableDecimalLogicalType` | `true`                                                                   | Use `java.math.BigDecimal` instead of `java.nio.ByteBuffer` for logical type `decimal`. |
+| `avroFieldVisibility`          | `public`                                                                 | Field visibility for the properties. Possible values: `private`, `public`.              |
+| `avroOptionalGetters`          | `false` (requires avro `1.10+`)                                          | Generate getters that return `Optional` for nullable fields.                            |
+| `avroStringType`               | `CharSequence`                                                           | Type for representing strings. Possible values: `CharSequence`, `String`, `Utf8`.       |
+| `avroUseNamespace`             | `false`                                                                  | Validate that directory layout reflects namespaces, i.e. `com/myorg/MyRecord.avsc`.     |
+| `avroVersion`                  | `1.12.0`                                                                 | Avro version to use in the project.                                                     |
+
+### Scoped settings (Compile/Test)
+
+| Name                                       | Default                                       | Description                                                                                          |
+|:-------------------------------------------|:----------------------------------------------|:-----------------------------------------------------------------------------------------------------|
+| `avroGenerate` / `target`                  | `sourceManaged` / `compiled_avro` / `$config` | Source directory for generated `.java` files.                                                        |
+| `avroSource`                               | `sourceDirectory` / `$config` / `avro`        | Default Avro source directory for `*.avsc`, `*.avdl` and `*.avpr` files.                             |
+| `avroSpecificRecords`                      | `Seq.empty`                                   | List of fully qualified Avro record class names to recompile with current avro version and settings. |
+| `avroUmanagedSourceDirectories`            | `Seq(avroSource)`                             | Unmanaged Avro source directories, which contain manually created sources.                           |
+| `avroUnpackDependencies` / `excludeFilter` | `HiddenFileFilter`                            | Filter for excluding avro specification files from unpacking.                                        |
+| `avroUnpackDependencies` / `includeFilter` | `AllPassFilter`                               | Filter for including avro specification files to unpack.                                             |
+| `avroUnpackDependencies` / `target`        | `sourceManaged` / `avro` / `$config`          | Target directory for schemas packaged in the dependencies                                            |
+| `packageAvro` / `artifactClassifier`       | `Some("avro")`                                | Classifier for avro artifact                                                                         |
+| `packageAvro` / `publishArtifact`          | `false`                                       | Enable / Disable avro artifact publishing                                                            |
+
+
+## Scoped Tasks (Compile/Test)
 
 | Name                     | Description                                                                                       |
 |:-------------------------|:--------------------------------------------------------------------------------------------------|
-| `avroUnpackDependencies` | Unpack avro schemas from dependencies. This task is automatically executed before `avroGenerate`. |
 | `avroGenerate`           | Generate Java sources for Avro schemas. This task is automatically executed before `compile`.     |
+| `avroUnpackDependencies` | Unpack avro schemas from dependencies. This task is automatically executed before `avroGenerate`. |
 | `packageAvro`            | Produces an avro artifact, such as a jar containing avro schemas.                                 |
 
 ## Examples
@@ -72,7 +84,7 @@ If you depend on an artifact with previously generated avro java classes with st
 you can recompile them with `String` by also adding the following
 
 ```sbt
-Compile / avroSpecificRecords += classOf[com.example.MyAvroRecord] // lib must be declared in project/plugins.sbt
+Compile / avroSpecificRecords += "com.example.MyAvroRecord" // lib must be added in the avro scope
 ```
 
 ## Packaging Avro files

api/src/main/java/com/github/sbt/avro/AvroCompiler.java

@@ -0,0 +1,18 @@
+package com.github.sbt.avro;
+
+import java.io.File;
+
+public interface AvroCompiler {
+
+    void setStringType(String stringType);
+    void setFieldVisibility(String fieldVisibility);
+    void setUseNamespace(boolean useNamespace);
+    void setEnableDecimalLogicalType(boolean enableDecimalLogicalType);
+    void setCreateSetters(boolean createSetters);
+    void setOptionalGetters(boolean optionalGetters);
+
+    void recompile(Class<?>[] records, File target) throws Exception;
+    void compileIdls(File[] idls, File target) throws Exception;
+    void compileAvscs(AvroFileRef[] avscs, File target) throws Exception;
+    void compileAvprs(File[] avprs, File target) throws Exception;
+}

src/main/java/com/github/sbt/avro/mojo/AvroFileRef.java → api/src/main/java/com/github/sbt/avro/AvroFileRef.java

@@ -1,4 +1,4 @@
-package com.github.sbt.avro.mojo;
+package com.github.sbt.avro;
 
 import java.io.File;
 import java.util.Objects;

bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java

@@ -0,0 +1,152 @@
+package com.github.sbt.avro;
+
+import org.apache.avro.Schema;
+import org.apache.avro.specific.SpecificRecord;
+
+import org.apache.avro.Protocol;
+import org.apache.avro.compiler.idl.Idl;
+import org.apache.avro.compiler.specific.SpecificCompiler;
+import org.apache.avro.compiler.specific.SpecificCompiler.FieldVisibility;
+import org.apache.avro.generic.GenericData.StringType;
+
+import java.io.File;
+import java.util.HashSet;
+import java.util.Set;
+
+public class AvroCompilerBridge implements AvroCompiler {
+
+    private static final AvroVersion AVRO_1_9_0 = new AvroVersion(1, 9, 0);
+    private static final AvroVersion AVRO_1_10_0 = new AvroVersion(1, 10, 0);
+
+    private final AvroVersion avroVersion = AvroVersion.getRuntimeVersion();
+
+    private StringType stringType;
+    private FieldVisibility fieldVisibility;
+    private boolean useNamespace;
+    private boolean enableDecimalLogicalType;
+    private boolean createSetters;
+    private boolean optionalGetters;
+
+    protected Schema.Parser createParser() {
+        return new Schema.Parser();
+    }
+
+    @Override
+    public void setStringType(String stringType) {
+        this.stringType = StringType.valueOf(stringType);
+    }
+
+    @Override
+    public void setFieldVisibility(String fieldVisibility) {
+        this.fieldVisibility = FieldVisibility.valueOf(fieldVisibility);
+    }
+
+    @Override
+    public void setUseNamespace(boolean useNamespace) {
+        this.useNamespace = useNamespace;
+    }
+
+    @Override
+    public void setEnableDecimalLogicalType(boolean enableDecimalLogicalType) {
+        this.enableDecimalLogicalType = enableDecimalLogicalType;
+    }
+
+    @Override
+    public void setCreateSetters(boolean createSetters) {
+        this.createSetters = createSetters;
+    }
+
+    @Override
+    public void setOptionalGetters(boolean optionalGetters) {
+        this.optionalGetters = optionalGetters;
+    }
+
+    @Override
+    public void recompile(Class<?>[] records, File target) throws Exception {
+        AvscFilesCompiler compiler = new AvscFilesCompiler(this::createParser);
+        compiler.setStringType(stringType);
+        compiler.setFieldVisibility(fieldVisibility);
+        compiler.setUseNamespace(useNamespace);
+        compiler.setEnableDecimalLogicalType(enableDecimalLogicalType);
+        compiler.setCreateSetters(createSetters);
+        if (avroVersion.compareTo(AVRO_1_9_0) >= 0) {
+            compiler.setGettersReturnOptional(optionalGetters);
+        }
+        if (avroVersion.compareTo(AVRO_1_10_0) >= 0) {
+            compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters);
+        }
+        compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/");
+
+        Set<Class<? extends SpecificRecord>> classes = new HashSet<>();
+        for (Class<?> record : records) {
+            System.out.println("Recompiling Avro record: " + record.getName());
+            classes.add((Class<? extends SpecificRecord>) record);
+        }
+        compiler.compileClasses(classes, target);
+    }
+
+    @Override
+    public void compileIdls(File[] idls, File target) throws Exception {
+        for (File idl : idls) {
+            System.out.println("Compiling Avro IDL: " + idl);
+            Idl parser = new Idl(idl);
+            Protocol protocol = parser.CompilationUnit();
+            SpecificCompiler compiler = new SpecificCompiler(protocol);
+            compiler.setStringType(stringType);
+            compiler.setFieldVisibility(fieldVisibility);
+            compiler.setEnableDecimalLogicalType(enableDecimalLogicalType);
+            compiler.setCreateSetters(createSetters);
+            if (avroVersion.compareTo(AVRO_1_9_0) >= 0) {
+                compiler.setGettersReturnOptional(optionalGetters);
+            }
+            if (avroVersion.compareTo(AVRO_1_10_0) >= 0) {
+                compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters);
+            }
+            compiler.compileToDestination(null, target);
+        }
+    }
+
+    @Override
+    public void compileAvscs(AvroFileRef[] avscs, File target) throws Exception {
+        AvscFilesCompiler compiler = new AvscFilesCompiler(this::createParser);
+        compiler.setStringType(stringType);
+        compiler.setFieldVisibility(fieldVisibility);
+        compiler.setUseNamespace(useNamespace);
+        compiler.setEnableDecimalLogicalType(enableDecimalLogicalType);
+        compiler.setCreateSetters(createSetters);
+        if (avroVersion.compareTo(AVRO_1_9_0) >= 0) {
+            compiler.setGettersReturnOptional(optionalGetters);
+        }
+        if (avroVersion.compareTo(AVRO_1_10_0) >= 0) {
+            compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters);
+        }
+        compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/");
+
+        Set<AvroFileRef> files = new HashSet<>();
+        for (AvroFileRef ref : avscs) {
+            System.out.println("Compiling Avro schema: " + ref.getFile());
+            files.add(ref);
+        }
+        compiler.compileFiles(Set.of(avscs), target);
+    }
+
+    @Override
+    public void compileAvprs(File[] avprs, File target) throws Exception {
+        for (File avpr : avprs) {
+            System.out.println("Compiling Avro protocol: " + avpr);
+            Protocol protocol = Protocol.parse(avpr);
+            SpecificCompiler compiler = new SpecificCompiler(protocol);
+            compiler.setStringType(stringType);
+            compiler.setFieldVisibility(fieldVisibility);
+            compiler.setEnableDecimalLogicalType(enableDecimalLogicalType);
+            compiler.setCreateSetters(createSetters);
+            if (avroVersion.compareTo(AVRO_1_9_0) >= 0) {
+                compiler.setGettersReturnOptional(optionalGetters);
+            }
+            if (avroVersion.compareTo(AVRO_1_10_0) >= 0) {
+                compiler.setOptionalGettersForNullableFieldsOnly(optionalGetters);
+            }
+            compiler.compileToDestination(null, target);
+        }
+    }
+}

bridge/src/main/java/com/github/sbt/avro/AvroVersion.java

@@ -0,0 +1,48 @@
+package com.github.sbt.avro;
+
+import org.apache.avro.Schema;
+
+public class AvroVersion implements Comparable<AvroVersion> {
+
+    private final int major;
+    private final int minor;
+    private final int patch;
+
+    public AvroVersion(int major, int minor, int patch) {
+        this.major = major;
+        this.minor = minor;
+        this.patch = patch;
+    }
+
+    int getMajor() {
+        return major;
+    }
+
+    int getMinor() {
+        return minor;
+    }
+
+    int getPatch() {
+        return patch;
+    }
+
+    static AvroVersion getRuntimeVersion() {
+        String[] parts = Schema.class.getPackage().getImplementationVersion().split("\\.", 3);
+        int major = Integer.parseInt(parts[0]);
+        int minor = Integer.parseInt(parts[1]);
+        int patch = Integer.parseInt(parts[2]);
+        return new AvroVersion(major, minor, patch);
+    }
+
+
+    @Override
+    public int compareTo(AvroVersion o) {
+        if (major != o.major) {
+            return major - o.major;
+        } else if (minor != o.minor) {
+            return minor - o.minor;
+        } else {
+            return patch - o.patch;
+        }
+    }
+}