Skip to content

Commit

Permalink
Support avro 1.12 new schema parser builder
Browse files Browse the repository at this point in the history
  • Loading branch information
RustedBones committed Aug 12, 2024
1 parent e99c4ab commit 5e98a21
Show file tree
Hide file tree
Showing 17 changed files with 135 additions and 54 deletions.
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.github.sbt.avro.mojo;

import org.apache.avro.AvroTypeException;
import org.apache.avro.Schema;
import org.apache.avro.SchemaParseException;
import org.apache.avro.compiler.specific.SpecificCompiler;
Expand Down Expand Up @@ -122,7 +123,7 @@ private boolean tryCompile(AvroFileRef src, File outputDirectory) {
try {
schema = schemaParser.parse(src.getFile());
validateParsedSchema(src, schema);
} catch (SchemaParseException e) {
} catch (SchemaParseException | AvroTypeException e) {
schemaParser = successfulSchemaParser;
compileExceptions.put(src, e);
return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,32 +3,14 @@ package com.github.sbt.avro
import com.github.sbt.avro.mojo.SchemaParserBuilder
import org.apache.avro.Schema

import scala.collection.JavaConverters._

case class DefaultSchemaParserBuilder(types: Iterable[Schema],
validate: Boolean,
validateDefaults: Boolean)
extends SchemaParserBuilder {

override def build(): Schema.Parser = {
val parser = new Schema.Parser
parser.addTypes(types.map(el => el.getFullName() -> el).toMap.asJava)
parser.setValidate(validate)
parser.setValidateDefaults(validateDefaults)
parser
}
}

object DefaultSchemaParserBuilder {
def default(): DefaultSchemaParserBuilder = {
template(new Schema.Parser())
}

def template(template: Schema.Parser): DefaultSchemaParserBuilder = {
DefaultSchemaParserBuilder(
template.getTypes.values().asScala,
template.getValidate,
template.getValidateDefaults
)
def default(): SchemaParserBuilder = {
val Array(1, minor, _) = classOf[Schema].getPackage.getImplementationVersion.split("\\.").take(3).map(_.toInt)
if (minor >= 12) {
NameValidatorSchemaParserBuilder()
} else {
LegacySchemaParserBuilder()
}
}
}
46 changes: 46 additions & 0 deletions src/main/scala/com/github/sbt/avro/LegacySchemaParserBuilder.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package com.github.sbt.avro

import com.github.sbt.avro.mojo.SchemaParserBuilder
import org.apache.avro.Schema

import scala.annotation.nowarn
import scala.collection.JavaConverters.*

// used until avro 1.11
// for avro 2.12+ use NameValidatorSchemaParserBuilder
case class LegacySchemaParserBuilder(
types: Iterable[Schema] = LegacySchemaParserBuilder.DefaultTypes,
validate: Boolean = LegacySchemaParserBuilder.DefaultValidate,
validateDefaults: Boolean = LegacySchemaParserBuilder.DefaultValidateDefaults)
extends SchemaParserBuilder {

override def build(): Schema.Parser = {
val parser = new Schema.Parser
// addTypes(Map<String, Schema> types) is the only API available in 1.8
parser.addTypes(types.map(el => el.getFullName -> el).toMap.asJava): @nowarn
LegacySchemaParserBuilder.setValidate(parser)(validate)
parser.setValidateDefaults(validateDefaults)
parser
}
}

object LegacySchemaParserBuilder {
// validate hase been removed in 1.12 in favor of a NameValidator
private def setValidate(parser: Schema.Parser)(validate: Boolean): Schema.Parser =
classOf[Schema.Parser]
.getMethod("setValidate", classOf[Boolean])
.invoke(parser, validate: java.lang.Boolean)
.asInstanceOf[Schema.Parser]

private def getValidate(parser: Schema.Parser): Boolean =
classOf[Schema.Parser]
.getMethod("getValidate")
.invoke(parser)
.asInstanceOf[Boolean]

private val defaultParser = new Schema.Parser

private val DefaultTypes: Iterable[Schema] = defaultParser.getTypes.values().asScala
private val DefaultValidate: Boolean = getValidate(defaultParser)
private val DefaultValidateDefaults: Boolean = defaultParser.getValidateDefaults
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package com.github.sbt.avro

import com.github.sbt.avro.mojo.SchemaParserBuilder
import org.apache.avro.{NameValidator, Schema}

import scala.collection.JavaConverters._

case class NameValidatorSchemaParserBuilder(
types: Iterable[Schema] = Iterable.empty,
validation: NameValidator = NameValidator.UTF_VALIDATOR,
validateDefaults: Boolean = true
) extends SchemaParserBuilder {

override def build(): Schema.Parser = {
val parser = new Schema.Parser(validation)
parser.addTypes(types.asJava)
parser.setValidateDefaults(validateDefaults)
}
}
1 change: 0 additions & 1 deletion src/sbt-test/sbt-avro/basic_1.10/project/build.properties

This file was deleted.

1 change: 1 addition & 0 deletions src/sbt-test/sbt-avro/basic_1.10/project/build.properties
2 changes: 1 addition & 1 deletion src/sbt-test/sbt-avro/basic_1.10/test
1 change: 0 additions & 1 deletion src/sbt-test/sbt-avro/basic_1.11/project/build.properties

This file was deleted.

1 change: 1 addition & 0 deletions src/sbt-test/sbt-avro/basic_1.11/project/build.properties
1 change: 0 additions & 1 deletion src/sbt-test/sbt-avro/basic_1.11/test

This file was deleted.

42 changes: 42 additions & 0 deletions src/sbt-test/sbt-avro/basic_1.11/test
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
> set avroSchemaParserBuilder := com.github.sbt.avro.LegacySchemaParserBuilder(validateDefaults = false)
> avroGenerate

$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/A.java
$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/B.java
$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/C.java
$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/D.java
$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/E.java
$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/_A.java
$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/_B.java
$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/_C.java
$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/_D.java
$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/_E.java

> compile

$ exists target/scala-2.13/classes/com/github/sbt/avro/test/A.class
$ exists target/scala-2.13/classes/com/github/sbt/avro/test/B.class
$ exists target/scala-2.13/classes/com/github/sbt/avro/test/C.class
$ exists target/scala-2.13/classes/com/github/sbt/avro/test/D.class
$ exists target/scala-2.13/classes/com/github/sbt/avro/test/E.class
$ exists target/scala-2.13/classes/com/github/sbt/avro/test/_A.class
$ exists target/scala-2.13/classes/com/github/sbt/avro/test/_B.class
$ exists target/scala-2.13/classes/com/github/sbt/avro/test/_C.class
$ exists target/scala-2.13/classes/com/github/sbt/avro/test/_D.class
$ exists target/scala-2.13/classes/com/github/sbt/avro/test/_E.class

> Test/compile

$ exists target/scala-2.13/src_managed/compiled_avro/test/com/github/sbt/avro/test/X.java
$ exists target/scala-2.13/src_managed/compiled_avro/test/com/github/sbt/avro/test/Y.java
$ exists target/scala-2.13/src_managed/compiled_avro/test/com/github/sbt/avro/test/Z.java
$ exists target/scala-2.13/test-classes/com/github/sbt/avro/test/X.class
$ exists target/scala-2.13/test-classes/com/github/sbt/avro/test/Y.class
$ exists target/scala-2.13/test-classes/com/github/sbt/avro/test/Z.class

> clean

> set avroSchemaParserBuilder := com.github.sbt.avro.LegacySchemaParserBuilder(validateDefaults = true)

# should fail because f.avsc has invalid default value
-> avroGenerate
5 changes: 4 additions & 1 deletion src/sbt-test/sbt-avro/basic_1.8/build.sbt
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
name := "basic-test"
scalaVersion := "2.13.11"
libraryDependencies += "org.apache.avro" % "avro" % avroCompilerVersion
libraryDependencies ++= Seq(
"org.apache.avro" % "avro" % avroCompilerVersion,
"joda-time" % "joda-time" % "2.7" // marked as optional in avro pom
)
1 change: 0 additions & 1 deletion src/sbt-test/sbt-avro/basic_1.8/project/build.properties

This file was deleted.

1 change: 1 addition & 0 deletions src/sbt-test/sbt-avro/basic_1.8/project/build.properties
2 changes: 1 addition & 1 deletion src/sbt-test/sbt-avro/basic_1.8/test
5 changes: 4 additions & 1 deletion src/sbt-test/sbt-avro/basic_1.9/build.sbt
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
name := "basic-test"
scalaVersion := "2.13.11"
libraryDependencies += "org.apache.avro" % "avro" % avroCompilerVersion
libraryDependencies ++= Seq(
"org.apache.avro" % "avro" % avroCompilerVersion,
"joda-time" % "joda-time" % "2.10.1" // marked as optional in avro pom
)
1 change: 0 additions & 1 deletion src/sbt-test/sbt-avro/basic_1.9/project/build.properties

This file was deleted.

1 change: 1 addition & 0 deletions src/sbt-test/sbt-avro/basic_1.9/project/build.properties
2 changes: 1 addition & 1 deletion src/sbt-test/sbt-avro/basic_1.9/test
6 changes: 3 additions & 3 deletions src/sbt-test/sbt-avro/basic_current/test
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
> set avroSchemaParserBuilder := com.github.sbt.avro.DefaultSchemaParserBuilder.default().copy(validateDefaults = false)
> set avroSchemaParserBuilder := com.github.sbt.avro.NameValidatorSchemaParserBuilder(validateDefaults = false)
> avroGenerate

$ exists target/scala-2.13/src_managed/compiled_avro/main/com/github/sbt/avro/test/A.java
Expand All @@ -25,7 +25,7 @@ $ exists target/scala-2.13/classes/com/github/sbt/avro/test/_C.class
$ exists target/scala-2.13/classes/com/github/sbt/avro/test/_D.class
$ exists target/scala-2.13/classes/com/github/sbt/avro/test/_E.class

> test:compile
> Test/compile

$ exists target/scala-2.13/src_managed/compiled_avro/test/com/github/sbt/avro/test/X.java
$ exists target/scala-2.13/src_managed/compiled_avro/test/com/github/sbt/avro/test/Y.java
Expand All @@ -36,7 +36,7 @@ $ exists target/scala-2.13/test-classes/com/github/sbt/avro/test/Z.class

> clean

> set avroSchemaParserBuilder := com.github.sbt.avro.DefaultSchemaParserBuilder.default().copy(validateDefaults = true)
> set avroSchemaParserBuilder := com.github.sbt.avro.NameValidatorSchemaParserBuilder(validateDefaults = true)

# should fail because f.avsc has invalid default value
-> avroGenerate
2 changes: 1 addition & 1 deletion src/sbt-test/sbt-avro/settings/project/plugins.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ val avroVersion = "1.12.0"
libraryDependencies ++= Seq(
"org.apache.avro" % "avro-compiler" % avroVersion,
// depend on test jar to get some generated records in the build
"org.apache.avro" % "avro" % avroVersion, classifier "tests"
"org.apache.avro" % "avro" % avroVersion classifier "tests"
)
14 changes: 0 additions & 14 deletions src/test/scala/com/github/sbt/avro/SbtAvroSpec.scala
Original file line number Diff line number Diff line change
Expand Up @@ -34,20 +34,6 @@ class SbtAvroSpec extends Specification {
new File(sourceDir, "_d.avsc"),
new File(sourceDir, "_e.avsc"))

val expectedOrderFullyQualifiedNames = Seq(
new File(sourceDir, "c.avsc"),
new File(sourceDir, "e.avsc"),
new File(sourceDir, "d.avsc"),
new File(sourceDir, "b.avsc"),
new File(sourceDir, "a.avsc"))

val expectedOrderSimpleNames = Seq(
new File(sourceDir, "_c.avsc"),
new File(sourceDir, "_e.avsc"),
new File(sourceDir, "_d.avsc"),
new File(sourceDir, "_b.avsc"),
new File(sourceDir, "_a.avsc"))

val sourceFiles = fullyQualifiedNames ++ simpleNames

val aJavaFile = new File(packageDir, "A.java")
Expand Down

0 comments on commit 5e98a21

Please sign in to comment.