From 9bf76f872f43d06f937877c1d226669a943e6ff9 Mon Sep 17 00:00:00 2001 From: Michel Davit Date: Fri, 8 Nov 2024 09:23:43 +0100 Subject: [PATCH] Use avro scope for external avro sources (#211) * Use avro scope for external avro sources Rely on the Avro scope to compile external sources. If avro sources are declared as Compile or Test dependencies, the plugin would automatically recompile sources, leading to duplicated or even conflicting classes. * Revert test jar dependency con compile schemas * Tune test to pull schema jar transitively --- README.md | 38 +++++++++++-------- .../scala/com/github/sbt/avro/SbtAvro.scala | 22 +++++++---- .../sbt-test/sbt-avro/publishing/build.sbt | 37 ++++++++++-------- .../transitive/src/test/resources/test.avsc | 2 +- 4 files changed, 60 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 56b1ddf..8887c61 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,6 @@ to generate the avro classes. | `avroAdditionalDependencies` | `avro-compiler % avroVersion % "avro"`, `avro % avroVersion % "compile"` | Additional dependencies to be added to library dependencies. | | `avroCompiler` | `com.github.sbt.avro.AvroCompilerBridge` | Sbt avro compiler class. | | `avroCreateSetters` | `true` | Generate setters. | -| `avroDependencyIncludeFilter` | `source` typed `avro` classifier artifacts | Filter for including modules containing avro dependencies. | | `avroEnableDecimalLogicalType` | `true` | Use `java.math.BigDecimal` instead of `java.nio.ByteBuffer` for logical type `decimal`. | | `avroFieldVisibility` | `public` | Field visibility for the properties. Possible values: `private`, `public`. | | `avroOptionalGetters` | `false` (requires avro `1.10+`) | Generate getters that return `Optional` for nullable fields. | @@ -51,17 +50,18 @@ to generate the avro classes. ### Scoped settings (Compile/Test) -| Name | Default | Description | -|:-------------------------------------------|:----------------------------------------------|:-----------------------------------------------------------------------------------------------------| -| `avroGenerate` / `target` | `sourceManaged` / `compiled_avro` / `$config` | Source directory for generated `.java` files. | -| `avroSource` | `sourceDirectory` / `$config` / `avro` | Default Avro source directory for `*.avsc`, `*.avdl` and `*.avpr` files. | -| `avroSpecificRecords` | `Seq.empty` | List of fully qualified Avro record class names to recompile with current avro version and settings. | -| `avroUmanagedSourceDirectories` | `Seq(avroSource)` | Unmanaged Avro source directories, which contain manually created sources. | -| `avroUnpackDependencies` / `excludeFilter` | `HiddenFileFilter` | Filter for excluding avro specification files from unpacking. | -| `avroUnpackDependencies` / `includeFilter` | `AllPassFilter` | Filter for including avro specification files to unpack. | -| `avroUnpackDependencies` / `target` | `sourceManaged` / `avro` / `$config` | Target directory for schemas packaged in the dependencies | -| `packageAvro` / `artifactClassifier` | `Some("avro")` | Classifier for avro artifact | -| `packageAvro` / `publishArtifact` | `false` | Enable / Disable avro artifact publishing | +| Name | Default | Description | +|:-------------------------------------------|:------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------| +| `avroGenerate` / `target` | `sourceManaged` / `compiled_avro` / `$config` | Source directory for generated `.java` files. | +| `avroSource` | `sourceDirectory` / `$config` / `avro` | Default Avro source directory for `*.avsc`, `*.avdl` and `*.avpr` files. | +| `avroSpecificRecords` | `Seq.empty` | List of fully qualified Avro record class names to recompile with current avro version and settings. | +| `avroDependencyIncludeFilter` | `Compile`: `source` typed `avro` classifier artifacts in `Avro` config
`Test`: nothing | Filter for including modules containing avro dependencies. | +| `avroUmanagedSourceDirectories` | `Seq(avroSource)` | Unmanaged Avro source directories, which contain manually created sources. | +| `avroUnpackDependencies` / `excludeFilter` | `HiddenFileFilter` | Filter for excluding avro specification files from unpacking. | +| `avroUnpackDependencies` / `includeFilter` | `AllPassFilter` | Filter for including avro specification files to unpack. | +| `avroUnpackDependencies` / `target` | `sourceManaged` / `avro` / `$config` | Target directory for schemas packaged in the dependencies | +| `packageAvro` / `artifactClassifier` | `Some("avro")` | Classifier for avro artifact | +| `packageAvro` / `publishArtifact` | `false` | Enable / Disable avro artifact publishing | ## Scoped Tasks (Compile/Test) @@ -103,15 +103,23 @@ Compile / packageAvro / publishArtifact := true You can specify a dependency on an avro source artifact that contains the schemas like so: ```sbt -libraryDependencies += "org" % "name" % "rev" classifier "avro" +libraryDependencies += "org" % "name" % "rev" % "avro" classifier "avro" ``` If some avro schemas are not packaged in a `source/avro` artifact, you can update the `avroDependencyIncludeFilter` setting to instruct the plugin to look for schemas in the desired dependency: ```sbt -libraryDependencies += "org" % "name" % "rev" // module containing avro schemas -avroDependencyIncludeFilter := avroDependencyIncludeFilter.value || moduleFilter(organization = "org", name = "name") +libraryDependencies += "org" % "name" % "rev" % "avro" // module containing avro schemas +Compile / avroDependencyIncludeFilter := configurationFilter("avro") && moduleFilter(organization = "org", name = "name") +``` + +If some artifact is meant to be used in the test scope only, you can do the following + +```sbt +libraryDependencies += "org" % "name" % "rev" % "avro" classifier "avro" +Compile / avroDependencyIncludeFilter ~= { old => old -- moduleFilter(organization = "org", name = "name") } +Test / avroDependencyIncludeFilter := configurationFilter("avro") && moduleFilter(organization = "org", name = "name") ``` # License diff --git a/plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala b/plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala index 54363f6..19a9dd0 100644 --- a/plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala +++ b/plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala @@ -62,18 +62,13 @@ object SbtAvro extends AutoPlugin { avroStringType := "CharSequence", avroUseNamespace := false, - // dependency management - avroDependencyIncludeFilter := artifactFilter( - `type` = Artifact.SourceType, - classifier = AvroClassifier - ), // addArtifact doesn't take publishArtifact setting in account artifacts ++= Classpaths.artifactDefs(avroArtifactTasks).value, packagedArtifacts ++= Classpaths.packaged(avroArtifactTasks).value, // use a custom folders to avoid potential conflict with other generators avroUnpackDependencies / target := sourceManaged.value / "avro", avroGenerate / target := sourceManaged.value / "compiled_avro", - // setup avro configuration. Use library management to fetch the compiler + // setup avro configuration. Use library management to fetch the compiler and schema sources ivyConfigurations ++= Seq(Avro), avroVersion := "1.12.0", avroAdditionalDependencies := Seq( @@ -90,6 +85,15 @@ object SbtAvro extends AutoPlugin { avroUnmanagedSourceDirectories := Seq(avroSource.value), avroSpecificRecords := Seq.empty, // dependencies + avroDependencyIncludeFilter := (configuration.value match { + case Compile => + // avro classifier artifact in Avro config are considered for compile scope + configurationFilter(Avro.name) && + artifactFilter(`type` = Artifact.SourceType, classifier = AvroClassifier) + case _ => + // ignore all dependencies for scopes other than compile + configurationFilter(NothingFilter) + }), avroUnpackDependencies / includeFilter := AllPassFilter, avroUnpackDependencies / excludeFilter := HiddenFileFilter, avroUnpackDependencies / target := configSrcSub(avroUnpackDependencies / target).value, @@ -121,6 +125,8 @@ object SbtAvro extends AutoPlugin { override def requires: Plugins = sbt.plugins.JvmPlugin + override def projectConfigurations: Seq[Configuration] = Seq(Avro) + override lazy val projectSettings: Seq[Setting[_]] = defaultSettings ++ inConfig(Avro)(Defaults.configSettings) ++ Seq(Compile, Test).flatMap(c => inConfig(c)(configScopedSettings)) @@ -169,11 +175,11 @@ object SbtAvro extends AutoPlugin { sbtPlugin.value, crossPaths.value ) - val conf = configuration.value.toConfigRef + val avroArtifacts = update.value .filter(avroDependencyIncludeFilter.value) .toSeq - .collect { case (`conf`, _, _, file) => file } + .map { case (_, _, _, f) => f } unpack( cacheBaseDirectory = cacheBaseDirectory, diff --git a/plugin/src/sbt-test/sbt-avro/publishing/build.sbt b/plugin/src/sbt-test/sbt-avro/publishing/build.sbt index eee5367..a3a4fe9 100644 --- a/plugin/src/sbt-test/sbt-avro/publishing/build.sbt +++ b/plugin/src/sbt-test/sbt-avro/publishing/build.sbt @@ -9,39 +9,44 @@ lazy val commonSettings = Seq( scalaVersion := "2.13.15" ) -lazy val javaOnlySettings = Seq( +lazy val avroOnlySettings = Seq( crossScalaVersions := Seq.empty, crossPaths := false, autoScalaLibrary := false, + // only create avro jar + Compile / packageAvro / publishArtifact := true, + Compile / packageBin / publishArtifact := false, + Compile / packageSrc / publishArtifact := false, + Compile / packageDoc / publishArtifact := false, ) lazy val `external`: Project = project .in(file("external")) .enablePlugins(SbtAvro) .settings(commonSettings) - .settings(javaOnlySettings) + .settings(avroOnlySettings) .settings( name := "external", version := "0.0.1-SNAPSHOT", - crossScalaVersions := Seq.empty, - crossPaths := false, - autoScalaLibrary := false, - Compile / packageAvro / publishArtifact := true ) lazy val `transitive`: Project = project .in(file("transitive")) .enablePlugins(SbtAvro) .settings(commonSettings) - .settings(javaOnlySettings) + .settings(avroOnlySettings) .settings( name := "transitive", version := "0.0.1-SNAPSHOT", - Compile / packageAvro / publishArtifact := true, - Test / publishArtifact := true, libraryDependencies ++= Seq( - "com.github.sbt" % "external" % "0.0.1-SNAPSHOT" classifier "avro", - ) + // when using avro scope, it won't be part of the pom dependencies -> intransitive + // to declare transitive dependency use the compile scope + "com.github.sbt" % "external" % "0.0.1-SNAPSHOT" classifier "avro" + ), + transitiveClassifiers += "avro", + Compile / avroDependencyIncludeFilter := artifactFilter(classifier = "avro"), + // create a test jar with a schema as resource + Test / packageBin / publishArtifact := true, ) lazy val root: Project = project @@ -52,12 +57,14 @@ lazy val root: Project = project name := "publishing-test", crossScalaVersions := Seq("2.13.15", "2.12.20"), libraryDependencies ++= Seq( - "com.github.sbt" % "transitive" % "0.0.1-SNAPSHOT" classifier "avro", - "com.github.sbt" % "transitive" % "0.0.1-SNAPSHOT" % Test classifier "tests", + "com.github.sbt" % "transitive" % "0.0.1-SNAPSHOT" % "avro" classifier "avro", // external as transitive + "com.github.sbt" % "transitive" % "0.0.1-SNAPSHOT" % "test" classifier "tests", "org.specs2" %% "specs2-core" % "4.20.9" % Test ), - // add additional transitive test jar - avroDependencyIncludeFilter := avroDependencyIncludeFilter.value || artifactFilter(name = "transitive", classifier = "tests"), + // add additional avro source test jar + // we unfortunatelly must recompile schemas from compile scope when test schema depends on it. + Test / avroDependencyIncludeFilter := (Compile / avroDependencyIncludeFilter).value || + artifactFilter(name = "transitive", classifier = "tests"), // exclude specific avsc file Compile / avroUnpackDependencies / excludeFilter := (Compile / avroUnpackDependencies / excludeFilter).value || "exclude.avsc", diff --git a/plugin/src/sbt-test/sbt-avro/publishing/transitive/src/test/resources/test.avsc b/plugin/src/sbt-test/sbt-avro/publishing/transitive/src/test/resources/test.avsc index 7b63a45..1a87638 100644 --- a/plugin/src/sbt-test/sbt-avro/publishing/transitive/src/test/resources/test.avsc +++ b/plugin/src/sbt-test/sbt-avro/publishing/transitive/src/test/resources/test.avsc @@ -12,4 +12,4 @@ "type": "com.github.sbt.avro.test.external.Avsc" } ] -} \ No newline at end of file +}