diff --git a/README.md b/README.md index 15850b3..b3d8ff7 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,6 @@ to generate the avro classes. | `avroFieldVisibility` | `public` | Field visibility for the properties. Possible values: `private`, `public`. | | `avroOptionalGetters` | `false` (requires avro `1.10+`) | Generate getters that return `Optional` for nullable fields. | | `avroStringType` | `CharSequence` | Type for representing strings. Possible values: `CharSequence`, `String`, `Utf8`. | -| `avroUseNamespace` | `false` | Validate that directory layout reflects namespaces, i.e. `com/myorg/MyRecord.avsc`. | | `avroVersion` | `1.12.0` | Avro version to use in the project. | ### Scoped settings (Compile/Test) diff --git a/api/src/main/java/com/github/sbt/avro/AvroCompiler.java b/api/src/main/java/com/github/sbt/avro/AvroCompiler.java index 5ffac68..dbdbe84 100644 --- a/api/src/main/java/com/github/sbt/avro/AvroCompiler.java +++ b/api/src/main/java/com/github/sbt/avro/AvroCompiler.java @@ -6,13 +6,12 @@ public interface AvroCompiler { void setStringType(String stringType); void setFieldVisibility(String fieldVisibility); - void setUseNamespace(boolean useNamespace); void setEnableDecimalLogicalType(boolean enableDecimalLogicalType); void setCreateSetters(boolean createSetters); void setOptionalGetters(boolean optionalGetters); void recompile(Class[] records, File target) throws Exception; void compileIdls(File[] idls, File target) throws Exception; - void compileAvscs(AvroFileRef[] avscs, File target) throws Exception; + void compileAvscs(File[] avscs, File target) throws Exception; void compileAvprs(File[] avprs, File target) throws Exception; } diff --git a/api/src/main/java/com/github/sbt/avro/AvroFileRef.java b/api/src/main/java/com/github/sbt/avro/AvroFileRef.java deleted file mode 100644 index 814bea3..0000000 --- a/api/src/main/java/com/github/sbt/avro/AvroFileRef.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.github.sbt.avro; - -import java.io.File; -import java.util.Objects; - -public class AvroFileRef implements Comparable { - - private final File root; - private final String path; - - public AvroFileRef(File root, String path) { - this.root = root; - this.path = path; - } - - public File getFile() { - return new File(root, path); - } - - public String pathToClassName() { - String woExt = path; - if (woExt.endsWith(".avsc")) { - woExt = woExt.substring(0, woExt.length() - 5); - } - return woExt.replace("/", "."); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - AvroFileRef that = (AvroFileRef) o; - return Objects.equals(root, that.root) && - Objects.equals(path, that.path); - } - - @Override - public int hashCode() { - return Objects.hash(root, path); - } - - @Override - public String toString() { - return getFile().toString(); - } - - @Override - public int compareTo(AvroFileRef o) { - int result = root.compareTo(o.root); - if (result == 0) { - result = path.compareTo(o.path); - } - return result; - } -} diff --git a/bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java b/bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java index 2df92d0..1d41939 100644 --- a/bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java +++ b/bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java @@ -21,7 +21,6 @@ public class AvroCompilerBridge implements AvroCompiler { protected StringType stringType; protected FieldVisibility fieldVisibility; - protected boolean useNamespace; protected boolean enableDecimalLogicalType; protected boolean createSetters; protected boolean optionalGetters; @@ -41,11 +40,6 @@ public void setFieldVisibility(String fieldVisibility) { this.fieldVisibility = FieldVisibility.valueOf(fieldVisibility); } - @Override - public void setUseNamespace(boolean useNamespace) { - this.useNamespace = useNamespace; - } - @Override public void setEnableDecimalLogicalType(boolean enableDecimalLogicalType) { this.enableDecimalLogicalType = enableDecimalLogicalType; @@ -101,21 +95,16 @@ public void compileIdls(File[] idls, File target) throws Exception { } @Override - public void compileAvscs(AvroFileRef[] avscs, File target) throws Exception { - List files = new ArrayList<>(avscs.length); - for (AvroFileRef ref : avscs) { - System.out.println("Compiling Avro schema: " + ref.getFile()); - files.add(ref); - } - Map schemas = parser.parseFiles(files); - if (useNamespace) { - for (Map.Entry s: schemas.entrySet()) { - validateParsedSchema(s.getKey(), s.getValue()); - } + public void compileAvscs(File[] avscs, File target) throws Exception { + List files = new ArrayList<>(avscs.length); + for (File schema : avscs) { + System.out.println("Compiling Avro schema: " + schema); + files.add(schema); } + Map schemas = parser.parseFiles(files); - for (Map.Entry entry: schemas.entrySet()) { - File file = entry.getKey().getFile(); + for (Map.Entry entry: schemas.entrySet()) { + File file = entry.getKey(); Schema schema = entry.getValue(); SpecificCompiler compiler = new SpecificCompiler(schema); configureCompiler(compiler); @@ -133,24 +122,4 @@ public void compileAvprs(File[] avprs, File target) throws Exception { compiler.compileToDestination(null, target); } } - - private void validateParsedSchema(AvroFileRef src, Schema schema) { - if (useNamespace) { - if (schema.getType() != Schema.Type.RECORD && schema.getType() != Schema.Type.ENUM) { - throw new SchemaGenerationException(String.format( - "Error compiling schema file %s. " - + "Only one root RECORD or ENUM type is allowed per file.", - src - )); - } else if (!src.pathToClassName().equals(schema.getFullName())) { - throw new SchemaGenerationException(String.format( - "Error compiling schema file %s. " - + "File class name %s does not match record class name %s", - src, - src.pathToClassName(), - schema.getFullName() - )); - } - } - } } diff --git a/bridge/src/main/java/com/github/sbt/avro/AvscFilesParser.java b/bridge/src/main/java/com/github/sbt/avro/AvscFilesParser.java index f50b546..a4bb548 100644 --- a/bridge/src/main/java/com/github/sbt/avro/AvscFilesParser.java +++ b/bridge/src/main/java/com/github/sbt/avro/AvscFilesParser.java @@ -4,6 +4,7 @@ import org.apache.avro.Schema; import java.io.IOException; +import java.io.File; import java.util.*; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -30,10 +31,10 @@ public void addTypes(Iterable types) { } } - public Map parseFiles(Collection files) { - Set unparsedFiles = new HashSet<>(files); - Map parsedFiles = new HashMap<>(); - Map parseExceptions = new HashMap<>(); + public Map parseFiles(Collection files) { + Set unparsedFiles = new HashSet<>(files); + Map parsedFiles = new HashMap<>(); + Map parseExceptions = new HashMap<>(); Schema.Parser parser = unstashParser(); boolean progressed = true; @@ -41,9 +42,9 @@ public Map parseFiles(Collection files) { progressed = false; parseExceptions.clear(); - for (AvroFileRef file : unparsedFiles) { + for (File file : unparsedFiles) { try { - Schema schema = parser.parse(file.getFile()); + Schema schema = parser.parse(file); parsedFiles.put(file, schema); progressed = true; stashParser(parser); @@ -64,7 +65,7 @@ public Map parseFiles(Collection files) { String message = Optional.ofNullable(parseExceptions.get(f)) .map(Exception::getMessage) .orElse("Unknown error"); - return f.getFile().getName() + ": " + message; + return f.getName() + ": " + message; }) .collect(Collectors.joining(",\n")); diff --git a/plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala b/plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala index 3dab52e..c2d960a 100644 --- a/plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala +++ b/plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala @@ -41,7 +41,6 @@ object SbtAvro extends AutoPlugin { val avroSource = settingKey[File]("Default Avro source directory for *.avsc, *.avdl and *.avpr files.") val avroStringType = settingKey[String]("Type for representing strings. Possible values: CharSequence, String, Utf8.") val avroUnmanagedSourceDirectories = settingKey[Seq[File]]("Unmanaged Avro source directories, which contain manually created sources.") - val avroUseNamespace = settingKey[Boolean]("Validate that directory layout reflects namespaces, i.e. com/myorg/MyRecord.avsc.") val avroVersion = settingKey[String]("Avro version to use in the project.") val avroGenerate = taskKey[Seq[File]]("Generate Java sources for Avro schemas.") @@ -60,7 +59,6 @@ object SbtAvro extends AutoPlugin { avroFieldVisibility := "public", avroOptionalGetters := false, avroStringType := "CharSequence", - avroUseNamespace := false, // addArtifact doesn't take publishArtifact setting in account artifacts ++= Classpaths.artifactDefs(avroArtifactTasks).value, @@ -142,11 +140,13 @@ object SbtAvro extends AutoPlugin { inStyle = FilesInfo.lastModified, outStyle = FilesInfo.exists ) { deps => - IO.createDirectory(extractTarget) + // dedicated directory per artifact to avoid name conflicts + val depTarget = extractTarget / jar.base + IO.createDirectory(depTarget) deps.flatMap { dep => val filter = includeFilter -- excludeFilter val (avroSpecs, filtered) = IO - .unzip(dep, extractTarget, AvroFilter) + .unzip(dep, depTarget, AvroFilter) .partition(filter.accept) IO.delete(filtered) if (avroSpecs.nonEmpty) { @@ -178,7 +178,7 @@ object SbtAvro extends AutoPlugin { .toSeq .map { case (_, _, _, f) => f } - unpack( + val unpacked = unpack( cacheBaseDirectory = cacheBaseDirectory, deps = avroArtifacts, extractTarget = (key / target).value, @@ -186,6 +186,11 @@ object SbtAvro extends AutoPlugin { excludeFilter = (key / excludeFilter).value, streams = (key / streams).value ) + + val previouslyUnpacked = key.previous.toSeq.flatten + IO.delete(previouslyUnpacked.diff(unpacked)) + + unpacked } private def sourceGeneratorTask(key: TaskKey[Seq[File]]) = Def.task { @@ -239,18 +244,13 @@ object SbtAvro extends AutoPlugin { compiler.setStringType(avroStringType.value) compiler.setFieldVisibility(avroFieldVisibility.value.toUpperCase) - compiler.setUseNamespace(avroUseNamespace.value) compiler.setEnableDecimalLogicalType(avroEnableDecimalLogicalType.value) compiler.setCreateSetters(avroCreateSetters.value) compiler.setOptionalGetters(avroOptionalGetters.value) val recs = records.map(avroClassLoader.loadClass) val avdls = srcDirs.flatMap(d => (d ** AvroAvdlFilter).get()) - val avscs = srcDirs.flatMap(d => - (d ** AvroAvscFilter) - .get() - .map(avsc => new AvroFileRef(d, avsc.relativeTo(d).get.toString)) - ) + val avscs = srcDirs.flatMap(d => (d ** AvroAvscFilter).get()) val avprs = srcDirs.flatMap(d => (d ** AvroAvrpFilter).get()) out.log.info( diff --git a/plugin/src/sbt-test/sbt-avro/publishing/build.sbt b/plugin/src/sbt-test/sbt-avro/publishing/build.sbt index a3a4fe9..f2f6ed7 100644 --- a/plugin/src/sbt-test/sbt-avro/publishing/build.sbt +++ b/plugin/src/sbt-test/sbt-avro/publishing/build.sbt @@ -69,11 +69,11 @@ lazy val root: Project = project Compile / avroUnpackDependencies / excludeFilter := (Compile / avroUnpackDependencies / excludeFilter).value || "exclude.avsc", Compile / checkUnpacked := { - exists(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "avdl.avdl") - exists(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "avpr.avpr") - exists(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "avsc.avsc") - absent(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "exclude.avsc") - exists(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "transitive" / "avsc.avsc") + exists(crossTarget.value / "src_managed" / "avro" / "main" / "external-avro" / "avdl.avdl") + exists(crossTarget.value / "src_managed" / "avro" / "main" / "external-avro" / "avpr.avpr") + exists(crossTarget.value / "src_managed" / "avro" / "main" / "external-avro" / "avsc.avsc") + absent(crossTarget.value / "src_managed" / "avro" / "main" / "external-avro" / "exclude.avsc") + exists(crossTarget.value / "src_managed" / "avro" / "main" / "transitive-avro" / "avsc.avsc") }, Compile / checkGenerated := { exists(crossTarget.value / "src_managed" / "compiled_avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "Avdl.java") @@ -82,7 +82,7 @@ lazy val root: Project = project exists(crossTarget.value / "src_managed" / "compiled_avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "transitive" / "Avsc.java") }, Test / checkUnpacked := { - exists(crossTarget.value / "src_managed" / "avro" / "test" / "test.avsc") + exists(crossTarget.value / "src_managed" / "avro" / "test" / "transitive-tests" / "test.avsc") }, Test / checkGenerated := { exists(crossTarget.value / "src_managed" / "compiled_avro" / "test" / "com" / "github" / "sbt" / "avro" / "test" / "transitive" / "Test.java") diff --git a/plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/com/github/sbt/avro/test/external/avdl.avdl b/plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/avdl.avdl similarity index 100% rename from plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/com/github/sbt/avro/test/external/avdl.avdl rename to plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/avdl.avdl diff --git a/plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/com/github/sbt/avro/test/external/avpr.avpr b/plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/avpr.avpr similarity index 100% rename from plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/com/github/sbt/avro/test/external/avpr.avpr rename to plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/avpr.avpr diff --git a/plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/com/github/sbt/avro/test/external/avsc.avsc b/plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/avsc.avsc similarity index 100% rename from plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/com/github/sbt/avro/test/external/avsc.avsc rename to plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/avsc.avsc diff --git a/plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/com/github/sbt/avro/test/external/exclude.avsc b/plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/exclude.avsc similarity index 100% rename from plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/com/github/sbt/avro/test/external/exclude.avsc rename to plugin/src/sbt-test/sbt-avro/publishing/external/src/main/avro/exclude.avsc diff --git a/plugin/src/sbt-test/sbt-avro/publishing/transitive/src/main/avro/com/github/sbt/avro/test/transitive/avsc.avsc b/plugin/src/sbt-test/sbt-avro/publishing/transitive/src/main/avro/avsc.avsc similarity index 100% rename from plugin/src/sbt-test/sbt-avro/publishing/transitive/src/main/avro/com/github/sbt/avro/test/transitive/avsc.avsc rename to plugin/src/sbt-test/sbt-avro/publishing/transitive/src/main/avro/avsc.avsc