From 837dbe316d028cbef0fb075689156452f1af3c0e Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Sun, 8 Oct 2023 19:32:08 +0100 Subject: [PATCH] upgrade to avro 1.11.3 due to CVE (#259) * upgrade to avro 1.11.3 due to CVE * suppress deprecation warnings * try suppress deprecation warnings * try to suppress deprecation warnings * Update AbstractAvroParquetBase.scala --- .../java/docs/javadsl/AvroParquetSinkTest.java | 1 + .../src/test/java/docs/javadsl/Examples.java | 1 + .../docs/scaladsl/AbstractAvroParquetBase.scala | 17 ++++++++++------- project/Dependencies.scala | 8 +++++--- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/avroparquet/src/test/java/docs/javadsl/AvroParquetSinkTest.java b/avroparquet/src/test/java/docs/javadsl/AvroParquetSinkTest.java index d72820e8b..a6b7b4da9 100644 --- a/avroparquet/src/test/java/docs/javadsl/AvroParquetSinkTest.java +++ b/avroparquet/src/test/java/docs/javadsl/AvroParquetSinkTest.java @@ -76,6 +76,7 @@ public void setup() { records.add(new GenericRecordBuilder(schema).set("id", "3").set("body", "body13").build()); } + @SuppressWarnings("deprecation") @Test public void createNewParquetFile() throws InterruptedException, IOException, TimeoutException, ExecutionException { diff --git a/avroparquet/src/test/java/docs/javadsl/Examples.java b/avroparquet/src/test/java/docs/javadsl/Examples.java index 406631a25..a1a884f5e 100644 --- a/avroparquet/src/test/java/docs/javadsl/Examples.java +++ b/avroparquet/src/test/java/docs/javadsl/Examples.java @@ -58,6 +58,7 @@ public class Examples { Source source = AvroParquetSource.create(reader); // #init-source + @SuppressWarnings("deprecation") public Examples() throws IOException { // #init-flow diff --git a/avroparquet/src/test/scala/docs/scaladsl/AbstractAvroParquetBase.scala b/avroparquet/src/test/scala/docs/scaladsl/AbstractAvroParquetBase.scala index 97a4052f8..580b271a4 100644 --- a/avroparquet/src/test/scala/docs/scaladsl/AbstractAvroParquetBase.scala +++ b/avroparquet/src/test/scala/docs/scaladsl/AbstractAvroParquetBase.scala @@ -22,6 +22,7 @@ import org.apache.parquet.hadoop.util.HadoopInputFile import org.apache.parquet.hadoop.{ ParquetReader, ParquetWriter } import org.scalacheck.Gen +import scala.annotation.nowarn import scala.util.Random trait AbstractAvroParquetBase { @@ -47,6 +48,7 @@ trait AbstractAvroParquetBase { val conf: Configuration = new Configuration() conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true) + @nowarn("msg=deprecated") def parquetWriter[T <: GenericRecord](file: String, conf: Configuration, schema: Schema): ParquetWriter[T] = AvroParquetWriter.builder[T](new Path(file)).withConf(conf).withSchema(schema).build() @@ -80,6 +82,7 @@ trait AbstractAvroParquetBase { // #prepare-source } + @nowarn("msg=deprecated") def sinkDocumentation(): Unit = { // #prepare-sink import com.sksamuel.avro4s.Record @@ -87,30 +90,30 @@ trait AbstractAvroParquetBase { import org.apache.hadoop.fs.Path import org.apache.parquet.avro.AvroReadSupport - val file: String = "./sample/path/test.parquet" - val conf: Configuration = new Configuration() + val file = "./sample/path/test.parquet" + val conf = new Configuration() conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true) - val writer: ParquetWriter[Record] = + val writer = AvroParquetWriter.builder[Record](new Path(file)).withConf(conf).withSchema(schema).build() // #prepare-sink if (writer != null) { // forces val usage } } + @nowarn("msg=deprecated") def initWriterDocumentation(): Unit = { // #init-writer import org.apache.avro.generic.GenericRecord import org.apache.hadoop.fs.Path import org.apache.parquet.avro.AvroParquetReader - import org.apache.parquet.hadoop.ParquetReader import org.apache.parquet.hadoop.util.HadoopInputFile - val file: String = "./sample/path/test.parquet" - val writer: ParquetWriter[GenericRecord] = + val file = "./sample/path/test.parquet" + val writer = AvroParquetWriter.builder[GenericRecord](new Path(file)).withConf(conf).withSchema(schema).build() // #init-writer // #init-reader - val reader: ParquetReader[GenericRecord] = + val reader = AvroParquetReader.builder[GenericRecord](HadoopInputFile.fromPath(new Path(file), conf)).withConf(conf).build() // #init-reader if (writer != null && reader != null) { // forces val usage diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 0dbd617a9..4abb9666c 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -25,6 +25,7 @@ object Dependencies { val InfluxDBJavaVersion = "2.15" + val AvroVersion = "1.11.3" val AwsSdk2Version = "2.17.113" val AwsSpiPekkoHttpVersion = "0.1.0" // Sync with plugins.sbt @@ -157,12 +158,13 @@ object Dependencies { "com.google.jimfs" % "jimfs" % "1.2" % Test)) val avro4sVersion: Def.Initialize[String] = Def.setting { - if (Common.isScala3.value) "5.0.4" else "4.1.1" + if (Common.isScala3.value) "5.0.5" else "4.1.1" } val AvroParquet = Seq( libraryDependencies ++= Seq( - "org.apache.parquet" % "parquet-avro" % "1.10.1", // Apache2 + "org.apache.parquet" % "parquet-avro" % "1.13.1", // Apache2 + "org.apache.avro" % "avro" % AvroVersion, ("org.apache.hadoop" % "hadoop-client" % "3.2.1" % Test).exclude("log4j", "log4j"), // Apache2 ("org.apache.hadoop" % "hadoop-common" % "3.2.1" % Test).exclude("log4j", "log4j"), // Apache2 "com.sksamuel.avro4s" %% "avro4s-core" % avro4sVersion.value % Test, @@ -214,7 +216,7 @@ object Dependencies { libraryDependencies ++= Seq( // https://github.com/googleapis/java-bigquerystorage/tree/master/proto-google-cloud-bigquerystorage-v1 "com.google.api.grpc" % "proto-google-cloud-bigquerystorage-v1" % "1.22.0" % "protobuf-src", - "org.apache.avro" % "avro" % "1.9.2" % "provided", + "org.apache.avro" % "avro" % AvroVersion % "provided", "org.apache.arrow" % "arrow-vector" % "4.0.0" % "provided", "io.grpc" % "grpc-auth" % org.apache.pekko.grpc.gen.BuildInfo.grpcVersion, "com.google.protobuf" % "protobuf-java" % protobufJavaVersion,