upgrade to avro 1.11.3 due to CVE (#259)

* upgrade to avro 1.11.3 due to CVE * suppress deprecation warnings * try suppress deprecation warnings * try to suppress deprecation warnings * Update AbstractAvroParquetBase.scala
apache · Oct 8, 2023 · 837dbe3 · 837dbe3
1 parent ce2b214
commit 837dbe3
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 10 deletions.
diff --git a/avroparquet/src/test/java/docs/javadsl/AvroParquetSinkTest.java b/avroparquet/src/test/java/docs/javadsl/AvroParquetSinkTest.java
@@ -76,6 +76,7 @@ public void setup() {
     records.add(new GenericRecordBuilder(schema).set("id", "3").set("body", "body13").build());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void createNewParquetFile()
       throws InterruptedException, IOException, TimeoutException, ExecutionException {

diff --git a/avroparquet/src/test/java/docs/javadsl/Examples.java b/avroparquet/src/test/java/docs/javadsl/Examples.java
@@ -58,6 +58,7 @@ public class Examples {
   Source<GenericRecord, NotUsed> source = AvroParquetSource.create(reader);
   // #init-source
 
+  @SuppressWarnings("deprecation")
   public Examples() throws IOException {
 
     // #init-flow

diff --git a/avroparquet/src/test/scala/docs/scaladsl/AbstractAvroParquetBase.scala b/avroparquet/src/test/scala/docs/scaladsl/AbstractAvroParquetBase.scala
@@ -22,6 +22,7 @@ import org.apache.parquet.hadoop.util.HadoopInputFile
 import org.apache.parquet.hadoop.{ ParquetReader, ParquetWriter }
 import org.scalacheck.Gen
 
+import scala.annotation.nowarn
 import scala.util.Random
 
 trait AbstractAvroParquetBase {
@@ -47,6 +48,7 @@ trait AbstractAvroParquetBase {
   val conf: Configuration = new Configuration()
   conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true)
 
+  @nowarn("msg=deprecated")
   def parquetWriter[T <: GenericRecord](file: String, conf: Configuration, schema: Schema): ParquetWriter[T] =
     AvroParquetWriter.builder[T](new Path(file)).withConf(conf).withSchema(schema).build()
 
@@ -80,37 +82,38 @@ trait AbstractAvroParquetBase {
     // #prepare-source
   }
 
+  @nowarn("msg=deprecated")
   def sinkDocumentation(): Unit = {
     // #prepare-sink
     import com.sksamuel.avro4s.Record
     import org.apache.hadoop.conf.Configuration
     import org.apache.hadoop.fs.Path
     import org.apache.parquet.avro.AvroReadSupport
 
-    val file: String = "./sample/path/test.parquet"
-    val conf: Configuration = new Configuration()
+    val file = "./sample/path/test.parquet"
+    val conf = new Configuration()
     conf.setBoolean(AvroReadSupport.AVRO_COMPATIBILITY, true)
-    val writer: ParquetWriter[Record] =
+    val writer =
       AvroParquetWriter.builder[Record](new Path(file)).withConf(conf).withSchema(schema).build()
     // #prepare-sink
     if (writer != null) { // forces val usage
     }
   }
 
+  @nowarn("msg=deprecated")
   def initWriterDocumentation(): Unit = {
     // #init-writer
     import org.apache.avro.generic.GenericRecord
     import org.apache.hadoop.fs.Path
     import org.apache.parquet.avro.AvroParquetReader
-    import org.apache.parquet.hadoop.ParquetReader
     import org.apache.parquet.hadoop.util.HadoopInputFile
 
-    val file: String = "./sample/path/test.parquet"
-    val writer: ParquetWriter[GenericRecord] =
+    val file = "./sample/path/test.parquet"
+    val writer =
       AvroParquetWriter.builder[GenericRecord](new Path(file)).withConf(conf).withSchema(schema).build()
     // #init-writer
     // #init-reader
-    val reader: ParquetReader[GenericRecord] =
+    val reader =
       AvroParquetReader.builder[GenericRecord](HadoopInputFile.fromPath(new Path(file), conf)).withConf(conf).build()
     // #init-reader
     if (writer != null && reader != null) { // forces val usage

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -25,6 +25,7 @@ object Dependencies {
 
   val InfluxDBJavaVersion = "2.15"
 
+  val AvroVersion = "1.11.3"
   val AwsSdk2Version = "2.17.113"
   val AwsSpiPekkoHttpVersion = "0.1.0"
   // Sync with plugins.sbt
@@ -157,12 +158,13 @@ object Dependencies {
       "com.google.jimfs" % "jimfs" % "1.2" % Test))
 
   val avro4sVersion: Def.Initialize[String] = Def.setting {
-    if (Common.isScala3.value) "5.0.4" else "4.1.1"
+    if (Common.isScala3.value) "5.0.5" else "4.1.1"
   }
 
   val AvroParquet = Seq(
     libraryDependencies ++= Seq(
-      "org.apache.parquet" % "parquet-avro" % "1.10.1", // Apache2
+      "org.apache.parquet" % "parquet-avro" % "1.13.1", // Apache2
+      "org.apache.avro" % "avro" % AvroVersion,
       ("org.apache.hadoop" % "hadoop-client" % "3.2.1" % Test).exclude("log4j", "log4j"), // Apache2
       ("org.apache.hadoop" % "hadoop-common" % "3.2.1" % Test).exclude("log4j", "log4j"), // Apache2
       "com.sksamuel.avro4s" %% "avro4s-core" % avro4sVersion.value % Test,
@@ -214,7 +216,7 @@ object Dependencies {
     libraryDependencies ++= Seq(
       // https://github.com/googleapis/java-bigquerystorage/tree/master/proto-google-cloud-bigquerystorage-v1
       "com.google.api.grpc" % "proto-google-cloud-bigquerystorage-v1" % "1.22.0" % "protobuf-src",
-      "org.apache.avro" % "avro" % "1.9.2" % "provided",
+      "org.apache.avro" % "avro" % AvroVersion % "provided",
       "org.apache.arrow" % "arrow-vector" % "4.0.0" % "provided",
       "io.grpc" % "grpc-auth" % org.apache.pekko.grpc.gen.BuildInfo.grpcVersion,
       "com.google.protobuf" % "protobuf-java" % protobufJavaVersion,