Merge pull request #47 from ilubis-pub/Upgrade-Spark-3.0.1

Upgrade to spark 3.0.1
zalando-incubator · Dec 10, 2020 · 655b678 · 655b678
2 parents 33ac3c2 + d7e31c6
commit 655b678
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 11 deletions.
diff --git a/build.sbt b/build.sbt
@@ -5,9 +5,10 @@ organization := "org.zalando"
 
 scalaVersion := "2.12.10"
 
-libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.4.7"  % Provided
+libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.0.1"  % Provided
 libraryDependencies += "com.typesafe.play" %% "play-json" % "2.9.1"
-dependencyOverrides ++= Set("com.fasterxml.jackson.core" % "jackson-databind" % "2.6.5")
+libraryDependencies += "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.11.2"
+dependencyOverrides ++= Set("com.fasterxml.jackson.core" % "jackson-databind" % "2.11.2")
 
 libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.1" % "test"
 

diff --git a/src/test/scala/org/zalando/spark/jsonschema/SchemaConverterTest.scala b/src/test/scala/org/zalando/spark/jsonschema/SchemaConverterTest.scala
@@ -284,9 +284,10 @@ class SchemaConverterTest extends FunSuite with Matchers with BeforeAndAfter {
       "integer" -> LongType,
       "boolean" -> BooleanType
     )
-    typeMap.foreach { case (name, atype) =>
-      val schema = SchemaConverter.convertContent(
-        s"""
+    typeMap.foreach {
+      case (name, atype) =>
+        val schema = SchemaConverter.convertContent(
+          s"""
           {
             "$$schema": "smallTestSchema",
             "type": "object",
@@ -300,13 +301,13 @@ class SchemaConverterTest extends FunSuite with Matchers with BeforeAndAfter {
             }
           }
         """
-      )
+        )
 
-      val expected = StructType(Array(
-        StructField("array", ArrayType(atype, containsNull = true), nullable = false)
-      ))
+        val expected = StructType(Array(
+          StructField("array", ArrayType(atype, containsNull = true), nullable = false)
+        ))
 
-      assert(schema === expected)
+        assert(schema === expected)
     }
   }
 

diff --git a/src/test/scala/org/zalando/spark/jsonschema/SparkTestEnv.scala b/src/test/scala/org/zalando/spark/jsonschema/SparkTestEnv.scala
@@ -19,7 +19,7 @@ object SparkTestEnv {
 
   def getTestResourceContent(relativePath: String): String = {
     Option(getClass.getResource(relativePath)) match {
-      case Some(relPath) => Source.fromURL (relPath).mkString
+      case Some(relPath) => Source.fromURL(relPath).mkString
       case None => throw new IllegalArgumentException(s"Path can not be reached: $relativePath")
     }
   }