Merge pull request #268 from dafreels/develop

Fixed a dependency with BigQuery Connector
Acxiom · Oct 28, 2021 · 9eb6a76 · 9eb6a76
2 parents 6be51c2 + 6f18727
commit 9eb6a76
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 10 deletions.
diff --git a/metalus-gcp/pom.xml b/metalus-gcp/pom.xml
@@ -61,11 +61,6 @@
             <artifactId>google-cloud-datastore</artifactId>
             <version>1.104.0</version>
         </dependency>
-        <dependency>
-            <groupId>com.google.cloud.spark</groupId>
-            <artifactId>spark-bigquery-with-dependencies_${scala.compat.version}</artifactId>
-            <version>0.21.1</version>
-        </dependency>
         <dependency>
             <groupId>io.grpc</groupId>
             <artifactId>grpc-core</artifactId>

diff --git a/metalus-gcp/src/main/resources/dependencies.json b/metalus-gcp/src/main/resources/dependencies.json
@@ -6,6 +6,11 @@
         "artifactId": "metalus-common_${scala.compat.version}-spark_${spark.compat.version}",
         "version": "${parent.version}"
       },
+      {
+        "groupId": "com.google.cloud.spark",
+        "artifactId": "spark-bigquery-with-dependencies_${scala.compat.version}",
+        "version": "0.21.1"
+      },
       {
         "groupId": "org.apache.spark",
         "artifactId": "spark-streaming_${scala.compat.version}",

diff --git a/metalus-gcp/src/main/scala/com/acxiom/gcp/fs/GCSFileManager.scala b/metalus-gcp/src/main/scala/com/acxiom/gcp/fs/GCSFileManager.scala
@@ -1,15 +1,14 @@
 package com.acxiom.gcp.fs
 
-import java.io._
-import java.net.URI
-import java.nio.channels.Channels
-
 import com.acxiom.pipeline.fs.{FileInfo, FileManager}
 import com.google.auth.oauth2.GoogleCredentials
 import com.google.cloud.storage.{BlobId, BlobInfo, Storage, StorageOptions}
 import org.json4s.DefaultFormats
 import org.json4s.native.Serialization
 
+import java.io._
+import java.net.URI
+import java.nio.channels.Channels
 import scala.collection.JavaConverters._
 
 object GCSFileManager {
@@ -22,7 +21,7 @@ object GCSFileManager {
   def prepareGCSFilePath(path: String, bucket: Option[String] = None): String = {
     if (path.startsWith("/")) {
       path.substring(1)
-    } else if (path.startsWith(s"gs:")) {
+    } else if (path.startsWith("gs:")) {
       new URI(path).normalize().toString
     } else {
       path