[REST-128] Added support for GCSMock

.github/workflows/restonomer-build.yml

@@ -22,11 +22,6 @@ jobs:
           cache: 'sbt'
       - name: Scalafmt Check
         run: sbt scalafmtCheckAll
-      - name: Run Fake GCS Server
-        uses: addnab/docker-run-action@v3
-        with:
-          image: fsouza/fake-gcs-server:latest
-          options: -d --name fake-gcs-server -p 4443:4443
       - name: Run Unit Tests
         run: sbt test
       - name: Run Integration Tests

build.sbt

@@ -24,6 +24,7 @@ val scalaParserCombinatorsVersion = "2.2.0"
 val gcsConnectorVersion = "hadoop3-2.2.2"
 val monovoreDeclineVersion = "2.4.1"
 val googleCloudStorageVersion = "2.24.0"
+val testContainersScalaVersion = "0.40.17"
 
 // ----- TOOL DEPENDENCIES ----- //
 
@@ -75,6 +76,8 @@ val monovoreDeclineDependencies = Seq("com.monovore" %% "decline" % monovoreDecl
 
 val googleCloudStorageDependencies = Seq("com.google.cloud" % "google-cloud-storage" % googleCloudStorageVersion)
 
+val testContainersScalaDependencies = Seq("com.dimafeng" %% "testcontainers-scala" % testContainersScalaVersion % Test)
+
 // ----- MODULE DEPENDENCIES ----- //
 
 val restonomerCoreDependencies =
@@ -91,7 +94,8 @@ val restonomerCoreDependencies =
     odelayDependencies ++
     gcsConnectorDependencies ++
     monovoreDeclineDependencies ++
-    googleCloudStorageDependencies
+    googleCloudStorageDependencies ++
+    testContainersScalaDependencies
 
 val restonomerSparkUtilsDependencies =
   sparkDependencies ++

docs/source/restonomer_getting_started.md

@@ -31,6 +31,8 @@ libraryDependencies += "com.clairvoyant.restonomer" %% "restonomer-core" % "2.1.
   User can create the restonomer context instance by passing the restonomer context directory path to the constructor 
   of RestonomerContext class.
 
+  Currently, user can provide the local file system path or GCS path for the restonomer context directory.
+
   ```scala
   private val restonomerContextDirectoryPath = "<restonomer_context_directory_path>"
   private val restonomerContext = RestonomerContext(restonomerContextDirectoryPath)

restonomer-core/src/it/scala/com/clairvoyant/restonomer/pagination/OffsetBasedPaginationIntegrationTest.scala

@@ -1,7 +1,6 @@
 package com.clairvoyant.restonomer.pagination
 
-import com.clairvoyant.restonomer.common.IntegrationTestDependencies
-import com.clairvoyant.restonomer.common.MockFileSystemPersistence
+import com.clairvoyant.restonomer.common.{IntegrationTestDependencies, MockFileSystemPersistence}
 
 class OffsetBasedPaginationIntegrationTest extends IntegrationTestDependencies with MockFileSystemPersistence {
 

restonomer-core/src/it/scala/com/clairvoyant/restonomer/transformation/CastColumnsBasedOnSuffixTransformationIntegrationTest.scala

@@ -1,7 +1,7 @@
 package com.clairvoyant.restonomer.transformation
 
 import com.clairvoyant.restonomer.common.{IntegrationTestDependencies, MockFileSystemPersistence}
-import org.apache.spark.sql.types.{DecimalType, DoubleType, StringType, StructField, StructType}
+import org.apache.spark.sql.types.*
 
 class CastColumnsBasedOnSuffixTransformationIntegrationTest
     extends IntegrationTestDependencies

restonomer-core/src/it/scala/com/clairvoyant/restonomer/transformation/ReplaceEmptyStringsWithNullsTransformationIntegrationTest.scala

@@ -1,7 +1,6 @@
 package com.clairvoyant.restonomer.transformation
 
-import com.clairvoyant.restonomer.common.IntegrationTestDependencies
-import com.clairvoyant.restonomer.common.MockFileSystemPersistence
+import com.clairvoyant.restonomer.common.{IntegrationTestDependencies, MockFileSystemPersistence}
 
 class ReplaceEmptyStringsWithNullsTransformationIntegrationTest
     extends IntegrationTestDependencies

restonomer-core/src/main/scala/com/clairvoyant/restonomer/core/app/RestonomerContext.scala

@@ -3,6 +3,7 @@ package com.clairvoyant.restonomer.core.app
 import com.clairvoyant.restonomer.core.config.{ConfigVariablesSubstitutor, GCSRestonomerContextLoader, LocalRestonomerContextLoader, RestonomerContextLoader}
 import com.clairvoyant.restonomer.core.exception.RestonomerException
 import com.clairvoyant.restonomer.core.model.{ApplicationConfig, CheckpointConfig}
+import com.google.cloud.storage.{Storage, StorageOptions}
 import zio.Config
 import zio.config.magnolia.*
 
@@ -15,10 +16,10 @@ object RestonomerContext {
       configVariablesFromApplicationArgs: Map[String, String] = Map()
   ): RestonomerContext = {
     val restonomerContextLoader =
-      if (restonomerContextDirectoryPath.startsWith("gs://"))
+      if (restonomerContextDirectoryPath.startsWith("gs://")) {
+        given gcsStorageClient: Storage = StorageOptions.getDefaultInstance().getService()
         GCSRestonomerContextLoader()
-      else
-        LocalRestonomerContextLoader()
+      } else LocalRestonomerContextLoader()
 
     if (restonomerContextLoader.fileExists(restonomerContextDirectoryPath))
       new RestonomerContext(restonomerContextLoader, restonomerContextDirectoryPath, configVariablesFromApplicationArgs)

restonomer-core/src/main/scala/com/clairvoyant/restonomer/core/app/RestonomerWorkflow.scala

@@ -17,7 +17,6 @@ import sttp.client3.Response
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, Future}
-import com.clairvoyant.restonomer.spark.utils.writer.DataFrameToGCSBucketWriter
 
 class RestonomerWorkflow(using sparkSession: SparkSession) {
 

restonomer-core/src/main/scala/com/clairvoyant/restonomer/core/config/GCSRestonomerContextLoader.scala

@@ -10,14 +10,10 @@ import java.io.{ByteArrayInputStream, File}
 import scala.annotation.tailrec
 import scala.io.Source
 
-class GCSRestonomerContextLoader extends RestonomerContextLoader {
-
-  given gcsStorageClient: Storage = StorageOptions.getDefaultInstance.getService
+class GCSRestonomerContextLoader(using gcsStorageClient: Storage) extends RestonomerContextLoader {
 
   override def fileExists(filePath: String): Boolean =
-    gcsStorageClient
-      .get(BlobId.of(getBucketName(filePath), getBlobName(filePath)))
-      .exists()
+    gcsStorageClient.get(BlobId.of(getBucketName(filePath), getBlobName(filePath))) != null
 
   override def readConfigFile(configFilePath: String): Source =
     Source.fromInputStream(

restonomer-core/src/main/scala/com/clairvoyant/restonomer/core/config/LocalRestonomerContextLoader.scala

@@ -1,11 +1,11 @@
 package com.clairvoyant.restonomer.core.config
 
-import java.nio.file.Files
-import java.nio.file.Paths
-import scala.io.Source
 import zio.Config
-import scala.annotation.tailrec
+
 import java.io.File
+import java.nio.file.{Files, Paths}
+import scala.annotation.tailrec
+import scala.io.Source
 
 class LocalRestonomerContextLoader extends RestonomerContextLoader {
 

restonomer-core/src/main/scala/com/clairvoyant/restonomer/core/config/RestonomerContextLoader.scala

@@ -1,17 +1,13 @@
 package com.clairvoyant.restonomer.core.config
 
 import com.clairvoyant.restonomer.core.exception.RestonomerException
-import zio.ConfigProvider
-import zio._
+import zio.*
 import zio.config.typesafe.*
 
 import java.io.File
 import scala.annotation.tailrec
 import scala.io.Source
-import scala.util.Failure
-import scala.util.Success
-import scala.util.Try
-import scala.util.Using
+import scala.util.{Failure, Success, Try, Using}
 
 trait RestonomerContextLoader {
 

restonomer-core/src/main/scala/com/clairvoyant/restonomer/core/util/GCSUtil.scala

@@ -1,10 +1,10 @@
 package com.clairvoyant.restonomer.core.util
 
-import com.google.cloud.storage.StorageOptions
-import com.google.cloud.storage.Storage
 import com.google.cloud.storage.Storage.BlobListOption
-import com.google.cloud.storage.Blob
-import collection.convert.ImplicitConversions.*
+import com.google.cloud.storage.StorageOptions.Builder
+import com.google.cloud.storage.{Blob, Storage, StorageOptions}
+
+import scala.jdk.CollectionConverters.*
 
 object GCSUtil {
 
@@ -18,6 +18,7 @@ object GCSUtil {
     gcsStorageClient
       .list(getBucketName(fullGCSPath), BlobListOption.prefix(getBlobName(fullGCSPath)))
       .iterateAll()
+      .asScala
       .toList
 
   def getBlobFullPath(blob: Blob): String = s"${GCS_PREFIX}${blob.getBucket}/${blob.getName}"

restonomer-core/src/test/resources/fake-gcs-server/data/test-bucket/restonomer_context/checkpoints/authentication/api_key_authentication/checkpoint_api_key_authentication_cookie.conf

@@ -0,0 +1,26 @@
+name = "checkpoint_api_key_authentication_cookie"
+
+data = {
+  data-request = {
+    url = "http://localhost:8080/api-key-auth-cookie"
+
+    authentication = {
+      type = "APIKeyAuthentication"
+      api-key-name = "test_api_key_name"
+      api-key-value = "test_api_key_value"
+      placeholder = "Cookie"
+    }
+  }
+
+  data-response = {
+    body = {
+      type = "JSON"
+    }
+
+    persistence = {
+      type = "FileSystem"
+      file-format = "parquet"
+      file-path = "/tmp/authentication/api_key_authentication"
+    }
+  }
+}

restonomer-core/src/test/resources/fake-gcs-server/data/test-bucket/restonomer_context/checkpoints/authentication/api_key_authentication/checkpoint_api_key_authentication_query_param.conf

@@ -0,0 +1,26 @@
+name = "checkpoint_api_key_authentication_query_param"
+
+data = {
+  data-request = {
+    url = "http://localhost:8080/api-key-auth-query-param"
+
+    authentication = {
+      type = "APIKeyAuthentication"
+      api-key-name = "test_api_key_name"
+      api-key-value = "test_api_key_value"
+      placeholder = "QueryParam"
+    }
+  }
+
+  data-response = {
+    body = {
+      type = "JSON"
+    }
+
+    persistence = {
+      type = "FileSystem"
+      file-format = "parquet"
+      file-path = "/tmp/authentication/api_key_authentication"
+    }
+  }
+}

restonomer-core/src/test/resources/fake-gcs-server/data/test-bucket/restonomer_context/checkpoints/authentication/api_key_authentication/checkpoint_api_key_authentication_request_header.conf

@@ -0,0 +1,26 @@
+name = "checkpoint_api_key_authentication_request_header"
+
+data = {
+  data-request = {
+    url = "http://localhost:8080/api-key-auth-request-header"
+
+    authentication = {
+      type = "APIKeyAuthentication"
+      api-key-name = "test_api_key_name"
+      api-key-value = "test_api_key_value"
+      placeholder = "RequestHeader"
+    }
+  }
+
+  data-response = {
+    body = {
+      type = "JSON"
+    }
+
+    persistence = {
+      type = "FileSystem"
+      file-format = "parquet"
+      file-path = "/tmp/authentication/api_key_authentication"
+    }
+  }
+}

restonomer-core/src/test/resources/fake-gcs-server/data/test-bucket/restonomer_context/checkpoints/authentication/aws_signature_authentication/checkpoint_aws_signature_authentication.conf

@@ -0,0 +1,25 @@
+name = "checkpoint_aws_signature_authentication"
+
+data = {
+  data-request = {
+    url = "http://localhost:8080/aws-signature-auth"
+
+    authentication = {
+      type = "AwsSignatureAuthentication"
+      access-key = "test_access_key"
+      secret-key = "test_secret_key"
+    }
+  }
+
+  data-response = {
+    body = {
+      type = "JSON"
+    }
+
+    persistence = {
+      type = "FileSystem"
+      file-format = "parquet"
+      file-path = "/tmp/authentication/aws_signature_authentication"
+    }
+  }
+}

restonomer-core/src/test/scala/com/clairvoyant/restonomer/core/common/CoreSpec.scala

@@ -2,9 +2,9 @@ package com.clairvoyant.restonomer.core.common
 
 import com.clairvoyant.restonomer.spark.utils.DataFrameMatchers
 import org.apache.spark.sql.SparkSession
-import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
 import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
+import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
 import sttp.client3.*
 import sttp.model.Method
 

restonomer-core/src/test/scala/com/clairvoyant/restonomer/core/common/GCSMockSpec.scala

@@ -1,20 +1,45 @@
 package com.clairvoyant.restonomer.core.common
 
+import com.dimafeng.testcontainers.GenericContainer
+import com.dimafeng.testcontainers.GenericContainer.{Def, FileSystemBind}
+import com.dimafeng.testcontainers.scalatest.TestContainerForAll
 import com.google.cloud.NoCredentials
 import com.google.cloud.storage.{BucketInfo, Storage, StorageOptions}
-import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, Suite}
+import com.google.cloud.storage.StorageOptions.Builder
+import org.scalatest.Suite
+import org.scalatest.flatspec.AnyFlatSpec
+import org.testcontainers.containers.BindMode
+import org.testcontainers.containers.wait.strategy.Wait
 
-trait GCSMockSpec {
-  val gcsMockPort: Int = 4443
-  val gcsMockEndpoint: String = s"http://0.0.0.0:$gcsMockPort"
+import java.net.URL
+import scala.io.Source
 
-  given gcsStorageClient: Storage =
+trait GCSMockSpec extends TestContainerForAll {
+  this: Suite =>
+
+  val gcsPrefix = "gs://"
+  val mockGCSBucketName = "test-bucket"
+  val mockBlobName = "restonomer_context/checkpoints"
+  val mockFullGCSPath = s"$gcsPrefix$mockGCSBucketName/$mockBlobName"
+  val mockGCSPort = 4443
+
+  override val containerDef: Def[GenericContainer] = GenericContainer.Def(
+    dockerImage = "fsouza/fake-gcs-server:latest",
+    exposedPorts = Seq(mockGCSPort),
+    command = Seq("-scheme=http"),
+    classpathResourceMapping = Seq(FileSystemBind("fake-gcs-server/data", "/data", BindMode.READ_ONLY))
+  )
+
+  implicit lazy val gcsStorageClient: Storage = withContainers { container =>
     StorageOptions
       .newBuilder()
-      .setHost(gcsMockEndpoint)
       .setProjectId("test-project")
       .setCredentials(NoCredentials.getInstance())
+      .setHost(s"http://${container.containerIpAddress}:${container.mappedPort(mockGCSPort)}")
       .build()
-      .getService()
+      .getService
+  }
+
+  lazy val gcsBucket = gcsStorageClient.get(mockGCSBucketName)
 
 }