Add alert & retry for delta/s3 initialization

snowplow-incubator · Jul 31, 2024 · ab03288 · ab03288
1 parent 30e9e2a
commit ab03288
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 3 deletions.
diff --git a/modules/aws/src/main/scala/com.snowplowanalytics.snowplow.lakes/AwsApp.scala b/modules/aws/src/main/scala/com.snowplowanalytics.snowplow.lakes/AwsApp.scala
@@ -14,6 +14,12 @@ import software.amazon.awssdk.services.s3.model.{NoSuchBucketException, S3Except
 import software.amazon.awssdk.services.sts.model.StsException
 import software.amazon.awssdk.services.glue.model.{AccessDeniedException => GlueAccessDeniedException}
 
+import org.apache.hadoop.fs.s3a.UnknownStoreException
+
+import java.nio.file.AccessDeniedException
+
+import org.apache.spark.sql.delta.DeltaIOException
+
 import com.snowplowanalytics.snowplow.sources.kinesis.{KinesisSource, KinesisSourceConfig}
 import com.snowplowanalytics.snowplow.sinks.kinesis.{KinesisSink, KinesisSinkConfig}
 
@@ -36,6 +42,16 @@ object AwsApp extends LoaderApp[KinesisSourceConfig, KinesisSinkConfig](BuildInf
     case _: StsException =>
       // No permission to assume the role given to authenticate to S3/Glue
       true
+    case _: UnknownStoreException =>
+      // no such bucket exist
+      true
+    case _: AccessDeniedException =>
+      // 1 - no permission to see s3 bucket
+      // 2 - not authorized to assume the role
+      true
+    case _: DeltaIOException =>
+      // no read/write permission in s3 bucket
+      true
     case t => TableFormatSetupError.check(t)
   }
 }
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -141,9 +141,9 @@ object Dependencies {
 
   val icebergDeltaRuntimeDependencies = Seq(
     iceberg,
-    delta                     % Runtime,
-    Spark.coreForIcebergDelta % Runtime,
-    Spark.sqlForIcebergDelta  % Runtime
+    delta,
+    Spark.coreForIcebergDelta,
+    Spark.sqlForIcebergDelta % Runtime
   )
 
   val coreDependencies = Seq(