From 0bce10094723382b8374891157f6c687494160bc Mon Sep 17 00:00:00 2001
From: Oguzhan Unlu <oguzhan@snowplowanalytics.com>
Date: Thu, 1 Aug 2024 17:33:54 +0300
Subject: [PATCH] Add alert & retry for delta/s3 initialization (#74)

---
 .../com.snowplowanalytics.snowplow.lakes/AwsApp.scala | 11 +++++++++++
 .../TableFormatSetupError.scala                       |  5 +++++
 project/Dependencies.scala                            |  6 +++---
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/modules/aws/src/main/scala/com.snowplowanalytics.snowplow.lakes/AwsApp.scala b/modules/aws/src/main/scala/com.snowplowanalytics.snowplow.lakes/AwsApp.scala
index c2f0cc5..4d5285b 100644
--- a/modules/aws/src/main/scala/com.snowplowanalytics.snowplow.lakes/AwsApp.scala
+++ b/modules/aws/src/main/scala/com.snowplowanalytics.snowplow.lakes/AwsApp.scala
@@ -14,6 +14,10 @@ import software.amazon.awssdk.services.s3.model.{NoSuchBucketException, S3Except
 import software.amazon.awssdk.services.sts.model.StsException
 import software.amazon.awssdk.services.glue.model.{AccessDeniedException => GlueAccessDeniedException}
 
+import org.apache.hadoop.fs.s3a.UnknownStoreException
+
+import java.nio.file.AccessDeniedException
+
 import com.snowplowanalytics.snowplow.sources.kinesis.{KinesisSource, KinesisSourceConfig}
 import com.snowplowanalytics.snowplow.sinks.kinesis.{KinesisSink, KinesisSinkConfig}
 
@@ -36,6 +40,13 @@ object AwsApp extends LoaderApp[KinesisSourceConfig, KinesisSinkConfig](BuildInf
     case _: StsException =>
       // No permission to assume the role given to authenticate to S3/Glue
       true
+    case _: UnknownStoreException =>
+      // no such bucket exist
+      true
+    case _: AccessDeniedException =>
+      // 1 - s3 bucket's permission policy denies all actions
+      // 2 - not authorized to assume the role
+      true
     case t => TableFormatSetupError.check(t)
   }
 }
diff --git a/packaging/delta-iceberg/src/main/scala/com.snowplowanalytics.snowplow.lakes/TableFormatSetupError.scala b/packaging/delta-iceberg/src/main/scala/com.snowplowanalytics.snowplow.lakes/TableFormatSetupError.scala
index f455c5c..0cb7e42 100644
--- a/packaging/delta-iceberg/src/main/scala/com.snowplowanalytics.snowplow.lakes/TableFormatSetupError.scala
+++ b/packaging/delta-iceberg/src/main/scala/com.snowplowanalytics.snowplow.lakes/TableFormatSetupError.scala
@@ -12,6 +12,8 @@ package com.snowplowanalytics.snowplow.lakes
 
 import org.apache.iceberg.exceptions.{ForbiddenException => IcebergForbiddenException, NotFoundException => IcebergNotFoundException}
 
+import org.apache.spark.sql.delta.DeltaIOException
+
 object TableFormatSetupError {
 
   // Check if given exception is specific to iceberg format
@@ -23,6 +25,9 @@ object TableFormatSetupError {
       case _: IcebergForbiddenException =>
         // No permission to create a table in Glue catalog
         true
+      case _: DeltaIOException =>
+        // no read/write permission in s3 bucket
+        true
       case _ =>
         false
     }
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index a8d4cb0..88fec70 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -141,9 +141,9 @@ object Dependencies {
 
   val icebergDeltaRuntimeDependencies = Seq(
     iceberg,
-    delta                     % Runtime,
-    Spark.coreForIcebergDelta % Runtime,
-    Spark.sqlForIcebergDelta  % Runtime
+    delta,
+    Spark.coreForIcebergDelta,
+    Spark.sqlForIcebergDelta % Runtime
   )
 
   val coreDependencies = Seq(