Add backoff for SF operations (#776)

lib/stripe-force/constants.rb

@@ -6,6 +6,7 @@ module Constants
     # application constants
     POLL_FREQUENCY = T.let(3 * 60, Integer)
     MAX_STRIPE_PRICE_PRECISION = 12
+    MAX_SF_RETRY_ATTEMPTS = 8
 
     SF_ORDER = 'Order'
     SF_ORDER_ITEM = 'OrderItem'

lib/stripe-force/jobs/salesforce_translate_record_job.rb

@@ -34,7 +34,7 @@ def self.perform(salesforce_account_id, stripe_user_id, livemode, sf_record_id)
 
     locker = Integrations::Locker.new(user)
     locker.lock_on_user do
-      sf_object = user.sf_client.find(sf_record_type, sf_record_id)
+      sf_object = backoff { user.sf_client.find(sf_record_type, sf_record_id) }
 
       StripeForce::Translate.perform(
         user: user,

lib/stripe-force/translate/mapper.rb

@@ -105,7 +105,7 @@ def extract_salesforce_object_field(sf_object, key_path)
           target_class = salesforce_type_from_id(target_object)
 
           if target_class
-            target_object = @user.sf_client.find(target_class, target_object)
+            target_object = backoff { @user.sf_client.find(target_class, target_object) }
           end
         end
       end

lib/stripe-force/translate/translate.rb

@@ -183,22 +183,24 @@ def create_user_failure(salesforce_object:, message:)
     # interestingly enough, if the external ID field does not exist we'll get a NOT_FOUND response
     # https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/dome_upsert.htm
 
-    sf_sync_record_id = sf.upsert!(
-      prefixed_stripe_field(SYNC_RECORD),
-      prefixed_stripe_field(SyncRecordFields::COMPOUND_ID.serialize),
-      {
-        SyncRecordFields::COMPOUND_ID => compound_external_id,
-
-        SyncRecordFields::PRIMARY_RECORD_ID => @origin_salesforce_object.Id,
-        SyncRecordFields::PRIMARY_OBJECT_TYPE => @origin_salesforce_object.sobject_type,
-
-        SyncRecordFields::SECONDARY_RECORD_ID => salesforce_object.Id,
-        SyncRecordFields::SECONDARY_OBJECT_TYPE => salesforce_object.sobject_type,
-
-        SyncRecordFields::RESOLUTION_MESSAGE => message,
-        SyncRecordFields::RESOLUTION_STATUS => SyncRecordResolutionStatuses::ERROR,
-      }.transform_keys(&:serialize).transform_keys(&method(:prefixed_stripe_field))
-    )
+    sf_sync_record_id = backoff do
+      sf.upsert!(
+        prefixed_stripe_field(SYNC_RECORD),
+        prefixed_stripe_field(SyncRecordFields::COMPOUND_ID.serialize),
+        {
+          SyncRecordFields::COMPOUND_ID => compound_external_id,
+
+          SyncRecordFields::PRIMARY_RECORD_ID => @origin_salesforce_object.Id,
+          SyncRecordFields::PRIMARY_OBJECT_TYPE => @origin_salesforce_object.sobject_type,
+
+          SyncRecordFields::SECONDARY_RECORD_ID => salesforce_object.Id,
+          SyncRecordFields::SECONDARY_OBJECT_TYPE => salesforce_object.sobject_type,
+
+          SyncRecordFields::RESOLUTION_MESSAGE => message,
+          SyncRecordFields::RESOLUTION_STATUS => SyncRecordResolutionStatuses::ERROR,
+        }.transform_keys(&:serialize).transform_keys(&method(:prefixed_stripe_field))
+      )
+    end
 
     log.debug 'sync record created', sync_record_id: sf_sync_record_id
   end
@@ -231,22 +233,24 @@ def create_user_success(salesforce_object:, stripe_object:)
     # interestingly enough, if the external ID field does not exist we'll get a NOT_FOUND response
     # https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/dome_upsert.htm
 
-    sf.upsert!(
-      prefixed_stripe_field(SYNC_RECORD),
-      prefixed_stripe_field(SyncRecordFields::COMPOUND_ID.serialize),
-      {
-        SyncRecordFields::COMPOUND_ID => compound_external_id,
+    backoff do
+      sf.upsert!(
+        prefixed_stripe_field(SYNC_RECORD),
+        prefixed_stripe_field(SyncRecordFields::COMPOUND_ID.serialize),
+        {
+          SyncRecordFields::COMPOUND_ID => compound_external_id,
 
-        SyncRecordFields::PRIMARY_RECORD_ID => @origin_salesforce_object.Id,
-        SyncRecordFields::PRIMARY_OBJECT_TYPE => @origin_salesforce_object.sobject_type,
+          SyncRecordFields::PRIMARY_RECORD_ID => @origin_salesforce_object.Id,
+          SyncRecordFields::PRIMARY_OBJECT_TYPE => @origin_salesforce_object.sobject_type,
 
-        SyncRecordFields::SECONDARY_RECORD_ID => salesforce_object.Id,
-        SyncRecordFields::SECONDARY_OBJECT_TYPE => salesforce_object.sobject_type,
+          SyncRecordFields::SECONDARY_RECORD_ID => salesforce_object.Id,
+          SyncRecordFields::SECONDARY_OBJECT_TYPE => salesforce_object.sobject_type,
 
-        SyncRecordFields::RESOLUTION_MESSAGE => message,
-        SyncRecordFields::RESOLUTION_STATUS => SyncRecordResolutionStatuses::SUCCESS,
-      }.transform_keys(&:serialize).transform_keys(&method(:prefixed_stripe_field))
-    )
+          SyncRecordFields::RESOLUTION_MESSAGE => message,
+          SyncRecordFields::RESOLUTION_STATUS => SyncRecordResolutionStatuses::SUCCESS,
+        }.transform_keys(&:serialize).transform_keys(&method(:prefixed_stripe_field))
+      )
+    end
   end
 
   sig do
@@ -291,10 +295,12 @@ def update_sf_stripe_id(sf_object, stripe_object, additional_salesforce_updates:
         field_name: stripe_id_field
     end
 
-    sf.update!(sf_object.sobject_type, {
-      SF_ID => sf_object.Id,
-      stripe_id_field => stripe_object_id,
-    }.merge(additional_salesforce_updates))
+    backoff do
+      sf.update!(sf_object.sobject_type, {
+        SF_ID => sf_object.Id,
+        stripe_id_field => stripe_object_id,
+      }.merge(additional_salesforce_updates))
+    end
 
     create_user_success(
       salesforce_object: sf_object,

lib/stripe-force/translate/utilities/salesforce_util.rb

@@ -7,6 +7,7 @@ module SalesforceUtil
     extend T::Sig
     include Kernel
 
+    include Integrations::Log
     include StripeForce::Constants
 
     # SF dates have no TZ data and come in as a simple 'YYYY-MM-DD'
@@ -116,5 +117,42 @@ def prefixed_stripe_field(field_name)
 
       custom_field_prefix + field_name
     end
+
+    sig { params(options: {}).returns(T.untyped) }
+    def backoff(options={})
+      count = 0
+
+      options[:attempts] ||= if ENV['SALESFORCE_BACKOFF_ATTEMPTS'].nil?
+        MAX_SF_RETRY_ATTEMPTS
+      else
+        ENV['SALESFORCE_BACKOFF_ATTEMPTS'].to_i
+      end
+
+      begin
+        count += 1
+
+        # runs the block (if given) and returns if no errors raised
+        yield if block_given?
+      rescue Restforce::ErrorCode::UnableToLockRow,
+             Restforce::ServerError,
+             Restforce::NotFoundError,
+             Faraday::ConnectionFailed,
+             Faraday::TimeoutError,
+             Restforce::ResponseError,
+             Restforce::UnauthorizedError => e
+
+        # log & raise error if all retries fail
+        if count >= options[:attempts]
+          log.warn 'finished retrying SF operation, raising error',
+            attempt: count,
+            error_class: e.class.to_s,
+            error_message: e.message
+          raise e
+        end
+
+        sleep(count * count)
+        retry
+      end
+    end
   end
 end

test/unit/test_salesforce_util.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+# typed: true
+require_relative '../test_helper'
+
+module Critic::Unit
+  class SalesforceUtilTest < Critic::UnitTest
+    describe '#backoff' do
+      it 'catch and retry known SF error' do
+        count = 0
+        assert_nothing_raised do
+          backoff do
+            # raise the error only once
+            if count == 0
+              count += 1
+              raise Restforce::ServerError.new("test")
+            end
+          end
+        end
+        assert_equal(1, count)
+      end
+
+      it 'known SF error is raised after max retries' do
+        count = 0
+        assert_raise(Restforce::ServerError) do
+            backoff do
+              count += 1
+              raise Restforce::ServerError.new("test")
+            end
+        end
+        assert_equal(MAX_SF_RETRY_ATTEMPTS, count)
+      end
+
+      it 'unknown Salesforce error is raised' do
+        count = 0
+        assert_raise(RuntimeError) do
+          backoff do
+            if count == 0
+              count += 1
+              raise "Unknown error"
+            end
+          end
+        end
+        assert_equal(1, count)
+      end
+    end
+  end
+end