diff --git a/lib/stripe-force/constants.rb b/lib/stripe-force/constants.rb index b2603658f9..1796f0fee8 100644 --- a/lib/stripe-force/constants.rb +++ b/lib/stripe-force/constants.rb @@ -6,6 +6,7 @@ module Constants # application constants POLL_FREQUENCY = T.let(3 * 60, Integer) MAX_STRIPE_PRICE_PRECISION = 12 + MAX_SF_RETRY_ATTEMPTS = 8 SF_ORDER = 'Order' SF_ORDER_ITEM = 'OrderItem' diff --git a/lib/stripe-force/jobs/salesforce_translate_record_job.rb b/lib/stripe-force/jobs/salesforce_translate_record_job.rb index 143280a316..0a2da72428 100644 --- a/lib/stripe-force/jobs/salesforce_translate_record_job.rb +++ b/lib/stripe-force/jobs/salesforce_translate_record_job.rb @@ -34,7 +34,7 @@ def self.perform(salesforce_account_id, stripe_user_id, livemode, sf_record_id) locker = Integrations::Locker.new(user) locker.lock_on_user do - sf_object = user.sf_client.find(sf_record_type, sf_record_id) + sf_object = backoff { user.sf_client.find(sf_record_type, sf_record_id) } StripeForce::Translate.perform( user: user, diff --git a/lib/stripe-force/translate/mapper.rb b/lib/stripe-force/translate/mapper.rb index 16e7f96afd..de3505f4d6 100644 --- a/lib/stripe-force/translate/mapper.rb +++ b/lib/stripe-force/translate/mapper.rb @@ -105,7 +105,7 @@ def extract_salesforce_object_field(sf_object, key_path) target_class = salesforce_type_from_id(target_object) if target_class - target_object = @user.sf_client.find(target_class, target_object) + target_object = backoff { @user.sf_client.find(target_class, target_object) } end end end diff --git a/lib/stripe-force/translate/translate.rb b/lib/stripe-force/translate/translate.rb index 1392064c51..c39982e44c 100644 --- a/lib/stripe-force/translate/translate.rb +++ b/lib/stripe-force/translate/translate.rb @@ -183,22 +183,24 @@ def create_user_failure(salesforce_object:, message:) # interestingly enough, if the external ID field does not exist we'll get a NOT_FOUND response # https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/dome_upsert.htm - sf_sync_record_id = sf.upsert!( - prefixed_stripe_field(SYNC_RECORD), - prefixed_stripe_field(SyncRecordFields::COMPOUND_ID.serialize), - { - SyncRecordFields::COMPOUND_ID => compound_external_id, - - SyncRecordFields::PRIMARY_RECORD_ID => @origin_salesforce_object.Id, - SyncRecordFields::PRIMARY_OBJECT_TYPE => @origin_salesforce_object.sobject_type, - - SyncRecordFields::SECONDARY_RECORD_ID => salesforce_object.Id, - SyncRecordFields::SECONDARY_OBJECT_TYPE => salesforce_object.sobject_type, - - SyncRecordFields::RESOLUTION_MESSAGE => message, - SyncRecordFields::RESOLUTION_STATUS => SyncRecordResolutionStatuses::ERROR, - }.transform_keys(&:serialize).transform_keys(&method(:prefixed_stripe_field)) - ) + sf_sync_record_id = backoff do + sf.upsert!( + prefixed_stripe_field(SYNC_RECORD), + prefixed_stripe_field(SyncRecordFields::COMPOUND_ID.serialize), + { + SyncRecordFields::COMPOUND_ID => compound_external_id, + + SyncRecordFields::PRIMARY_RECORD_ID => @origin_salesforce_object.Id, + SyncRecordFields::PRIMARY_OBJECT_TYPE => @origin_salesforce_object.sobject_type, + + SyncRecordFields::SECONDARY_RECORD_ID => salesforce_object.Id, + SyncRecordFields::SECONDARY_OBJECT_TYPE => salesforce_object.sobject_type, + + SyncRecordFields::RESOLUTION_MESSAGE => message, + SyncRecordFields::RESOLUTION_STATUS => SyncRecordResolutionStatuses::ERROR, + }.transform_keys(&:serialize).transform_keys(&method(:prefixed_stripe_field)) + ) + end log.debug 'sync record created', sync_record_id: sf_sync_record_id end @@ -231,22 +233,24 @@ def create_user_success(salesforce_object:, stripe_object:) # interestingly enough, if the external ID field does not exist we'll get a NOT_FOUND response # https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/dome_upsert.htm - sf.upsert!( - prefixed_stripe_field(SYNC_RECORD), - prefixed_stripe_field(SyncRecordFields::COMPOUND_ID.serialize), - { - SyncRecordFields::COMPOUND_ID => compound_external_id, + backoff do + sf.upsert!( + prefixed_stripe_field(SYNC_RECORD), + prefixed_stripe_field(SyncRecordFields::COMPOUND_ID.serialize), + { + SyncRecordFields::COMPOUND_ID => compound_external_id, - SyncRecordFields::PRIMARY_RECORD_ID => @origin_salesforce_object.Id, - SyncRecordFields::PRIMARY_OBJECT_TYPE => @origin_salesforce_object.sobject_type, + SyncRecordFields::PRIMARY_RECORD_ID => @origin_salesforce_object.Id, + SyncRecordFields::PRIMARY_OBJECT_TYPE => @origin_salesforce_object.sobject_type, - SyncRecordFields::SECONDARY_RECORD_ID => salesforce_object.Id, - SyncRecordFields::SECONDARY_OBJECT_TYPE => salesforce_object.sobject_type, + SyncRecordFields::SECONDARY_RECORD_ID => salesforce_object.Id, + SyncRecordFields::SECONDARY_OBJECT_TYPE => salesforce_object.sobject_type, - SyncRecordFields::RESOLUTION_MESSAGE => message, - SyncRecordFields::RESOLUTION_STATUS => SyncRecordResolutionStatuses::SUCCESS, - }.transform_keys(&:serialize).transform_keys(&method(:prefixed_stripe_field)) - ) + SyncRecordFields::RESOLUTION_MESSAGE => message, + SyncRecordFields::RESOLUTION_STATUS => SyncRecordResolutionStatuses::SUCCESS, + }.transform_keys(&:serialize).transform_keys(&method(:prefixed_stripe_field)) + ) + end end sig do @@ -291,10 +295,12 @@ def update_sf_stripe_id(sf_object, stripe_object, additional_salesforce_updates: field_name: stripe_id_field end - sf.update!(sf_object.sobject_type, { - SF_ID => sf_object.Id, - stripe_id_field => stripe_object_id, - }.merge(additional_salesforce_updates)) + backoff do + sf.update!(sf_object.sobject_type, { + SF_ID => sf_object.Id, + stripe_id_field => stripe_object_id, + }.merge(additional_salesforce_updates)) + end create_user_success( salesforce_object: sf_object, diff --git a/lib/stripe-force/translate/utilities/salesforce_util.rb b/lib/stripe-force/translate/utilities/salesforce_util.rb index d86a695298..b219e6bae8 100644 --- a/lib/stripe-force/translate/utilities/salesforce_util.rb +++ b/lib/stripe-force/translate/utilities/salesforce_util.rb @@ -7,6 +7,7 @@ module SalesforceUtil extend T::Sig include Kernel + include Integrations::Log include StripeForce::Constants # SF dates have no TZ data and come in as a simple 'YYYY-MM-DD' @@ -116,5 +117,42 @@ def prefixed_stripe_field(field_name) custom_field_prefix + field_name end + + sig { params(options: {}).returns(T.untyped) } + def backoff(options={}) + count = 0 + + options[:attempts] ||= if ENV['SALESFORCE_BACKOFF_ATTEMPTS'].nil? + MAX_SF_RETRY_ATTEMPTS + else + ENV['SALESFORCE_BACKOFF_ATTEMPTS'].to_i + end + + begin + count += 1 + + # runs the block (if given) and returns if no errors raised + yield if block_given? + rescue Restforce::ErrorCode::UnableToLockRow, + Restforce::ServerError, + Restforce::NotFoundError, + Faraday::ConnectionFailed, + Faraday::TimeoutError, + Restforce::ResponseError, + Restforce::UnauthorizedError => e + + # log & raise error if all retries fail + if count >= options[:attempts] + log.warn 'finished retrying SF operation, raising error', + attempt: count, + error_class: e.class.to_s, + error_message: e.message + raise e + end + + sleep(count * count) + retry + end + end end end diff --git a/test/unit/test_salesforce_util.rb b/test/unit/test_salesforce_util.rb new file mode 100644 index 0000000000..4bde65c07c --- /dev/null +++ b/test/unit/test_salesforce_util.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true +# typed: true +require_relative '../test_helper' + +module Critic::Unit + class SalesforceUtilTest < Critic::UnitTest + describe '#backoff' do + it 'catch and retry known SF error' do + count = 0 + assert_nothing_raised do + backoff do + # raise the error only once + if count == 0 + count += 1 + raise Restforce::ServerError.new("test") + end + end + end + assert_equal(1, count) + end + + it 'known SF error is raised after max retries' do + count = 0 + assert_raise(Restforce::ServerError) do + backoff do + count += 1 + raise Restforce::ServerError.new("test") + end + end + assert_equal(MAX_SF_RETRY_ATTEMPTS, count) + end + + it 'unknown Salesforce error is raised' do + count = 0 + assert_raise(RuntimeError) do + backoff do + if count == 0 + count += 1 + raise "Unknown error" + end + end + end + assert_equal(1, count) + end + end + end +end