misc(throttling): Implement throttling for all aggregator api calls e…

…xcept hubspot and anrok

app/jobs/integration_customers/create_job.rb

@@ -5,6 +5,7 @@ class CreateJob < ApplicationJob
     queue_as 'integrations'
 
     retry_on LagoHttpClient::HttpError, wait: :polynomially_longer, attempts: 3
+    retry_on BaseService::ThrottlingError, wait: :polynomially_longer, attempts: 100
 
     def perform(integration_customer_params:, integration:, customer:)
       result = IntegrationCustomers::CreateService.call(params: integration_customer_params, integration:, customer:)

app/jobs/integration_customers/update_job.rb

@@ -5,6 +5,7 @@ class UpdateJob < ApplicationJob
     queue_as 'integrations'
 
     retry_on LagoHttpClient::HttpError, wait: :polynomially_longer, attempts: 3
+    retry_on BaseService::ThrottlingError, wait: :polynomially_longer, attempts: 100
 
     def perform(integration_customer_params:, integration:, integration_customer:)
       result = IntegrationCustomers::UpdateService.call(

app/jobs/integrations/aggregator/credit_notes/create_job.rb

@@ -12,6 +12,7 @@ class CreateJob < ApplicationJob
 
         retry_on LagoHttpClient::HttpError, wait: :polynomially_longer, attempts: 3
         retry_on RequestLimitError, wait: :polynomially_longer, attempts: 100
+        retry_on BaseService::ThrottlingError, wait: :polynomially_longer, attempts: 100
 
         def perform(credit_note:)
           result = Integrations::Aggregator::CreditNotes::CreateService.call(credit_note:)

app/jobs/integrations/aggregator/invoices/create_job.rb

@@ -12,6 +12,7 @@ class CreateJob < ApplicationJob
 
         retry_on LagoHttpClient::HttpError, wait: :polynomially_longer, attempts: 3
         retry_on RequestLimitError, wait: :polynomially_longer, attempts: 100
+        retry_on BaseService::ThrottlingError, wait: :polynomially_longer, attempts: 100
 
         def perform(invoice:)
           result = Integrations::Aggregator::Invoices::CreateService.call(invoice:)

app/jobs/integrations/aggregator/payments/create_job.rb

@@ -13,6 +13,7 @@ class CreateJob < ApplicationJob
         retry_on LagoHttpClient::HttpError, wait: :polynomially_longer, attempts: 5
         retry_on Integrations::Aggregator::BasePayload::Failure, wait: :polynomially_longer, attempts: 10
         retry_on RequestLimitError, wait: :polynomially_longer, attempts: 100
+        retry_on BaseService::ThrottlingError, wait: :polynomially_longer, attempts: 100
 
         def perform(payment:)
           result = Integrations::Aggregator::Payments::CreateService.call(payment:)

app/services/base_service.rb

@@ -13,6 +13,8 @@ def initialize(result, message)
     end
   end
 
+  class ThrottlingError < StandardError; end
+
   class NotFoundFailure < FailedResult
     attr_reader :resource
 

app/services/integrations/aggregator/base_service.rb

@@ -50,6 +50,26 @@ def provider_key
         end
       end
 
+      def throttle!(*providers)
+        providers.each do |provider_name|
+          if provider == provider_name.to_s
+            raise BaseService::ThrottlingError unless Throttling.for(provider_name.to_sym).check(:client, throttle_key)
+          end
+        end
+      end
+
+      def throttle_key
+        # Hubspot and Xero calls are throttled globally, others are throttled per api key or token
+        case provider
+        when 'netsuite'
+          integration.token_id
+        when 'anrok'
+          integration.api_key
+        else
+          provider.to_s
+        end
+      end
+
       def http_client
         LagoHttpClient::Client.new(endpoint_url)
       end

app/services/integrations/aggregator/companies/create_service.rb

@@ -14,6 +14,8 @@ def initialize(integration:, customer:, subsidiary_id:)
         def call
           Integrations::Hubspot::Companies::DeployPropertiesService.call(integration:)
 
+          throttle!(:hubspot)
+
           response = http_client.post_with_response(params, headers)
           body = JSON.parse(response.body)
 

app/services/integrations/aggregator/companies/update_service.rb

@@ -15,6 +15,8 @@ def initialize(integration:, integration_customer:)
         def call
           Integrations::Hubspot::Companies::DeployPropertiesService.call(integration:)
 
+          throttle!(:hubspot)
+
           response = http_client.put_with_response(params, headers)
           body = JSON.parse(response.body)
 

app/services/integrations/aggregator/contacts/create_service.rb

@@ -14,6 +14,8 @@ def initialize(integration:, customer:, subsidiary_id:)
         def call
           Integrations::Hubspot::Contacts::DeployPropertiesService.call(integration:)
 
+          throttle!(:anrok, :hubspot, :netsuite, :xero)
+
           response = http_client.post_with_response(params, headers)
           body = JSON.parse(response.body)
 

app/services/integrations/aggregator/contacts/update_service.rb

@@ -13,6 +13,8 @@ def initialize(integration:, integration_customer:)
         def call
           Integrations::Hubspot::Contacts::DeployPropertiesService.call(integration:)
 
+          throttle!(:anrok, :hubspot, :netsuite, :xero)
+
           response = http_client.put_with_response(params, headers)
           body = JSON.parse(response.body)
 

app/services/integrations/aggregator/credit_notes/create_service.rb

@@ -20,6 +20,8 @@ def call
           return result unless credit_note.finalized?
           return result if payload.integration_credit_note
 
+          throttle!(:anrok, :netsuite, :xero)
+
           response = http_client.post_with_response(payload.body, headers)
           body = JSON.parse(response.body)
 

app/services/integrations/aggregator/invoices/create_service.rb

@@ -14,6 +14,8 @@ def call
           return result unless invoice.finalized?
           return result if payload.integration_invoice
 
+          throttle!(:anrok, :netsuite, :xero)
+
           response = http_client.post_with_response(payload.body, headers)
           body = JSON.parse(response.body)
 

app/services/integrations/aggregator/payments/create_service.rb

@@ -20,6 +20,8 @@ def call
           return result unless invoice.finalized?
           return result if payload.integration_payment
 
+          throttle!(:netsuite, :xero)
+
           response = http_client.post_with_response(payload.body, headers)
           body = JSON.parse(response.body)
 

config/initializers/throttling.rb

@@ -6,7 +6,7 @@
 ##
 # Configuration of 'sidekiq-throttled' gem
 #
-# This is the limit of concurrent API calls for Xero and Anrok
+# This is the limit of concurrent API calls for Xero and Netsuite
 Sidekiq::Throttled::Registry.add(:concurrency_limit, concurrency: {limit: 5})
 
 ##
@@ -17,23 +17,29 @@
 
 # Limits per integration and per API key
 Throttling.limits = {
-  hubspot_requests: { # Rate limit: 110 requests per connected account 'hubspot'
+  hubspot: { # Rate limit: 110 requests per 10 seconds
     tensecondly: {
       limit: 110,
       period: 10
     }
   },
-  xero_requests: {
-    minutely: {
+  xero: {
+    minutely: { # Rate limit: 60 requests per minute
       limit: 60,
       period: 60
     },
     daily: {
-      limit: 5000,
+      limit: 5000, # Rate limit: 5000 requests per day
       period: 86400
     }
   },
-  netsuite_requests: { # Rate limit: 10 requests per second per API key. 'integration.client_id' is used
+  netsuite: { # Rate limit: 10 requests per second
+    secondly: {
+      limit: 10,
+      period: 1
+    }
+  },
+  anrok: { # Rate limit: 10 requests per second
     secondly: {
       limit: 10,
       period: 1
@@ -42,6 +48,7 @@
 }
 
 # Examples of how to use the throttling gem
-# Throttling.for(:hubspot_requests).check(:api_key, 'hubspot')
-# Throttling.for(:xero_requests).check(:api_key, integration.client_id)
-# Throttling.for(:netsuite_requests).check(:api_key, integration.client_id)
+# Throttling.for(:hubspot).check(:client, 'hubspot')
+# Throttling.for(:xero).check(:client, 'xero')
+# Throttling.for(:netsuite).check(:client, integration.token_id)
+# Throttling.for(:anrok).check(:client, integration.api_key)