Merge branch 'main' into cherry-pick-ce-commit-fbef958eb62663f1a298ed…

…b6daa8ab2d6fc99dd4
Multiwoven · Sep 5, 2024 · 87cdd1d · 87cdd1d
2 parents fdd96ad + 4146f1b
commit 87cdd1d
Show file tree

Hide file tree

Showing 153 changed files with 4,833 additions and 1,015 deletions.
diff --git a/integrations/Gemfile b/integrations/Gemfile
@@ -67,7 +67,11 @@ gem "mysql2"
 
 gem "aws-sdk-sts"
 
-gem "ruby-oci8"
+gem "ruby-oci8", "~> 2.2.12"
+
+gem "aws-sdk-sagemaker"
+
+gem "aws-sdk-sagemakerruntime"
 
 group :development, :test do
   gem "simplecov", require: false

diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock
@@ -7,12 +7,14 @@ GIT
 PATH
   remote: .
   specs:
-    multiwoven-integrations (0.7.9)
+    multiwoven-integrations (0.10.0)
       activesupport
       async-websocket
       aws-sdk-athena
+      aws-sdk-cloudwatchlogs
       aws-sdk-s3
       aws-sdk-sts
+      aws-sigv4
       csv
       dry-schema
       dry-struct
@@ -67,6 +69,9 @@ GEM
     aws-sdk-athena (1.83.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
+    aws-sdk-cloudwatchlogs (1.82.0)
+      aws-sdk-core (~> 3, >= 3.193.0)
+      aws-sigv4 (~> 1.1)
     aws-sdk-core (3.196.1)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.651.0)
@@ -79,6 +84,12 @@ GEM
       aws-sdk-core (~> 3, >= 3.194.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.8)
+    aws-sdk-sagemaker (1.229.0)
+      aws-sdk-core (~> 3, >= 3.188.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-sagemakerruntime (1.63.0)
+      aws-sdk-core (~> 3, >= 3.193.0)
+      aws-sigv4 (~> 1.1)
     aws-sdk-sts (1.11.0)
       aws-sdk-core (~> 3, >= 3.110.0)
       aws-sigv4 (~> 1.1)
@@ -338,6 +349,8 @@ DEPENDENCIES
   async-websocket (~> 0.8.0)
   aws-sdk-athena
   aws-sdk-s3
+  aws-sdk-sagemaker
+  aws-sdk-sagemakerruntime
   aws-sdk-sts
   byebug
   csv
@@ -360,7 +373,7 @@ DEPENDENCIES
   rspec (~> 3.0)
   rubocop (~> 1.21)
   ruby-limiter
-  ruby-oci8
+  ruby-oci8 (~> 2.2.12)
   ruby-odbc!
   rubyzip
   sequel

diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb
@@ -32,6 +32,8 @@
 require "iterable-api-client"
 require "aws-sdk-sts"
 require "ruby-oci8"
+require "aws-sdk-sagemaker"
+require "aws-sdk-sagemakerruntime"
 
 # Service
 require_relative "integrations/config"
@@ -62,6 +64,8 @@
 require_relative "integrations/source/amazon_s3/client"
 require_relative "integrations/source/maria_db/client"
 require_relative "integrations/source/oracle_db/client"
+require_relative "integrations/source/databrics_model/client"
+require_relative "integrations/source/aws_sagemaker_model/client"
 
 # Destination
 require_relative "integrations/destination/klaviyo/client"
@@ -81,6 +85,7 @@
 require_relative "integrations/destination/maria_db/client"
 require_relative "integrations/destination/databricks_lakehouse/client"
 require_relative "integrations/destination/oracle_db/client"
+require_relative "integrations/destination/microsoft_excel/client"
 
 module Multiwoven
   module Integrations

diff --git a/integrations/lib/multiwoven/integrations/core/base_connector.rb b/integrations/lib/multiwoven/integrations/core/base_connector.rb
@@ -63,7 +63,16 @@ def success_status
       end
 
       def failure_status(error)
-        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
+        message = error&.message || "failed"
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: message).to_multiwoven_message
+      end
+
+      def auth_headers(access_token)
+        {
+          "Accept" => "application/json",
+          "Authorization" => "Bearer #{access_token}",
+          "Content-Type" => "application/json"
+        }
       end
     end
   end

diff --git a/integrations/lib/multiwoven/integrations/core/constants.rb b/integrations/lib/multiwoven/integrations/core/constants.rb
@@ -36,14 +36,26 @@ module Constants
       AIRTABLE_BASES_ENDPOINT = "https://api.airtable.com/v0/meta/bases"
       AIRTABLE_GET_BASE_SCHEMA_ENDPOINT = "https://api.airtable.com/v0/meta/bases/{baseId}/tables"
 
-      AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
-      AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]
+      MS_EXCEL_AUTH_ENDPOINT = "https://graph.microsoft.com/v1.0/me"
+      MS_EXCEL_TABLE_ROW_WRITE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
+      "workbook/worksheets/%<sheet_name>s/tables/%<table_name>s/rows"
+      MS_EXCEL_TABLE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/workbook/"\
+      "worksheets/%<sheet_name>s/tables?$select=name"
+      MS_EXCEL_FILES_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/root/children"
+      MS_EXCEL_WORKSHEETS_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
+      "workbook/worksheets"
+      MS_EXCEL_SHEET_RANGE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
+      "workbook/worksheets/%<sheet_name>s/range(address='A1:Z1')/usedRange?$select=values"
+
+      DATABRICKS_HEALTH_URL  = "https://%<databricks_host>s/api/2.0/serving-endpoints/%<endpoint_name>s"
+      DATABRICKS_SERVING_URL = "https://%<databricks_host>s/serving-endpoints/%<endpoint_name>s/invocations"
 
       # HTTP
       HTTP_GET = "GET"
       HTTP_POST = "POST"
       HTTP_PUT = "PUT"
       HTTP_DELETE = "DELETE"
+      HTTP_PATCH = "PATCH"
 
       # google sheets
       GOOGLE_SHEETS_SCOPE = "https://www.googleapis.com/auth/drive"

diff --git a/integrations/lib/multiwoven/integrations/core/http_client.rb b/integrations/lib/multiwoven/integrations/core/http_client.rb
@@ -19,13 +19,14 @@ def build_request(method, uri, payload, headers)
                           when Constants::HTTP_GET then Net::HTTP::Get
                           when Constants::HTTP_POST then Net::HTTP::Post
                           when Constants::HTTP_PUT then Net::HTTP::Put
+                          when Constants::HTTP_PATCH then Net::HTTP::Patch
                           when Constants::HTTP_DELETE then Net::HTTP::Delete
                           else raise ArgumentError, "Unsupported HTTP method: #{method}"
                           end
 
           request = request_class.new(uri)
           headers.each { |key, value| request[key] = value }
-          request.body = payload.to_json if payload && %w[POST PUT].include?(method.upcase)
+          request.body = payload.to_json if payload && %w[POST PUT PATCH].include?(method.upcase)
           request
         end
       end

diff --git a/integrations/lib/multiwoven/integrations/destination/airtable/client.rb b/integrations/lib/multiwoven/integrations/destination/airtable/client.rb
@@ -109,14 +109,6 @@ def create_payload(records)
             }
           end
 
-          def auth_headers(access_token)
-            {
-              "Accept" => "application/json",
-              "Authorization" => "Bearer #{access_token}",
-              "Content-Type" => "application/json"
-            }
-          end
-
           def base_id_exists?(bases, base_id)
             return if extract_bases(bases).any? { |base| base["id"] == base_id }
 

diff --git a/integrations/lib/multiwoven/integrations/destination/airtable/config/meta.json b/integrations/lib/multiwoven/integrations/destination/airtable/config/meta.json
@@ -1,7 +1,7 @@
 {
   "data": {
     "name": "Airtable",
-    "title": "airtable",
+    "title": "Airtable",
     "connector_type": "destination",
     "category": "Productivity Tools",
     "documentation_url": "https://docs.multiwoven.com/destinations/productivity-tools/airtable",

diff --git a/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json b/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json
@@ -1,7 +1,7 @@
 {
   "data": {
     "name": "DatabricksLakehouse",
-    "title": "Databricks Datawarehouse",
+    "title": "Databricks Data Warehouse",
     "connector_type": "destination",
     "category": "Database",
     "documentation_url": "https://docs.multiwoven.com/destinations/databricks_lakehouse",

diff --git a/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb b/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb
@@ -110,14 +110,6 @@ def extract_schema_and_data(records, json_schema)
         [schema, data]
       end
 
-      def auth_headers(access_token)
-        {
-          "Accept" => "application/json",
-          "Authorization" => "Bearer #{access_token}",
-          "Content-Type" => "application/json"
-        }
-      end
-
       def ad_account_exists?(response, ad_account_id)
         return if extract_data(response).any? { |ad_account| ad_account["id"] == "act_#{ad_account_id}" }
 

diff --git a/...rations/lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json b/...rations/lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json
@@ -3,7 +3,7 @@
     "name": "FacebookCustomAudience",
     "title": "Facebook Custom Audiences",
     "connector_type": "destination",
-    "category": "Adtech",
+    "category": "Ad-Tech",
     "documentation_url": "https://docs.mutliwoven.com",
     "github_issue_label": "destination-facebook",
     "icon": "icon.svg",

diff --git a/integrations/lib/multiwoven/integrations/destination/http/config/meta.json b/integrations/lib/multiwoven/integrations/destination/http/config/meta.json
@@ -3,7 +3,7 @@
     "name": "Http",
     "title": "http",
     "connector_type": "destination",
-    "category": "Http",
+    "category": "HTTP",
     "documentation_url": "https://docs.multiwoven.com/destinations/http",
     "github_issue_label": "destination-http",
     "icon": "icon.svg",

diff --git a/integrations/lib/multiwoven/integrations/destination/hubspot/client.rb b/integrations/lib/multiwoven/integrations/destination/hubspot/client.rb
@@ -55,13 +55,15 @@ def initialize_client(config)
           end
 
           def process_records(records, stream)
+            log_message_array = []
             write_success = 0
             write_failure = 0
             properties = stream.json_schema.with_indifferent_access[:properties]
             records.each do |record_object|
               record = extract_data(record_object, properties)
-              send_data_to_hubspot(stream.name, record)
+              request, response = *send_data_to_hubspot(stream.name, record)
               write_success += 1
+              log_message_array << log_request_response("info", request, response)
             rescue StandardError => e
               handle_exception(e, {
                                  context: "HUBSPOT:CRM:WRITE:EXCEPTION",
@@ -70,15 +72,17 @@ def process_records(records, stream)
                                  sync_run_id: @sync_config.sync_run_id
                                })
               write_failure += 1
+              log_message_array << log_request_response("error", request, e.message)
             end
-            tracking_message(write_success, write_failure)
+            tracking_message(write_success, write_failure, log_message_array)
           end
 
           def send_data_to_hubspot(stream_name, record = {})
             args = build_args(@action, stream_name, record)
             hubspot_stream = @client.crm.send(stream_name)
             hubspot_data = { simple_public_object_input_for_create: args }
-            hubspot_stream.basic_api.send(@action, hubspot_data)
+            response = hubspot_stream.basic_api.send(@action, hubspot_data)
+            [args, response]
           end
 
           def build_args(action, stream_name, record)
@@ -108,12 +112,6 @@ def load_catalog
             read_json(CATALOG_SPEC_PATH)
           end
 
-          def tracking_message(success, failure)
-            Multiwoven::Integrations::Protocol::TrackingMessage.new(
-              success: success, failed: failure
-            ).to_multiwoven_message
-          end
-
           def log_debug(message)
             Multiwoven::Integrations::Service.logger.debug(message)
           end