From 84efae321a6c94e296fc479e86673ea1cfe06e98 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 31 Jul 2024 16:25:22 +0530 Subject: [PATCH 01/74] chore(CE): verify user mail in signup using devise (#265) Co-authored-by: afthab vp --- server/app/contracts/auth_contracts.rb | 9 +- .../app/controllers/api/v1/auth_controller.rb | 28 +-- .../resend_verification_code.rb | 41 ----- .../resend_verification_email.rb | 18 ++ .../app/interactors/authentication/signup.rb | 33 +--- server/app/models/user.rb | 2 +- .../mailer/confirmation_instructions.html.erb | 171 ++++++++++++++++++ .../send_confirmation_code.html.erb | 2 - server/config/initializers/devise.rb | 1 + server/config/routes.rb | 2 +- ...0240726095056_add_confirmable_to_devise.rb | 8 + server/db/schema.rb | 5 +- server/spec/contracts/auth_contracts_spec.rb | 15 +- .../api/v1/auth_controller_spec.rb | 35 ++-- .../resend_verification_email_spec.rb | 59 ++++++ .../interactors/authentication/signup_spec.rb | 20 +- server/spec/mailers/device_mailer_spec.rb | 30 +++ .../api/v1/connector_definitions_spec.rb | 4 + .../api/v1/connectors_controller_spec.rb | 4 + .../requests/api/v1/models_controller_spec.rb | 4 + .../api/v1/reports_controller_spec.rb | 1 + .../api/v1/schedule_syncs_controller_spec.rb | 1 + .../api/v1/sync_records_controller_spec.rb | 4 + .../api/v1/sync_runs_controller_spec.rb | 4 + .../requests/api/v1/syncs_controller_spec.rb | 1 + .../requests/api/v1/users_controller_spec.rb | 4 + .../api/v1/workspaces_controller_spec.rb | 4 + 27 files changed, 373 insertions(+), 137 deletions(-) delete mode 100644 server/app/interactors/authentication/resend_verification_code.rb create mode 100644 server/app/interactors/authentication/resend_verification_email.rb create mode 100644 server/app/views/devise/mailer/confirmation_instructions.html.erb delete mode 100644 server/app/views/user_mailer/send_confirmation_code.html.erb create mode 100644 server/db/migrate/20240726095056_add_confirmable_to_devise.rb create mode 100644 server/spec/interactors/authentication/resend_verification_email_spec.rb diff --git a/server/app/contracts/auth_contracts.rb b/server/app/contracts/auth_contracts.rb index 580ad92f..6706147d 100644 --- a/server/app/contracts/auth_contracts.rb +++ b/server/app/contracts/auth_contracts.rb @@ -58,14 +58,9 @@ class ResetPassword < Dry::Validation::Contract end end - class VerifyCode < Dry::Validation::Contract + class VerifyUser < Dry::Validation::Contract params do - required(:email).filled(:string) - required(:confirmation_code).filled(:string) - end - - rule(:email) do - key.failure("has invalid email format") unless /\A[\w+\-.]+@[a-z\d-]+(\.[a-z\d-]+)*\.[a-z]+\z/i.match?(value) + required(:confirmation_token).filled(:string) end end diff --git a/server/app/controllers/api/v1/auth_controller.rb b/server/app/controllers/api/v1/auth_controller.rb index c305f5d5..9a0f1197 100644 --- a/server/app/controllers/api/v1/auth_controller.rb +++ b/server/app/controllers/api/v1/auth_controller.rb @@ -28,15 +28,7 @@ def login def signup result = Signup.call(params:) if result.success? - render json: { - data: { - type: "token", - id: result.token, - attributes: { - token: result.token - } - } - }, status: :created + render json: result.user, status: :created else render_error(message: result.errors, status: :unprocessable_entity, details: nil) @@ -81,17 +73,11 @@ def reset_password end end - def verify_code - unless params[:email] && params[:confirmation_code] - return render json: { errors: [{ detail: "Missing required parameters" }] }, status: :bad_request - end - - user = User.find_by(email: params[:email]) - - if user&.confirmation_code == params[:confirmation_code] - user.update!(confirmed_at: Time.current, confirmation_code: nil) + def verify_user + confirmed_user = User.confirm_by_token(params[:confirmation_token]) + if confirmed_user.errors.empty? render json: { data: { type: "message", - id: user.id, + id: confirmed_user.id, attributes: { message: "Account verified successfully!" } } }, status: :ok else @@ -100,11 +86,11 @@ def verify_code end def resend_verification - result = Authentication::ResendVerificationCode.call(params:) + result = Authentication::ResendVerificationEmail.call(params:) if result.success? render json: { data: { type: "message", id: SecureRandom.uuid, - attributes: { message: "Verification code resent successfully." } } }, + attributes: { message: "Email verification link sent successfully!" } } }, status: :ok else render json: { errors: [{ detail: result.error || result.errors }] }, status: :unprocessable_entity diff --git a/server/app/interactors/authentication/resend_verification_code.rb b/server/app/interactors/authentication/resend_verification_code.rb deleted file mode 100644 index 5f85558f..00000000 --- a/server/app/interactors/authentication/resend_verification_code.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -# app/interactors/authentication/resend_verification_code.rb - -module Authentication - class ResendVerificationCode - include Interactor - - def call - find_user - assign_new_confirmation_code - save_user - send_confirmation_email - end - - private - - attr_accessor :user - - def find_user - self.user = User.find_by(email: context.params[:email]) - context.fail!(error: "User not found.", status: :not_found) unless user - end - - def assign_new_confirmation_code - user.confirmation_code = generate_confirmation_code - end - - def save_user - context.fail!(errors: user.errors.full_messages) unless user.save - end - - def send_confirmation_email - UserMailer.send_confirmation_code(user).deliver_now - end - - def generate_confirmation_code - rand(100_000..999_999).to_s - end - end -end diff --git a/server/app/interactors/authentication/resend_verification_email.rb b/server/app/interactors/authentication/resend_verification_email.rb new file mode 100644 index 00000000..573b92f2 --- /dev/null +++ b/server/app/interactors/authentication/resend_verification_email.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Authentication + class ResendVerificationEmail + include Interactor + + def call + user = User.find_by(email: context.params[:email]) + context.fail!(error: "User not found.", status: :not_found) unless user + if !user.confirmed? + user.send_confirmation_instructions + context.message = "Please check your email to confirm your account." + else + context.fail!(error: "Account already confirmed.", status: :unprocessable_entity) + end + end + end +end diff --git a/server/app/interactors/authentication/signup.rb b/server/app/interactors/authentication/signup.rb index c728c2cc..6bab519c 100644 --- a/server/app/interactors/authentication/signup.rb +++ b/server/app/interactors/authentication/signup.rb @@ -8,11 +8,13 @@ def call ActiveRecord::Base.transaction do create_new_user create_organization_and_workspace - # Commenting out the assign_confirmation_code and send_confirmation_email steps - # assign_confirmation_code - # send_confirmation_email save_user - confirm_user_and_generate_token if user.persisted? + if user.persisted? + user.send_confirmation_instructions + context.message = "Signup successful! Please check your email to confirm your account." + else + context.fail!(errors: user.errors.full_messages) + end end rescue ActiveRecord::RecordInvalid => e context.fail!(error: e.message) @@ -38,21 +40,6 @@ def create_organization_and_workspace create_workspace end - def confirm_user_and_generate_token - # Confirm the user - user.update!(confirmed_at: Time.current) - # Generate JWT token, similar to the Login interactor - token, payload = Warden::JWTAuth::UserEncoder.new.call(user, :user, nil) - user.update!(jti: payload["jti"]) - - context.token = token - context.message = "Signup and confirmation successful!" - end - - def assign_confirmation_code - user.confirmation_code = generate_confirmation_code - end - def create_organization self.organization = Organization.new(name: context.params[:company_name]) organization.save @@ -81,13 +68,5 @@ def create_workspace_user role: Role.find_by(role_name: "Admin") ) end - - def send_confirmation_email - UserMailer.send_confirmation_code(user).deliver_now - end - - def generate_confirmation_code - rand(100_000..999_999).to_s - end end end diff --git a/server/app/models/user.rb b/server/app/models/user.rb index 0b13c6b7..ab46c7d7 100644 --- a/server/app/models/user.rb +++ b/server/app/models/user.rb @@ -20,7 +20,7 @@ class User < ApplicationRecord # Include default devise modules. Others available are: # :confirmable, :lockable, :timeoutable, :trackable and :omniauthable - devise :invitable, :database_authenticatable, :registerable, + devise :invitable, :database_authenticatable, :registerable, :confirmable, :recoverable, :rememberable, :validatable, :lockable, :timeoutable, :jwt_authenticatable, jwt_revocation_strategy: self diff --git a/server/app/views/devise/mailer/confirmation_instructions.html.erb b/server/app/views/devise/mailer/confirmation_instructions.html.erb new file mode 100644 index 00000000..ccd2b51e --- /dev/null +++ b/server/app/views/devise/mailer/confirmation_instructions.html.erb @@ -0,0 +1,171 @@ + +<% query_params = { email:@resource.email, confirmation_token: @token } %> +<% confirmation_url = "#{ENV['UI_HOST']}/verify-user?#{query_params.to_query}" %> + + + + + + +
+ + + + + + + + + + + + + +
+ AI Squared Logo +
+

+ Verify your email +

+

+ To complete signup and start using AI Squared, just click the verification button below. +

+ + + +
+

+ This link will expire after it is clicked once. Please use it to verify your email. +

+

+
Cheers,
Team AI Squared +

+
+
+

+ Our blog + | + Docs + | + Contact Us +

+

+ © 2024 AI Squared. All Rights Reserved. +

+
+
+

AI Squared, 1300 I Street NW,

+

Suite 400 E, Washington, DC 20005

+
+
+
+ + \ No newline at end of file diff --git a/server/app/views/user_mailer/send_confirmation_code.html.erb b/server/app/views/user_mailer/send_confirmation_code.html.erb deleted file mode 100644 index 7a9943fe..00000000 --- a/server/app/views/user_mailer/send_confirmation_code.html.erb +++ /dev/null @@ -1,2 +0,0 @@ -

Welcome to Multiwoven

-

Your confirmation code is: <%= @confirmation_code %>

diff --git a/server/config/initializers/devise.rb b/server/config/initializers/devise.rb index a26776a3..ea3a14ce 100644 --- a/server/config/initializers/devise.rb +++ b/server/config/initializers/devise.rb @@ -316,6 +316,7 @@ # The default HTTP method used to sign out a resource. Default is :delete. config.sign_out_via = :delete + config.reconfirmable = false # ==> OmniAuth # Add a new OmniAuth provider. Check the wiki for more information on setting diff --git a/server/config/routes.rb b/server/config/routes.rb index d539f6f5..bf6ecbd6 100644 --- a/server/config/routes.rb +++ b/server/config/routes.rb @@ -10,7 +10,7 @@ namespace :v1 do # Authentication Routes post "signup", to: "auth#signup" - post "verify_code", to: "auth#verify_code" + get "verify_user", to: "auth#verify_user" post "login", to: "auth#login" delete "logout", to: "auth#logout" post "forgot_password", to: "auth#forgot_password" diff --git a/server/db/migrate/20240726095056_add_confirmable_to_devise.rb b/server/db/migrate/20240726095056_add_confirmable_to_devise.rb new file mode 100644 index 00000000..8af18b24 --- /dev/null +++ b/server/db/migrate/20240726095056_add_confirmable_to_devise.rb @@ -0,0 +1,8 @@ +class AddConfirmableToDevise < ActiveRecord::Migration[7.1] + def change + add_column :users, :confirmation_token, :string + add_column :users, :confirmation_sent_at, :datetime + + safety_assured { add_index :users, :confirmation_token, unique: true } + end +end diff --git a/server/db/schema.rb b/server/db/schema.rb index 2909bb15..d6b61860 100644 --- a/server/db/schema.rb +++ b/server/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_07_25_060832) do +ActiveRecord::Schema[7.1].define(version: 2024_07_26_095056) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -159,6 +159,9 @@ t.string "invited_by_type" t.bigint "invited_by_id" t.integer "invitations_count", default: 0 + t.string "confirmation_token" + t.datetime "confirmation_sent_at" + t.index ["confirmation_token"], name: "index_users_on_confirmation_token", unique: true t.index ["email"], name: "index_users_on_email", unique: true t.index ["invitation_token"], name: "index_users_on_invitation_token", unique: true t.index ["invited_by_id"], name: "index_users_on_invited_by_id" diff --git a/server/spec/contracts/auth_contracts_spec.rb b/server/spec/contracts/auth_contracts_spec.rb index 6c46a2d9..2a4ef4f6 100644 --- a/server/spec/contracts/auth_contracts_spec.rb +++ b/server/spec/contracts/auth_contracts_spec.rb @@ -136,28 +136,23 @@ end end - describe AuthContracts::VerifyCode do + describe AuthContracts::VerifyUser do subject(:contract) { described_class.new } context "when valid inputs are provided" do - let(:valid_inputs) do - { - email: "user@example.com", - confirmation_code: "code123" - } - end + let(:valid_inputs) { { confirmation_token: "code12345" } } it "passes validation" do expect(contract.call(valid_inputs)).to be_success end end - context "when invalid email format is provided" do - let(:invalid_inputs) { { email: "not_an_email", confirmation_code: "code123" } } + context "when confirmation_token is empty" do + let(:invalid_inputs) { { confirmation_token: "" } } it "fails validation" do result = contract.call(invalid_inputs) - expect(result.errors[:email]).to include("has invalid email format") + expect(result.errors[:confirmation_token]).to include("must be filled") end end end diff --git a/server/spec/controllers/api/v1/auth_controller_spec.rb b/server/spec/controllers/api/v1/auth_controller_spec.rb index b7f2ab20..10d1c178 100644 --- a/server/spec/controllers/api/v1/auth_controller_spec.rb +++ b/server/spec/controllers/api/v1/auth_controller_spec.rb @@ -17,7 +17,7 @@ def response_errors end let(:user_attributes) { attributes_for(:user) } - let(:user) { create(:user, :verified) } + let(:user) { create(:user) } describe "POST #signup" do context "with valid parameters" do @@ -25,8 +25,8 @@ def response_errors post :signup, params: user_attributes expect(response).to have_http_status(:created) - expect(response_data["type"]).to eq("token") - expect(response_data["attributes"]["token"]).not_to be_nil + expect(response_data["type"]).to eq("users") + expect(response_data["attributes"]["name"]).not_to be_nil end end @@ -53,6 +53,7 @@ def response_errors describe "POST #login" do context "with valid parameters" do it "logs in a user and returns a token" do + user.confirm post :login, params: { email: user.email, password: user.password } expect(response).to have_http_status(:ok) @@ -132,12 +133,12 @@ def response_errors end end - describe "POST #verify_code" do - let(:user_with_code) { create(:user, confirmation_code: "123456") } + describe "GET #verify_user" do + context "with valid confirmation token" do + let(:confirmation_token) { user.confirmation_token } - context "with valid confirmation code" do it "verifies the user and returns a success message" do - post :verify_code, params: { email: user_with_code.email, confirmation_code: user_with_code.confirmation_code } + get :verify_user, params: { confirmation_token: } expect(response).to have_http_status(:ok) expect(response_data["attributes"]["message"]).to eq("Account verified successfully!") @@ -146,7 +147,7 @@ def response_errors context "with invalid confirmation code" do it "does not verify the user and returns an error" do - post :verify_code, params: { email: user_with_code.email, confirmation_code: "wrong" } + get :verify_user, params: { confirmation_token: "wrong123" } expect(response).to have_http_status(:unprocessable_entity) expect(response_errors).not_to be_empty @@ -155,7 +156,7 @@ def response_errors context "with no parameters" do it "returns a bad request status with an error message" do - post :verify_code + get :verify_user expect(response).to have_http_status(:bad_request) expect(response_errors).not_to be_empty @@ -166,12 +167,18 @@ def response_errors describe "POST #resend_verification" do let(:unverified_user) { create(:user) } # Assuming this creates an unverified user - context "resending verification code" do - it "sends a new verification code" do + context "resending verification email" do + it "sends a new verification email" do post :resend_verification, params: { email: unverified_user.email } - expect(response).to have_http_status(:ok) - expect(response_data["attributes"]["message"]).to eq("Verification code resent successfully.") + expect(response_data["attributes"]["message"]).to eq("Email verification link sent successfully!") + end + + it "returns an error already confirmed" do + unverified_user.confirm + post :resend_verification, params: { email: unverified_user.email } + expect(response).to have_http_status(:unprocessable_entity) + expect(response_errors[0]["detail"]).to include("Account already confirmed") end end @@ -181,6 +188,7 @@ def response_errors expect(response).to have_http_status(:unprocessable_entity) expect(response_errors).not_to be_empty + expect(response_errors[0]["detail"]).to include("User not found") end end end @@ -195,6 +203,7 @@ def response_errors context "when it is an authenticated user" do it "returns success and logout user" do + user.confirm request.headers.merge!(auth_headers(user, 0)) delete :logout expect(response).to have_http_status(:ok) diff --git a/server/spec/interactors/authentication/resend_verification_email_spec.rb b/server/spec/interactors/authentication/resend_verification_email_spec.rb new file mode 100644 index 00000000..e263d509 --- /dev/null +++ b/server/spec/interactors/authentication/resend_verification_email_spec.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Authentication::ResendVerificationEmail, type: :interactor do + describe ".call" do + let(:email) { "test@example.com" } + let(:user) { create(:user, email:, confirmed_at: nil) } + let(:context) { described_class.call(params: { email: }) } + + context "when user is found" do + before do + allow(User).to receive(:find_by).with(email:).and_return(user) + end + + context "when user is not confirmed" do + it "sends confirmation instructions" do + expect(user).to receive(:send_confirmation_instructions) + context + end + + it "sets a success message" do + expect(context.message).to eq("Please check your email to confirm your account.") + end + + it "does not fail the context" do + expect(context).not_to be_failure + end + end + + context "when user is already confirmed" do + let(:user) { create(:user, email:, confirmed_at: Time.current) } + + it "does not send confirmation instructions" do + expect(user).not_to receive(:send_confirmation_instructions) + context + end + + it "fails the context with an appropriate error" do + expect(context).to be_failure + expect(context.error).to eq("Account already confirmed.") + expect(context.status).to eq(:unprocessable_entity) + end + end + end + + context "when user is not found" do + before do + allow(User).to receive(:find_by).with(email:).and_return(nil) + end + + it "fails the context with an appropriate error" do + expect(context).to be_failure + expect(context.error).to eq("User not found.") + expect(context.status).to eq(:not_found) + end + end + end +end diff --git a/server/spec/interactors/authentication/signup_spec.rb b/server/spec/interactors/authentication/signup_spec.rb index af4d2b24..31674f02 100644 --- a/server/spec/interactors/authentication/signup_spec.rb +++ b/server/spec/interactors/authentication/signup_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -# spec/interactors/authentication/signup_spec.rb - require "rails_helper" RSpec.describe Authentication::Signup, type: :interactor do @@ -23,19 +21,11 @@ expect(context).to be_success end - it "confirms the user" do - expect(User.find_by(email: params[:email])).to be_nil - - # Execute the interactor - context - + it "creates and does not confirm the user" do + expect { context }.to change(User, :count).by(1) user = User.find_by(email: params[:email]) expect(user).not_to be_nil - expect(user.confirmed_at).not_to be_nil - end - - it "provides a JWT token" do - expect(context.token).not_to be_nil + expect(user.confirmed_at).to be_nil end it "creates a new user" do @@ -49,6 +39,10 @@ it "creates a new workspace" do expect { context }.to change(Workspace, :count).by(1) end + + it "returns a success message" do + expect(context.message).to eq("Signup successful! Please check your email to confirm your account.") + end end context "when company_name is not present" do diff --git a/server/spec/mailers/device_mailer_spec.rb b/server/spec/mailers/device_mailer_spec.rb index b0c96ab3..655a6745 100644 --- a/server/spec/mailers/device_mailer_spec.rb +++ b/server/spec/mailers/device_mailer_spec.rb @@ -96,4 +96,34 @@ expect(mail.body.encoded).to match("Your password has been changed") end end + + describe "confirmation_instructions" do + let(:user) { create(:user) } + let(:token) { "testtoken" } + let(:mail) { DeviseMailer.confirmation_instructions(user, token) } + + before do + allow(ENV).to receive(:[]).with("UI_HOST").and_return("https://example.com") + user.update(confirmation_sent_at: Time.current) + end + + it "renders the headers" do + expect(mail.subject).to eq("Confirmation instructions") + expect(mail.to).to eq([user.email]) + end + + it "renders the body" do + query_params = [ + ["confirmation_token", token], + ["email", user.email] + ] + expect(mail.body.encoded).to match("Verify your email") + expect(mail.body.encoded) + .to match("To complete signup and start using AI Squared, just click the verification button below.") + doc = Nokogiri::HTML(mail.body.encoded) + link = doc.at_css("a")["href"] + reset_url = "https://example.com/verify-user?#{URI.encode_www_form(query_params)}" + expect(link).to eq(reset_url) + end + end end diff --git a/server/spec/requests/api/v1/connector_definitions_spec.rb b/server/spec/requests/api/v1/connector_definitions_spec.rb index 09646637..42d8e008 100644 --- a/server/spec/requests/api/v1/connector_definitions_spec.rb +++ b/server/spec/requests/api/v1/connector_definitions_spec.rb @@ -11,6 +11,10 @@ let(:viewer_role) { create(:role, :viewer) } let(:member_role) { create(:role, :member) } + before do + user.confirm + end + describe "GET /api/v1/connector_definitions" do context "when it is an unauthenticated user" do it "returns unauthorized" do diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb index e060326e..0ed99941 100644 --- a/server/spec/requests/api/v1/connectors_controller_spec.rb +++ b/server/spec/requests/api/v1/connectors_controller_spec.rb @@ -15,6 +15,10 @@ ] end + before do + user.confirm + end + describe "GET /api/v1/connectors" do context "when it is an unauthenticated user" do it "returns unauthorized" do diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb index fef7697f..8f2721ed 100644 --- a/server/spec/requests/api/v1/models_controller_spec.rb +++ b/server/spec/requests/api/v1/models_controller_spec.rb @@ -18,6 +18,10 @@ let(:viewer_role) { create(:role, :viewer) } let(:member_role) { create(:role, :member) } + before do + user.confirm + end + describe "GET /api/v1/models" do context "when it is an unauthenticated user" do it "returns unauthorized" do diff --git a/server/spec/requests/api/v1/reports_controller_spec.rb b/server/spec/requests/api/v1/reports_controller_spec.rb index c2a248bb..e6e5e114 100644 --- a/server/spec/requests/api/v1/reports_controller_spec.rb +++ b/server/spec/requests/api/v1/reports_controller_spec.rb @@ -16,6 +16,7 @@ let(:member_role) { create(:role, :member) } before do + user.confirm create(:catalog, connector: connectors.find { |connector| connector.name == "klavio1" }, workspace:) create(:catalog, connector: connectors.find { |connector| connector.name == "redshift" }, workspace:) end diff --git a/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb b/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb index 706a7749..a9131a4b 100644 --- a/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb +++ b/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb @@ -17,6 +17,7 @@ end before do + user.confirm create(:catalog, connector: connectors.find { |connector| connector.name == "klavio1" }, workspace:) create(:catalog, connector: connectors.find { |connector| connector.name == "redshift" }, workspace:) end diff --git a/server/spec/requests/api/v1/sync_records_controller_spec.rb b/server/spec/requests/api/v1/sync_records_controller_spec.rb index 94252262..990e5eaa 100644 --- a/server/spec/requests/api/v1/sync_records_controller_spec.rb +++ b/server/spec/requests/api/v1/sync_records_controller_spec.rb @@ -26,6 +26,10 @@ let(:viewer_role) { create(:role, :viewer) } let(:member_role) { create(:role, :member) } + before do + user.confirm + end + describe "GET /api/v1/syncs/sync_id/sync_runs/sync_run_id/sync_records" do context "when it is an unauthenticated user" do it "returns unauthorized" do diff --git a/server/spec/requests/api/v1/sync_runs_controller_spec.rb b/server/spec/requests/api/v1/sync_runs_controller_spec.rb index 1b4cc73f..18454f4a 100644 --- a/server/spec/requests/api/v1/sync_runs_controller_spec.rb +++ b/server/spec/requests/api/v1/sync_runs_controller_spec.rb @@ -24,6 +24,10 @@ let(:viewer_role) { create(:role, :viewer) } let(:member_role) { create(:role, :member) } + before do + user.confirm + end + describe "GET /api/v1/syncs/sync_id/sync_runs" do context "when it is an unauthenticated user" do it "returns unauthorized" do diff --git a/server/spec/requests/api/v1/syncs_controller_spec.rb b/server/spec/requests/api/v1/syncs_controller_spec.rb index 9ca82fb3..41d813c9 100644 --- a/server/spec/requests/api/v1/syncs_controller_spec.rb +++ b/server/spec/requests/api/v1/syncs_controller_spec.rb @@ -21,6 +21,7 @@ end before do + user.confirm create(:catalog, connector: connectors.find { |connector| connector.name == "klavio1" }, workspace:) create(:catalog, connector: connectors.find { |connector| connector.name == "redshift" }, workspace:) end diff --git a/server/spec/requests/api/v1/users_controller_spec.rb b/server/spec/requests/api/v1/users_controller_spec.rb index 0edf2d5e..51ec3637 100644 --- a/server/spec/requests/api/v1/users_controller_spec.rb +++ b/server/spec/requests/api/v1/users_controller_spec.rb @@ -9,6 +9,10 @@ let(:viewer_role) { create(:role, :viewer) } let(:member_role) { create(:role, :member) } + before do + user.confirm + end + describe "GET /api/v1/users/me" do context "when it is an unauthenticated user" do it "returns unauthorized" do diff --git a/server/spec/requests/api/v1/workspaces_controller_spec.rb b/server/spec/requests/api/v1/workspaces_controller_spec.rb index b9cb5d0d..bc6fe1c2 100644 --- a/server/spec/requests/api/v1/workspaces_controller_spec.rb +++ b/server/spec/requests/api/v1/workspaces_controller_spec.rb @@ -9,6 +9,10 @@ let(:viewer_role) { create(:role, :viewer) } let(:member_role) { create(:role, :member) } + before do + user.confirm + end + describe "GET /api/v1/workspaces" do context "when it is an unauthenticated user" do it "returns unauthorized" do From 0a0edcbf0218af1920867d45ce389036116393ae Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 31 Jul 2024 18:17:01 +0530 Subject: [PATCH 02/74] feat(CE): Add sync run type column to sync run records table --- ui/src/components/DataTable/Table.tsx | 7 +- ui/src/components/TypeTag/TypeTag.tsx | 31 +++++ ui/src/components/TypeTag/index.ts | 1 + ui/src/hooks/useErrorToast.tsx | 39 ++++++ .../Syncs/SyncRecords/SyncRecordsTopBar.tsx | 6 + .../Syncs/SyncRuns/SyncRunTableItem.tsx | 130 +++++++++++++----- .../Activate/Syncs/SyncRuns/SyncRuns.tsx | 45 +++--- .../Syncs/SyncRuns/SyncRunsColumns.tsx | 97 +++++++++++++ .../__tests__/SyncRunsTableItem.test.tsx | 2 + ui/src/views/Activate/Syncs/constants.ts | 4 + ui/src/views/Activate/Syncs/types.ts | 2 + 11 files changed, 303 insertions(+), 61 deletions(-) create mode 100644 ui/src/components/TypeTag/TypeTag.tsx create mode 100644 ui/src/components/TypeTag/index.ts create mode 100644 ui/src/hooks/useErrorToast.tsx create mode 100644 ui/src/views/Activate/Syncs/SyncRuns/SyncRunsColumns.tsx diff --git a/ui/src/components/DataTable/Table.tsx b/ui/src/components/DataTable/Table.tsx index f624e169..f230c9cc 100644 --- a/ui/src/components/DataTable/Table.tsx +++ b/ui/src/components/DataTable/Table.tsx @@ -1,13 +1,14 @@ import { Table, Tbody, Td, Th, Thead, Tr } from '@chakra-ui/react'; -import { flexRender, getCoreRowModel, useReactTable, ColumnDef } from '@tanstack/react-table'; +import { flexRender, getCoreRowModel, useReactTable, ColumnDef, Row } from '@tanstack/react-table'; import { useState } from 'react'; type DataTableProps = { columns: ColumnDef[]; data: TData[]; + onRowClick?: (row: Row) => void; }; -const DataTable = ({ data, columns }: DataTableProps) => { +const DataTable = ({ data, columns, onRowClick }: DataTableProps) => { const [rowSelection, setRowSelection] = useState({}); const table = useReactTable({ @@ -46,7 +47,7 @@ const DataTable = ({ data, columns }: DataTableProps onRowClick?.(row)} + onClick={() => onRowClick?.(row)} backgroundColor='gray.100' > {row.getVisibleCells().map((cell) => ( diff --git a/ui/src/components/TypeTag/TypeTag.tsx b/ui/src/components/TypeTag/TypeTag.tsx new file mode 100644 index 00000000..28656191 --- /dev/null +++ b/ui/src/components/TypeTag/TypeTag.tsx @@ -0,0 +1,31 @@ +import { Tag, Text, Icon, Box } from '@chakra-ui/react'; +import { IconType } from 'react-icons/lib'; + +type TypeTagProps = { + label: string; + leftIcon?: IconType; +}; + +const TypeTag = ({ label, leftIcon }: TypeTagProps) => ( + + + + + {label} + + + +); + +export default TypeTag; diff --git a/ui/src/components/TypeTag/index.ts b/ui/src/components/TypeTag/index.ts new file mode 100644 index 00000000..967b4cd1 --- /dev/null +++ b/ui/src/components/TypeTag/index.ts @@ -0,0 +1 @@ +export { default } from './TypeTag'; diff --git a/ui/src/hooks/useErrorToast.tsx b/ui/src/hooks/useErrorToast.tsx new file mode 100644 index 00000000..94474929 --- /dev/null +++ b/ui/src/hooks/useErrorToast.tsx @@ -0,0 +1,39 @@ +import { useCallback, useEffect } from 'react'; +import useCustomToast from '@/hooks/useCustomToast'; +import { CustomToastStatus } from '@/components/Toast'; +import { ErrorResponse } from '@/services/common'; + +export const useErrorToast = (isError: boolean, data: any, isFetched: boolean, message: string) => { + const showToast = useCustomToast(); + + useEffect(() => { + if (isError || (!data && isFetched)) { + showToast({ + title: `Error: ${message}`, + description: message, + status: CustomToastStatus.Error, + position: 'bottom-right', + }); + } + }, [isError, data, isFetched, showToast, message]); +}; + +export const useAPIErrorsToast = () => { + const showToast = useCustomToast(); + + const showAPIErrorsToast = useCallback( + (errors: ErrorResponse[]) => { + errors.forEach((error) => { + showToast({ + status: CustomToastStatus.Warning, + title: error.detail, + position: 'bottom-right', + isClosable: true, + }); + }); + }, + [showToast], + ); + + return showAPIErrorsToast; +}; diff --git a/ui/src/views/Activate/Syncs/SyncRecords/SyncRecordsTopBar.tsx b/ui/src/views/Activate/Syncs/SyncRecords/SyncRecordsTopBar.tsx index c563f13f..2a967cb1 100644 --- a/ui/src/views/Activate/Syncs/SyncRecords/SyncRecordsTopBar.tsx +++ b/ui/src/views/Activate/Syncs/SyncRecords/SyncRecordsTopBar.tsx @@ -10,10 +10,12 @@ import moment from 'moment'; import { useEffect } from 'react'; import { useStore } from '@/stores'; import { useSyncStore } from '@/stores/useSyncStore'; +import { useAPIErrorsToast } from '@/hooks/useErrorToast'; export const SyncRecordsTopBar = ({ syncId, syncRunId }: { syncId: string; syncRunId: string }) => { const activeWorkspaceId = useStore((state) => state.workspaceId); const selectedSync = useSyncStore((state) => state.selectedSync); + const apiErrorToast = useAPIErrorsToast(); const toast = useCustomToast(); @@ -25,6 +27,10 @@ export const SyncRecordsTopBar = ({ syncId, syncRunId }: { syncId: string; syncR enabled: activeWorkspaceId > 0, }); + if (syncRunData?.errors && syncRunData?.errors.length > 0) { + apiErrorToast(syncRunData.errors); + } + const VIEW_SYNC_RUN_RECORDS_STEPS: Step[] = [ { name: 'Syncs', diff --git a/ui/src/views/Activate/Syncs/SyncRuns/SyncRunTableItem.tsx b/ui/src/views/Activate/Syncs/SyncRuns/SyncRunTableItem.tsx index c7c2bd48..43da041c 100644 --- a/ui/src/views/Activate/Syncs/SyncRuns/SyncRunTableItem.tsx +++ b/ui/src/views/Activate/Syncs/SyncRuns/SyncRunTableItem.tsx @@ -4,54 +4,114 @@ import moment from 'moment'; import StatusTag from '@/components/StatusTag'; import { ResultEntity } from './ResultEntity'; import { StatusTagText, StatusTagVariants } from '@/components/StatusTag/StatusTag'; +import TypeTag from '@/components/TypeTag'; +import { FiCheckCircle, FiRefreshCw } from 'react-icons/fi'; -type TableItem = { +type TableItemProps = { field: SyncRunsColumnFields; data: SyncRunsResponse; }; -export const TableItem = ({ field, data }: TableItem): JSX.Element => { - switch (field) { - case 'start_time': - return ( - - {moment(data.attributes.started_at).format('DD/MM/YYYY')} at{' '} - {moment(data.attributes.started_at).format('HH:mm a')} - - ); +const StartTime = ({ started_at }: { started_at: string | null }) => { + if (!started_at) return null; - case 'duration': - return {data.attributes.duration?.toPrecision(3)} seconds; + return ( + + {moment(started_at).format('DD/MM/YYYY')} at {moment(started_at).format('HH:mm a')} + + ); +}; - case 'skipped_rows': - return {data.attributes.skipped_rows} rows; +const Duration = ({ duration }: { duration: number | null }) => { + if (duration == null) return null; - case 'rows_queried': - return {data.attributes.total_query_rows} rows; + return {duration.toPrecision(3)} seconds; +}; + +const SyncRunType = ({ sync_run_type }: { sync_run_type: string }) => ( + +); + +const SkippedRows = ({ skipped_rows }: { skipped_rows: number }) => ( + {skipped_rows} rows +); - case 'status': { - const variant = data.attributes.status as StatusTagVariants; - const tagText = StatusTagText[variant]; +const RowsQueried = ({ total_query_rows }: { total_query_rows: number }) => ( + {total_query_rows} rows +); - return ; - } +const Status = ({ status }: { status: StatusTagVariants }) => { + const tagText = StatusTagText[status]; + return ; +}; + +const Results = ({ + successful_rows, + failed_rows, + total_query_rows, +}: { + successful_rows: number; + failed_rows: number; + total_query_rows: number; +}) => ( + + + + +); + +export const TableItem = ({ field, data }: TableItemProps): JSX.Element => { + if (!data?.attributes) return <>; + + const { + started_at, + duration, + sync_run_type, + skipped_rows, + total_query_rows, + status, + successful_rows, + failed_rows, + } = data.attributes; + + switch (field) { + case 'start_time': + return ; + case 'duration': + return ; + case 'sync_run_type': + return ; + case 'skipped_rows': + return ; + case 'rows_queried': + return ; + case 'status': + return ; case 'results': return ( - - - - + ); + default: + return <>; } }; + +export default TableItem; diff --git a/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx b/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx index 84b2e7f8..cd0aa12d 100644 --- a/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx +++ b/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx @@ -2,12 +2,12 @@ import { useQuery } from '@tanstack/react-query'; import { useNavigate, useParams, useSearchParams } from 'react-router-dom'; import { getSyncRunsBySyncId } from '@/services/syncs'; import { useMemo, useState, useEffect } from 'react'; -import { SYNC_RUNS_COLUMNS } from '@/views/Activate/Syncs/constants'; -import { Box } from '@chakra-ui/react'; +import { Box, Image, Text } from '@chakra-ui/react'; import Loader from '@/components/Loader'; -import Table from '@/components/Table'; -import { TableItem } from '@/views/Activate/Syncs/SyncRuns/SyncRunTableItem'; import Pagination from '@/components/Pagination'; +import { SyncRunsColumns } from './SyncRunsColumns'; +import DataTable from '@/components/DataTable'; +import SyncRunEmptyImage from '@/assets/images/empty-state-illustration.svg'; const SyncRuns = () => { const { syncId } = useParams(); @@ -34,24 +34,7 @@ const SyncRuns = () => { const syncList = data?.data; - const tableData = useMemo(() => { - const rows = (syncList ?? []).map((data) => { - return SYNC_RUNS_COLUMNS.reduce( - (acc, { key }) => ({ - ...acc, - [key]: , - id: data.id, - }), - {}, - ); - }); - - return { - columns: SYNC_RUNS_COLUMNS, - data: rows, - error: '', - }; - }, [data]); + const allColumns = useMemo(() => [...SyncRunsColumns], [SyncRunsColumns]); const handleNextPage = () => { setCurrentPage((prevPage) => Math.min(prevPage + 1)); @@ -67,7 +50,23 @@ const SyncRuns = () => { ) : ( - + {data?.data?.length === 0 || !data?.data ? ( + + + + No rows found + + + ) : ( + + )} { + const tagText = StatusTagText[value]; + return ; +}; + +const StartTimeCell = ({ value }: { value: string }) => { + if (!value) return null; + return ( + + {moment(value).format('DD/MM/YYYY')} at {moment(value).format('HH:mm a')} + + ); +}; + +const SyncRunTypeCell = ({ value }: { value: string }) => { + return ; +}; + +const DurationCell = ({ value }: { value: number }) => { + if (value == null) return null; + return {value.toPrecision(3)} seconds; +}; + +const RowCountCell = ({ value, label }: { value: number; label: string }) => { + return ( + + {value} {label} + + ); +}; + +const ResultsCell = ({ value }: { value: SyncRunsResponse['attributes'] }) => { + return ( + + + + + ); +}; + +export const SyncRunsColumns: ColumnDef[] = [ + { + accessorKey: 'attributes.status', + header: () => Status, + cell: (info) => , + }, + { + accessorKey: 'attributes.started_at', + header: () => Start Time, + cell: (info) => , + }, + { + accessorKey: 'attributes.sync_run_type', + header: () => Sync Run Type, + cell: (info) => , + }, + { + accessorKey: 'attributes.duration', + header: () => Duration, + cell: (info) => , + }, + { + accessorKey: 'attributes.total_query_rows', + header: () => Rows Queried, + cell: (info) => , + }, + { + accessorKey: 'attributes.skipped_rows', + header: () => Skipped Rows, + cell: (info) => , + }, + { + accessorKey: 'attributes', + header: () => Results, + cell: (info) => , + }, +]; diff --git a/ui/src/views/Activate/Syncs/__tests__/SyncRunsTableItem.test.tsx b/ui/src/views/Activate/Syncs/__tests__/SyncRunsTableItem.test.tsx index 7c64421a..8d12d853 100644 --- a/ui/src/views/Activate/Syncs/__tests__/SyncRunsTableItem.test.tsx +++ b/ui/src/views/Activate/Syncs/__tests__/SyncRunsTableItem.test.tsx @@ -24,6 +24,7 @@ const mockSyncRunsData: SyncRunsResponse[] = [ successful_rows: 500, failed_rows: 0, error: null, + sync_run_type: 'general', }, }, { @@ -45,6 +46,7 @@ const mockSyncRunsData: SyncRunsResponse[] = [ successful_rows: 450, failed_rows: 50, error: null, + sync_run_type: 'general', }, }, ]; diff --git a/ui/src/views/Activate/Syncs/constants.ts b/ui/src/views/Activate/Syncs/constants.ts index 28ea22b7..cc0a6681 100644 --- a/ui/src/views/Activate/Syncs/constants.ts +++ b/ui/src/views/Activate/Syncs/constants.ts @@ -28,6 +28,10 @@ export const SYNC_RUNS_COLUMNS: SyncRunsColumnEntity[] = [ key: 'start_time', name: 'start_time', }, + { + key: 'sync_run_type', + name: 'Sync Type', + }, { key: 'duration', name: 'Duration', diff --git a/ui/src/views/Activate/Syncs/types.ts b/ui/src/views/Activate/Syncs/types.ts index cf0db7f7..21d29dc3 100644 --- a/ui/src/views/Activate/Syncs/types.ts +++ b/ui/src/views/Activate/Syncs/types.ts @@ -151,6 +151,7 @@ export type SyncRunsResponse = { successful_rows: number; failed_rows: number; error: ErrorResponse | null; + sync_run_type: string; }; id: string; type: 'sync_runs'; @@ -159,6 +160,7 @@ export type SyncRunsResponse = { export type SyncRunsColumnFields = | 'status' | 'start_time' + | 'sync_run_type' | 'duration' | 'rows_queried' | 'skipped_rows' From 0deced4ed685eb5e25eb4305eb0b6f8dccba4ae3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:59:47 +0530 Subject: [PATCH 03/74] feat(CE): test sync extractor changes (#270) Co-authored-by: afthab vp --- .../activities/create_sync_run_activity.rb | 3 +- .../temporal/activities/extractor_activity.rb | 11 +- .../app/temporal/workflows/sync_workflow.rb | 4 +- server/lib/reverse_etl/extractors/base.rb | 60 ++++++++++ .../extractors/incremental_delta.rb | 60 ---------- .../extractors/test_sync_extractor.rb | 96 +++++++++++++++ server/lib/reverse_etl/loaders/standard.rb | 2 +- .../reverse_etl/utils/random_query_builder.rb | 19 +++ .../extractors/test_sync_extractor_spec.rb | 113 ++++++++++++++++++ .../utils/random_query_builder_spec.rb | 61 ++++++++++ .../create_sync_run_activity_spec.rb | 21 +++- .../activities/extractor_activity_spec.rb | 41 ++++++- .../temporal/workflows/sync_workflow_spec.rb | 11 ++ 13 files changed, 430 insertions(+), 72 deletions(-) create mode 100644 server/lib/reverse_etl/extractors/test_sync_extractor.rb create mode 100644 server/lib/reverse_etl/utils/random_query_builder.rb create mode 100644 server/spec/lib/reverse_etl/extractors/test_sync_extractor_spec.rb create mode 100644 server/spec/lib/reverse_etl/utils/random_query_builder_spec.rb diff --git a/server/app/temporal/activities/create_sync_run_activity.rb b/server/app/temporal/activities/create_sync_run_activity.rb index 2bee7ed2..48b485e5 100644 --- a/server/app/temporal/activities/create_sync_run_activity.rb +++ b/server/app/temporal/activities/create_sync_run_activity.rb @@ -7,13 +7,14 @@ class CreateSyncRunActivity < Temporal::Activity backoff: 1, max_attempts: 3 ) - def execute(sync_id) + def execute(sync_id, sync_run_type) sync = Sync.find(sync_id) sync_run = SyncRun.find_or_initialize_by(sync_id:, status: :pending) do |run| run.workspace_id = sync.workspace_id run.source_id = sync.source_id run.destination_id = sync.destination_id run.model_id = sync.model_id + run.sync_run_type = sync_run_type end sync_run.save! if sync_run.new_record? sync_run.id diff --git a/server/app/temporal/activities/extractor_activity.rb b/server/app/temporal/activities/extractor_activity.rb index 3cad0739..415168e5 100644 --- a/server/app/temporal/activities/extractor_activity.rb +++ b/server/app/temporal/activities/extractor_activity.rb @@ -30,7 +30,16 @@ def execute(sync_run_id) private def select_extractor(sync_run) - sync_mode = sync_run.sync.sync_mode.to_sym + return test_sync_extractor if sync_run.test? + + sync_mode_extractor(sync_run.sync.sync_mode.to_sym) + end + + def test_sync_extractor + ReverseEtl::Extractors::TestSyncExtractor.new + end + + def sync_mode_extractor(sync_mode) case sync_mode when :incremental ReverseEtl::Extractors::IncrementalDelta.new diff --git a/server/app/temporal/workflows/sync_workflow.rb b/server/app/temporal/workflows/sync_workflow.rb index 2b1c1779..e86396c3 100644 --- a/server/app/temporal/workflows/sync_workflow.rb +++ b/server/app/temporal/workflows/sync_workflow.rb @@ -3,11 +3,11 @@ module Workflows class SyncWorkflow < Temporal::Workflow include Activities - def execute(sync_id) + def execute(sync_id, sync_run_type = "general") sync = FetchSyncActivity.execute!(sync_id) return if sync.disabled? - sync_run_id = CreateSyncRunActivity.execute!(sync.id) + sync_run_id = CreateSyncRunActivity.execute!(sync.id, sync_run_type) ExtractorActivity.execute!(sync_run_id) diff --git a/server/lib/reverse_etl/extractors/base.rb b/server/lib/reverse_etl/extractors/base.rb index d4365fde..88743d00 100644 --- a/server/lib/reverse_etl/extractors/base.rb +++ b/server/lib/reverse_etl/extractors/base.rb @@ -57,6 +57,66 @@ def log_sync_run_error(sync_run) stack_trace: nil }.to_s) end + + def process_record(record, sync_run, model) + primary_key = record.data.with_indifferent_access[model.primary_key] + raise StandardError, "Primary key cannot be blank" if primary_key.blank? + + find_or_initialize_sync_record(sync_run, primary_key) + rescue StandardError => e + # ::Utils::ExceptionReporter.report(e, { + # sync_run_id: sync_run.id + # }) + Rails.logger.error({ + error_message: e.message, + sync_run_id: sync_run.id, + sync_id: sync_run.sync_id, + stack_trace: Rails.backtrace_cleaner.clean(e.backtrace) + }.to_s) + nil + end + + def find_or_initialize_sync_record(sync_run, primary_key) + # In parallel processing, we encountered a scenario where one thread was processing and had not yet persisted + # to the database, while another thread attempted to create a new record with the same primary key + # for a synchronization. To prevent this, we used database constraints. However, + # in future cases where a synchronization contains both create and update operations, + # there might be a risk of losing either the update or the create due to these concurrent operations. + # we can use ActiveRecord::Base.transaction to prevent such scenarios + SyncRecord.find_by(sync_id: sync_run.sync_id, primary_key:) || + sync_run.sync_records.new(sync_id: sync_run.sync_id, primary_key:, created_at: DateTime.current) + end + + def new_record?(sync_record, fingerprint) + sync_record.new_record? || sync_record.fingerprint != fingerprint + end + + def action(sync_record) + sync_record.new_record? ? :destination_insert : :destination_update + end + + def update_or_create_sync_record(sync_record, record, sync_run, fingerprint) + unless sync_record && new_record?(sync_record, fingerprint) + primary_key = record.data.with_indifferent_access[sync_run.sync.model.primary_key] + Rails.logger.info({ + message: "Skipping sync record", + primary_key:, + sync_id: sync_run.sync_id, + sync_run_id: sync_run.id, + sync_record_id: sync_record&.id + }.to_s) + + return false + end + sync_record.assign_attributes( + sync_run_id: sync_run.id, + action: action(sync_record), + fingerprint:, + record: record.data, + status: "pending" + ) + sync_record.save! + end end end end diff --git a/server/lib/reverse_etl/extractors/incremental_delta.rb b/server/lib/reverse_etl/extractors/incremental_delta.rb index 6e8a91ad..39b96787 100644 --- a/server/lib/reverse_etl/extractors/incremental_delta.rb +++ b/server/lib/reverse_etl/extractors/incremental_delta.rb @@ -44,66 +44,6 @@ def process_records(records, sync_run, model) update_or_create_sync_record(sync_record, record, sync_run, fingerprint) ? 0 : 1 end.sum end - - def process_record(record, sync_run, model) - primary_key = record.data.with_indifferent_access[model.primary_key] - raise StandardError, "Primary key cannot be blank" if primary_key.blank? - - find_or_initialize_sync_record(sync_run, primary_key) - rescue StandardError => e - # ::Utils::ExceptionReporter.report(e, { - # sync_run_id: sync_run.id - # }) - Rails.logger.error({ - error_message: e.message, - sync_run_id: sync_run.id, - sync_id: sync_run.sync_id, - stack_trace: Rails.backtrace_cleaner.clean(e.backtrace) - }.to_s) - nil - end - - def find_or_initialize_sync_record(sync_run, primary_key) - # In parallel processing, we encountered a scenario where one thread was processing and had not yet persisted - # to the database, while another thread attempted to create a new record with the same primary key - # for a synchronization. To prevent this, we used database constraints. However, - # in future cases where a synchronization contains both create and update operations, - # there might be a risk of losing either the update or the create due to these concurrent operations. - # we can use ActiveRecord::Base.transaction to prevent such scenarios - SyncRecord.find_by(sync_id: sync_run.sync_id, primary_key:) || - sync_run.sync_records.new(sync_id: sync_run.sync_id, primary_key:, created_at: DateTime.current) - end - - def new_record?(sync_record, fingerprint) - sync_record.new_record? || sync_record.fingerprint != fingerprint - end - - def action(sync_record) - sync_record.new_record? ? :destination_insert : :destination_update - end - - def update_or_create_sync_record(sync_record, record, sync_run, fingerprint) - unless sync_record && new_record?(sync_record, fingerprint) - primary_key = record.data.with_indifferent_access[sync_run.sync.model.primary_key] - Rails.logger.info({ - message: "Skipping sync record", - primary_key:, - sync_id: sync_run.sync_id, - sync_run_id: sync_run.id, - sync_record_id: sync_record&.id - }.to_s) - - return false - end - sync_record.assign_attributes( - sync_run_id: sync_run.id, - action: action(sync_record), - fingerprint:, - record: record.data, - status: "pending" - ) - sync_record.save! - end end end end diff --git a/server/lib/reverse_etl/extractors/test_sync_extractor.rb b/server/lib/reverse_etl/extractors/test_sync_extractor.rb new file mode 100644 index 00000000..c665761a --- /dev/null +++ b/server/lib/reverse_etl/extractors/test_sync_extractor.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module ReverseEtl + module Extractors + class TestSyncExtractor < Base + def read(sync_run_id, activity) + sync_run = SyncRun.find(sync_run_id) + return log_sync_run_error(sync_run) unless sync_run.may_query? + + sync_run.query! + + result = fetch_records(sync_run) + + process_result(result, sync_run) + + heartbeat(activity, sync_run) + + # change state querying to queued + sync_run.queue! + end + + private + + def fetch_records(sync_run) + source_client = setup_source_client(sync_run.sync) + modified_sync_config = build_random_record_query_sync_config(sync_run.sync.to_protocol) + result = source_client.read(modified_sync_config) + if result.nil? || !result.is_a?(Array) || result.count != 1 + Rails.logger.error({ + error_message: "Expected exactly one record in the result query = #{modified_sync_config.model.query}. + Result = #{result.inspect}", + sync_run_id: sync_run.id, + sync_id: sync_run.sync_id, + stack_trace: nil + }.to_s) + raise "Expected exactly one record in the result, but got #{result.inspect}" + end + result + end + + def build_random_record_query_sync_config(sync_config) + random_query = ReverseEtl::Utils::RandomQueryBuilder.build_random_record_query(sync_config) + new_model = build_new_model(sync_config.model, random_query) + + modified_sync_config = Multiwoven::Integrations::Protocol::SyncConfig.new( + model: new_model.to_protocol, + source: sync_config.source, + destination: sync_config.destination, + stream: sync_config.stream, + sync_mode: sync_config.sync_mode, + destination_sync_mode: sync_config.destination_sync_mode, + cursor_field: sync_config.cursor_field, + current_cursor_field: sync_config.current_cursor_field, + sync_id: sync_config.sync_id + ) + modified_sync_config.offset = 0 + modified_sync_config.limit = 1 + modified_sync_config.sync_run_id = sync_config.sync_run_id + modified_sync_config + end + + def build_new_model(existing_model, new_query) + Model.new( + name: existing_model.name, + query: new_query, + query_type: existing_model.query_type, + primary_key: existing_model.primary_key + ) + end + + def process_result(result, sync_run) + record = result.first.record + fingerprint = generate_fingerprint(record.data) + model = sync_run.sync.model + sync_record = process_record(record, sync_run, model) + skipped_rows = update_or_create_sync_record(sync_record, record, sync_run, fingerprint) ? 0 : 1 + sync_run.update(current_offset: 0, total_query_rows: 1, skipped_rows:) + end + + def heartbeat(activity, sync_run) + response = activity.heartbeat + return unless response.cancel_requested + + sync_run.failed! + sync_run.sync_records.delete_all + Rails.logger.error({ + error_message: "Cancel activity request received", + sync_run_id: sync_run.id, + sync_id: sync_run.sync_id, + stack_trace: nil + }.to_s) + raise StandardError, "Cancel activity request received" + end + end + end +end diff --git a/server/lib/reverse_etl/loaders/standard.rb b/server/lib/reverse_etl/loaders/standard.rb index 67b6f5a7..897cd2b4 100644 --- a/server/lib/reverse_etl/loaders/standard.rb +++ b/server/lib/reverse_etl/loaders/standard.rb @@ -16,7 +16,7 @@ def write(sync_run_id, activity) sync_config = sync.to_protocol sync_config.sync_run_id = sync_run.id.to_s - if sync_config.stream.batch_support + if sync_config.stream.batch_support && !sync_run.test? process_batch_records(sync_run, sync, sync_config, activity) else process_individual_records(sync_run, sync, sync_config, activity) diff --git a/server/lib/reverse_etl/utils/random_query_builder.rb b/server/lib/reverse_etl/utils/random_query_builder.rb new file mode 100644 index 00000000..b8edaabe --- /dev/null +++ b/server/lib/reverse_etl/utils/random_query_builder.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module ReverseEtl + module Utils + class RandomQueryBuilder + def self.build_random_record_query(sync_config) + existing_query = sync_config.model.query + query_type = sync_config.source.query_type || "raw_sql" + + case query_type.to_sym + when :soql + existing_query + when :raw_sql + "SELECT * FROM (#{existing_query}) AS subquery ORDER BY RANDOM()" + end + end + end + end +end diff --git a/server/spec/lib/reverse_etl/extractors/test_sync_extractor_spec.rb b/server/spec/lib/reverse_etl/extractors/test_sync_extractor_spec.rb new file mode 100644 index 00000000..6329d0ff --- /dev/null +++ b/server/spec/lib/reverse_etl/extractors/test_sync_extractor_spec.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe ReverseEtl::Extractors::TestSyncExtractor do + let(:source) { create(:connector, connector_type: "source", connector_name: "Snowflake") } + let(:destination) { create(:connector, connector_type: "destination") } + let!(:catalog) { create(:catalog, connector: destination) } + let(:sync) { create(:sync, source:, destination:) } + let(:sync_run1) do + create(:sync_run, sync:, workspace: sync.workspace, source:, destination:, model: sync.model, status: "started") + end + let(:activity) { instance_double("ExtractorActivity") } + + let(:client) { instance_double(Multiwoven::Integrations::Source::Snowflake::Client) } + let(:record1) do + Multiwoven::Integrations::Protocol::RecordMessage.new( + data: { "id" => 1, "email" => "test1@mail.com", "first_name" => "John", "Last Name" => "Doe" }, + emitted_at: DateTime.now.to_i + ).to_multiwoven_message + end + + before do + sync.model.update(primary_key: "id") + allow_any_instance_of(described_class).to receive(:setup_source_client).and_return(client) + allow(client).to receive(:read).and_return([record1]) + allow(sync_run1.sync.source).to receive_message_chain(:connector_client, :new).and_return(client) + allow(activity).to receive(:heartbeat).and_return(activity) + allow(activity).to receive(:cancel_requested).and_return(false) + end + + describe "#read" do + subject { described_class.new } + + context "when there is a new record" do + it "creates a new sync record" do + expect(subject).to receive(:heartbeat).exactly(:once) + expect { subject.read(sync_run1.id, activity) }.to change { sync_run1.sync_records.count }.by(1) + sync_run1.reload + expect(sync_run1.current_offset).to eq(0) + expect(sync_run1.total_query_rows).to eq(1) + expect(sync_run1.skipped_rows).to eq(0) + end + end + + context "when an existing record is updated" do + it "updates the existing sync record with fingerprint change" do + # First sync run + expect(sync_run1).to have_state(:started) + subject.read(sync_run1.id, activity) + sync_run1.reload + expect(sync_run1.sync_records.count).to eq(1) + expect(sync_run1).to have_state(:queued) + expect(sync_run1.current_offset).to eq(0) + expect(sync_run1.total_query_rows).to eq(1) + expect(sync_run1.skipped_rows).to eq(0) + + initial_sync_record = sync_run1.sync_records.find_by(primary_key: record1.record.data["id"]) + expect(initial_sync_record.fingerprint).to eq(subject.send(:generate_fingerprint, record1.record.data)) + expect(initial_sync_record.action).to eq("destination_insert") + + modified_record1 = Multiwoven::Integrations::Protocol::RecordMessage.new( + data: record1.record.data.merge({ "modified_field" => "new_value" }), + emitted_at: DateTime.now.to_i + ).to_multiwoven_message + + allow(client).to receive(:read).and_return([modified_record1]) + + # Second sync run + sync_run2 = create(:sync_run, sync:, workspace: sync.workspace, source:, destination:, model: sync.model, + status: "started") + expect(sync_run2).to have_state(:started) + subject.read(sync_run2.id, activity) + sync_run2.reload + expect(sync_run2).to have_state(:queued) + expect(sync_run2.sync_records.count).to eq(1) + expect(sync_run2.current_offset).to eq(0) + expect(sync_run2.total_query_rows).to eq(1) + expect(sync_run2.skipped_rows).to eq(0) + + updated_sync_record = sync_run2.sync_records.find_by(primary_key: record1.record.data["id"]) + expect(updated_sync_record.fingerprint).not_to eq(initial_sync_record.fingerprint) + expect(updated_sync_record.action).to eq("destination_update") + expect(updated_sync_record.record).to eq(modified_record1.record.data) + end + + it "handles heartbeat timeout and updates sync run state" do + expect(sync_run1).to have_state(:started) + allow(activity).to receive(:cancel_requested).and_return(true) + expect { subject.read(sync_run1.id, activity) } + .to raise_error(StandardError, "Cancel activity request received") + sync_run1.reload + expect(sync_run1.sync_records.count).to eq(0) + expect(sync_run1).to have_state(:failed) + end + end + + context "when sync run is in pending state" do + let(:sync_run_pending) do + create(:sync_run, sync:, workspace: sync.workspace, source:, destination:, model: sync.model, status: "pending") + end + + it "does not process the record" do + expect(sync_run_pending).to have_state(:pending) + expect(subject).not_to receive(:heartbeat) + expect(subject).not_to receive(:setup_source_client) + expect(subject).not_to receive(:process_record) + subject.read(sync_run_pending.id, activity) + expect(sync_run_pending).to have_state(:pending) + end + end + end +end diff --git a/server/spec/lib/reverse_etl/utils/random_query_builder_spec.rb b/server/spec/lib/reverse_etl/utils/random_query_builder_spec.rb new file mode 100644 index 00000000..366deb0b --- /dev/null +++ b/server/spec/lib/reverse_etl/utils/random_query_builder_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require "rails_helper" + +module ReverseEtl + module Utils + RSpec.describe RandomQueryBuilder do + let(:existing_query) { "SELECT * FROM table" } + let(:source) { create(:connector, connector_type: "source", connector_name: "Snowflake") } + let(:source_salesforce) do + create(:connector, connector_type: "source", connector_name: "SalesforceConsumerGoodsCloud") + end + let(:destination) { create(:connector, connector_type: "destination") } + let!(:catalog) { create(:catalog, connector: destination) } + let(:model) { create(:model, connector: source, query: existing_query) } + let(:model_salesforce) { create(:model, connector: source, query: existing_query) } + + describe ".build_random_record_query" do + context "when query_type is raw_sql" do + let(:sync) do + create(:sync, model:, source:, destination:) + end + let(:sync_config) { sync.to_protocol } + + it "returns the query with ORDER BY RANDOM()" do + query = described_class.build_random_record_query(sync_config) + + expected_query = "SELECT * FROM (#{existing_query}) AS subquery ORDER BY RANDOM()" + expect(query).to eq(expected_query) + end + end + + context "when query_type is soql" do + let(:sync_salesforce) do + create(:sync, model: model_salesforce, source: source_salesforce, destination:) + end + let(:sync_config_salesforce) { sync_salesforce.to_protocol } + + it "returns the query" do + query = described_class.build_random_record_query(sync_config_salesforce) + expect(query).to eq(existing_query) + end + end + + context "when query_type is not specified" do + let(:sync) do + create(:sync, model:, source:, destination:) + end + let(:sync_config) { sync.to_protocol } + + it "assumes raw_sql and returns the query with ORDER BY RANDOM()" do + query = described_class.build_random_record_query(sync_config) + + expected_query = "SELECT * FROM (#{existing_query}) AS subquery ORDER BY RANDOM()" + expect(query).to eq(expected_query) + end + end + end + end + end +end diff --git a/server/spec/temporal/activities/create_sync_run_activity_spec.rb b/server/spec/temporal/activities/create_sync_run_activity_spec.rb index 1c4d5ced..6d5ed142 100644 --- a/server/spec/temporal/activities/create_sync_run_activity_spec.rb +++ b/server/spec/temporal/activities/create_sync_run_activity_spec.rb @@ -14,9 +14,9 @@ let(:activity) { Activities::CreateSyncRunActivity.new(mock_context) } context "when no pending SyncRun exists" do - it "creates a new SyncRun with pending status" do + it "creates a new SyncRun with type general and pending status" do expect do - sync_run_id = activity.execute(sync.id) + sync_run_id = activity.execute(sync.id, "general") sync_run = SyncRun.find(sync_run_id) expect(sync_run).to have_state(:pending) expect(sync_run.sync_id).to eq(sync.id) @@ -24,6 +24,21 @@ expect(sync_run.source_id).to eq(sync.source_id) expect(sync_run.destination_id).to eq(sync.destination_id) expect(sync_run.model_id).to eq(sync.model_id) + expect(sync_run.sync_run_type).to eq("general") + end.to change(SyncRun, :count).by(1) + end + + it "creates a new SyncRun with type test and pending status" do + expect do + sync_run_id = activity.execute(sync.id, "test") + sync_run = SyncRun.find(sync_run_id) + expect(sync_run).to have_state(:pending) + expect(sync_run.sync_id).to eq(sync.id) + expect(sync_run.workspace_id).to eq(sync.workspace_id) + expect(sync_run.source_id).to eq(sync.source_id) + expect(sync_run.destination_id).to eq(sync.destination_id) + expect(sync_run.model_id).to eq(sync.model_id) + expect(sync_run.sync_run_type).to eq("test") end.to change(SyncRun, :count).by(1) end end @@ -35,7 +50,7 @@ it "create a new SyncRun" do expect do - sync_run_id = activity.execute(sync.id) + sync_run_id = activity.execute(sync.id, "general") sync_run = SyncRun.find(sync_run_id) expect(sync_run).to have_state(:pending) expect(sync_run.sync_id).to eq(sync.id) diff --git a/server/spec/temporal/activities/extractor_activity_spec.rb b/server/spec/temporal/activities/extractor_activity_spec.rb index ea72e09b..9ed48938 100644 --- a/server/spec/temporal/activities/extractor_activity_spec.rb +++ b/server/spec/temporal/activities/extractor_activity_spec.rb @@ -97,7 +97,7 @@ end end - describe "#select_extractor" do + describe "#sync_mode_extractor" do let(:sync_run) { instance_double("SyncRun", sync: instance_double("Sync", sync_mode:)) } let(:mock_context) { double("context") } let(:activity) { Activities::ExtractorActivity.new(mock_context) } @@ -106,7 +106,7 @@ let(:sync_mode) { "incremental" } it "returns an instance of IncrementalDelta extractor" do - extractor = activity.send(:select_extractor, sync_run) + extractor = activity.send(:sync_mode_extractor, sync_run.sync.sync_mode.to_sym) expect(extractor).to be_a(ReverseEtl::Extractors::IncrementalDelta) end end @@ -115,7 +115,7 @@ let(:sync_mode) { "full_refresh" } it "returns an instance of FullRefresh extractor" do - extractor = activity.send(:select_extractor, sync_run) + extractor = activity.send(:sync_mode_extractor, sync_run.sync.sync_mode.to_sym) expect(extractor).to be_a(ReverseEtl::Extractors::FullRefresh) end end @@ -125,9 +125,42 @@ it "raises an error" do expect do - activity.send(:select_extractor, sync_run) + activity.send(:sync_mode_extractor, sync_run.sync.sync_mode.to_sym) end.to raise_error(RuntimeError, "Unsupported sync mode: #{sync_mode}") end end end + + describe "#select_extractor" do + let(:destination) { create(:connector, connector_type: "destination") } + let!(:catalog) { create(:catalog, connector: destination) } + let!(:sync) { create(:sync, destination:) } + let(:sync_run) { create(:sync_run, sync:, sync_run_type: "general") } + let(:sync_run_test) { create(:sync_run, sync:, sync_run_type: "test") } + let(:mock_context) { double("context") } + let(:activity) { Activities::ExtractorActivity.new(mock_context) } + + context "when sync_run_type is test" do + it "returns an instance of TestSyncExtractor" do + extractor = activity.send(:select_extractor, sync_run_test) + expect(extractor).to be_a(ReverseEtl::Extractors::TestSyncExtractor) + end + end + + context "when sync_run_type is not test in incremental" do + it "returns an instance of IncrementalDelta extractor for incremental sync_mode" do + sync.update(sync_mode: "incremental") + extractor = activity.send(:select_extractor, sync_run) + expect(extractor).to be_a(ReverseEtl::Extractors::IncrementalDelta) + end + end + + context "when sync_run_type is not test in fullrefresh" do + it "returns an instance of FullRefresh extractor for full_refresh sync_mode" do + sync.update(sync_mode: "full_refresh") + extractor = activity.send(:select_extractor, sync_run) + expect(extractor).to be_a(ReverseEtl::Extractors::FullRefresh) + end + end + end end diff --git a/server/spec/temporal/workflows/sync_workflow_spec.rb b/server/spec/temporal/workflows/sync_workflow_spec.rb index 75b120fd..f18dbe81 100644 --- a/server/spec/temporal/workflows/sync_workflow_spec.rb +++ b/server/spec/temporal/workflows/sync_workflow_spec.rb @@ -23,5 +23,16 @@ expect(Activities::ExtractorActivity).to have_received(:execute!).with(sync.sync_runs.first.id) expect(Activities::LoaderActivity).to have_received(:execute!).with(sync.sync_runs.first.id) expect(Activities::ReporterActivity).to have_received(:execute!).with(sync.sync_runs.first.id) + expect(sync.sync_runs.first.sync_run_type).to eq("general") + end + + it "executes sync workflow with sync run type test" do + # TODO: Add more tests + subject.execute_locally(sync.id, "test") + + expect(Activities::ExtractorActivity).to have_received(:execute!).with(sync.sync_runs.first.id) + expect(Activities::LoaderActivity).to have_received(:execute!).with(sync.sync_runs.first.id) + expect(Activities::ReporterActivity).to have_received(:execute!).with(sync.sync_runs.first.id) + expect(sync.sync_runs.first.sync_run_type).to eq("test") end end From 4aaec59c91dda60e7f5248a5c99f10dc9cba6b6f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 1 Aug 2024 19:09:52 +0530 Subject: [PATCH 04/74] refactor(CE): wrapped main layout with protector --- ui/src/App.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ui/src/App.tsx b/ui/src/App.tsx index d6f34375..f27b078a 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -67,7 +67,14 @@ const App = (): JSX.Element => { ))} - }> + + + + } + > {MAIN_PAGE_ROUTES.map(({ url, component, name }) => ( {component}} /> ))} From c30838279e9bf5249df92cb19d7fc33264873ccb Mon Sep 17 00:00:00 2001 From: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> Date: Fri, 2 Aug 2024 04:56:40 -0400 Subject: [PATCH 05/74] chore(CE): update server gem 0.5.2 (#275) --- server/Gemfile | 2 +- server/Gemfile.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/Gemfile b/server/Gemfile index 61071a5a..35703cb6 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -13,7 +13,7 @@ gem "interactor", "~> 3.0" gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git" -gem "multiwoven-integrations", "~> 0.5.1" +gem "multiwoven-integrations", "~> 0.5.2" gem "temporal-ruby", github: "coinbase/temporal-ruby" diff --git a/server/Gemfile.lock b/server/Gemfile.lock index 3a3a9225..f95ef77b 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -1774,7 +1774,7 @@ GEM addressable (~> 2.8) process_executer (~> 1.1) rchardet (~> 1.8) - gli (2.21.1) + gli (2.21.3) globalid (1.2.1) activesupport (>= 6.1) google-apis-bigquery_v2 (0.72.0) @@ -1894,7 +1894,7 @@ GEM msgpack (1.7.2) multi_json (1.15.0) multipart-post (2.4.1) - multiwoven-integrations (0.5.1) + multiwoven-integrations (0.5.2) activesupport async-websocket aws-sdk-athena @@ -2191,7 +2191,7 @@ DEPENDENCIES jwt kaminari liquid - multiwoven-integrations (~> 0.5.1) + multiwoven-integrations (~> 0.5.2) mysql2 newrelic_rpm parallel From 68dd88dbbffbb52c232165e74d19ecf03482ffec Mon Sep 17 00:00:00 2001 From: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> Date: Fri, 2 Aug 2024 05:02:31 -0400 Subject: [PATCH 06/74] feat(CE): add oracle db destination connector (#277) --- .github/workflows/integrations-ci.yml | 8 + .github/workflows/integrations-main.yml | 8 + integrations/Gemfile | 2 + integrations/Gemfile.lock | 6 +- integrations/lib/multiwoven/integrations.rb | 2 + .../destination/oracle_db/client.rb | 112 ++++++++++++++ .../destination/oracle_db/config/meta.json | 15 ++ .../destination/oracle_db/config/spec.json | 47 ++++++ .../destination/oracle_db/icon.svg | 4 + .../lib/multiwoven/integrations/rollout.rb | 3 +- integrations/multiwoven-integrations.gemspec | 1 + .../destination/oracle_db/client_spec.rb | 146 ++++++++++++++++++ 12 files changed, 352 insertions(+), 2 deletions(-) create mode 100644 integrations/lib/multiwoven/integrations/destination/oracle_db/client.rb create mode 100644 integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json create mode 100644 integrations/lib/multiwoven/integrations/destination/oracle_db/config/spec.json create mode 100644 integrations/lib/multiwoven/integrations/destination/oracle_db/icon.svg create mode 100644 integrations/spec/multiwoven/integrations/destination/oracle_db/client_spec.rb diff --git a/.github/workflows/integrations-ci.yml b/.github/workflows/integrations-ci.yml index 4e8c44d9..1eca6149 100644 --- a/.github/workflows/integrations-ci.yml +++ b/.github/workflows/integrations-ci.yml @@ -31,6 +31,14 @@ jobs: sudo mv libduckdb/libduckdb.so /usr/local/lib sudo ldconfig /usr/local/lib + - name: Download and Install Oracle Instant Client + run: | + sudo apt-get install -y libaio1 alien + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm + sudo alien -i --scripts oracle-instantclient*.rpm + rm -f oracle-instantclient*.rpm + - name: Install dependencies run: | gem install bundler diff --git a/.github/workflows/integrations-main.yml b/.github/workflows/integrations-main.yml index d9ecb8ad..fd43f306 100644 --- a/.github/workflows/integrations-main.yml +++ b/.github/workflows/integrations-main.yml @@ -39,6 +39,14 @@ jobs: sudo mv libduckdb/libduckdb.so /usr/local/lib sudo ldconfig /usr/local/lib + - name: Download and Install Oracle Instant Client + run: | + sudo apt-get install -y libaio1 alien + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm + sudo alien -i --scripts oracle-instantclient*.rpm + rm -f oracle-instantclient*.rpm + - name: Install dependencies run: bundle install working-directory: ./integrations diff --git a/integrations/Gemfile b/integrations/Gemfile index b28b59ce..1b82fbae 100644 --- a/integrations/Gemfile +++ b/integrations/Gemfile @@ -67,6 +67,8 @@ gem "mysql2" gem "aws-sdk-sts" +gem "ruby-oci8" + group :development, :test do gem "simplecov", require: false gem "simplecov_json_formatter", require: false diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index fb0b8ee2..5843cd3a 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.5.2) + multiwoven-integrations (0.6.0) activesupport async-websocket aws-sdk-athena @@ -28,6 +28,7 @@ PATH rake restforce ruby-limiter + ruby-oci8 ruby-odbc rubyzip sequel @@ -275,6 +276,8 @@ GEM rubocop-ast (1.31.3) parser (>= 3.3.1.0) ruby-limiter (2.3.0) + ruby-oci8 (2.2.12) + ruby-oci8 (2.2.12-x64-mingw-ucrt) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) rubyzip (2.3.2) @@ -357,6 +360,7 @@ DEPENDENCIES rspec (~> 3.0) rubocop (~> 1.21) ruby-limiter + ruby-oci8 ruby-odbc! rubyzip sequel diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb index d305120f..dd8636cb 100644 --- a/integrations/lib/multiwoven/integrations.rb +++ b/integrations/lib/multiwoven/integrations.rb @@ -31,6 +31,7 @@ require "duckdb" require "iterable-api-client" require "aws-sdk-sts" +require "ruby-oci8" # Service require_relative "integrations/config" @@ -78,6 +79,7 @@ require_relative "integrations/destination/iterable/client" require_relative "integrations/destination/maria_db/client" require_relative "integrations/destination/databricks_lakehouse/client" +require_relative "integrations/destination/oracle_db/client" module Multiwoven module Integrations diff --git a/integrations/lib/multiwoven/integrations/destination/oracle_db/client.rb b/integrations/lib/multiwoven/integrations/destination/oracle_db/client.rb new file mode 100644 index 00000000..da848bf8 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/oracle_db/client.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Multiwoven::Integrations::Destination + module Oracle + include Multiwoven::Integrations::Core + class Client < DestinationConnector + def check_connection(connection_config) + connection_config = connection_config.with_indifferent_access + create_connection(connection_config) + ConnectionStatus.new( + status: ConnectionStatusType["succeeded"] + ).to_multiwoven_message + rescue StandardError => e + ConnectionStatus.new( + status: ConnectionStatusType["failed"], message: e.message + ).to_multiwoven_message + end + + def discover(connection_config) + records = [] + connection_config = connection_config.with_indifferent_access + query = "SELECT table_name, column_name, data_type, nullable + FROM all_tab_columns + WHERE owner = '#{connection_config[:username].upcase}' + ORDER BY table_name, column_id" + conn = create_connection(connection_config) + cursor = conn.exec(query) + while (row = cursor.fetch) + records << row + end + catalog = Catalog.new(streams: create_streams(records)) + catalog.to_multiwoven_message + rescue StandardError => e + handle_exception( + "ORACLE:DISCOVER:EXCEPTION", + "error", + e + ) + end + + def write(sync_config, records, action = "destination_insert") + connection_config = sync_config.destination.connection_specification.with_indifferent_access + table_name = sync_config.stream.name + primary_key = sync_config.model.primary_key + conn = create_connection(connection_config) + + write_success = 0 + write_failure = 0 + log_message_array = [] + + records.each do |record| + query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key) + query = query.gsub(";", "") + logger.debug("ORACLE:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}") + begin + response = conn.exec(query) + conn.exec("COMMIT") + write_success += 1 + log_message_array << log_request_response("info", query, response) + rescue StandardError => e + handle_exception(e, { + context: "ORACLE:RECORD:WRITE:EXCEPTION", + type: "error", + sync_id: sync_config.sync_id, + sync_run_id: sync_config.sync_run_id + }) + write_failure += 1 + log_message_array << log_request_response("error", query, e.message) + end + end + tracking_message(write_success, write_failure, log_message_array) + rescue StandardError => e + handle_exception(e, { + context: "ORACLE:RECORD:WRITE:EXCEPTION", + type: "error", + sync_id: sync_config.sync_id, + sync_run_id: sync_config.sync_run_id + }) + end + + private + + def create_connection(connection_config) + OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}") + end + + def create_streams(records) + group_by_table(records).map do |_, r| + Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns])) + end + end + + def group_by_table(records) + result = {} + records.each_with_index do |entry, index| + table_name = entry[0] + column_data = { + column_name: entry[1], + data_type: entry[2], + is_nullable: entry[3] == "Y" + } + result[index] ||= {} + result[index][:tablename] = table_name + result[index][:columns] = [column_data] + end + result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries| + { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } } + end + end + end + end +end diff --git a/integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json new file mode 100644 index 00000000..5662698c --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json @@ -0,0 +1,15 @@ +{ + "data": { + "name": "Oracle", + "title": "Oracle", + "connector_type": "destination", + "category": "Database", + "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle", + "github_issue_label": "destination-oracle", + "icon": "icon.svg", + "license": "MIT", + "release_stage": "alpha", + "support_level": "community", + "tags": ["language:ruby", "multiwoven"] + } +} diff --git a/integrations/lib/multiwoven/integrations/destination/oracle_db/config/spec.json b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/spec.json new file mode 100644 index 00000000..2eed10f6 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/spec.json @@ -0,0 +1,47 @@ +{ + "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle", + "stream_type": "dynamic", + "connector_query_type": "raw_sql", + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Oracle", + "type": "object", + "required": ["host", "port", "sid", "username", "password"], + "properties": { + "host": { + "description": "The Oracle host.", + "examples": ["localhost"], + "type": "string", + "title": "Host", + "order": 0 + }, + "port": { + "description": "The Oracle port number.", + "examples": ["1521"], + "type": "string", + "title": "Port", + "order": 1 + }, + "sid": { + "description": "The name of your service in Oracle.", + "examples": ["ORCLPDB1"], + "type": "string", + "title": "SID", + "order": 2 + }, + "username": { + "description": "The username used to authenticate and connect.", + "type": "string", + "title": "Username", + "order": 3 + }, + "password": { + "description": "The password corresponding to the username used for authentication.", + "type": "string", + "multiwoven_secret": true, + "title": "Password", + "order": 4 + } + } + } +} \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/destination/oracle_db/icon.svg b/integrations/lib/multiwoven/integrations/destination/oracle_db/icon.svg new file mode 100644 index 00000000..3f4e051f --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/oracle_db/icon.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 76ece7b2..03ec5a0a 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.5.2" + VERSION = "0.6.0" ENABLED_SOURCES = %w[ Snowflake @@ -34,6 +34,7 @@ module Integrations Iterable MariaDB DatabricksLakehouse + Oracle ].freeze end end diff --git a/integrations/multiwoven-integrations.gemspec b/integrations/multiwoven-integrations.gemspec index 34aec384..498a2d19 100644 --- a/integrations/multiwoven-integrations.gemspec +++ b/integrations/multiwoven-integrations.gemspec @@ -53,6 +53,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "rake" spec.add_runtime_dependency "restforce" spec.add_runtime_dependency "ruby-limiter" + spec.add_runtime_dependency "ruby-oci8" spec.add_runtime_dependency "ruby-odbc" spec.add_runtime_dependency "rubyzip" spec.add_runtime_dependency "sequel" diff --git a/integrations/spec/multiwoven/integrations/destination/oracle_db/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/oracle_db/client_spec.rb new file mode 100644 index 00000000..7774e2e5 --- /dev/null +++ b/integrations/spec/multiwoven/integrations/destination/oracle_db/client_spec.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +RSpec.describe Multiwoven::Integrations::Destination::Oracle::Client do + include WebMock::API + + before(:each) do + WebMock.disable_net_connect!(allow_localhost: true) + end + + let(:client) { described_class.new } + let(:connection_config) do + { + host: "localhost", + port: "1521", + servicename: "PDB1", + username: "oracle_user", + password: "oracle_password" + } + end + let(:sync_config_json) do + { + source: { + name: "Sample Source Connector", + type: "source", + connection_specification: { + private_api_key: "test_api_key" + } + }, + destination: { + name: "Databricks", + type: "destination", + connection_specification: connection_config + }, + model: { + name: "ExampleModel", + query: "SELECT col1, col2, col3 FROM test_table_1", + query_type: "raw_sql", + primary_key: "col1" + }, + sync_mode: "incremental", + destination_sync_mode: "insert", + stream: { + name: "table", + action: "create", + json_schema: {}, + supported_sync_modes: %w[incremental] + } + } + end + + let(:oracle_connection) { instance_double(OCI8) } + let(:cursor) { instance_double("OCI8::Cursor") } + + describe "#check_connection" do + context "when the connection is successful" do + it "returns a succeeded connection status" do + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(true) + message = client.check_connection(sync_config_json[:destination][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("succeeded") + expect(result.message).to be_nil + end + end + + context "when the connection fails" do + it "returns a failed connection status with an error message" do + allow(client).to receive(:create_connection).and_raise(StandardError, "Connection failed") + message = client.check_connection(sync_config_json[:destination][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("failed") + expect(result.message).to include("Connection failed") + end + end + end + + describe "#discover" do + it "discovers schema successfully" do + response = %w[test_table col1 NUMBER Y] + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(cursor) + allow(cursor).to receive(:fetch).and_return(response, nil) + message = client.discover(sync_config_json[:destination][:connection_specification]) + expect(message.catalog).to be_an(Multiwoven::Integrations::Protocol::Catalog) + first_stream = message.catalog.streams.first + expect(first_stream).to be_a(Multiwoven::Integrations::Protocol::Stream) + expect(first_stream.name).to eq("test_table") + expect(first_stream.json_schema).to be_an(Hash) + expect(first_stream.json_schema["type"]).to eq("object") + expect(first_stream.json_schema["properties"]).to eq({ "col1" => { "type" => "string" } }) + end + end + + describe "#write" do + context "when the write operation is successful" do + it "increments the success count" do + sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json( + sync_config_json.to_json + ) + record = [ + { "col1" => 1, "col2" => "first", "col3" => 1.1 } + ] + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(1) + allow(cursor).to receive(:fetch).and_return(1, nil) + response = client.write(sync_config, record) + expect(response.tracking.success).to eq(record.size) + expect(response.tracking.failed).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") + end + end + + context "when the write operation fails" do + it "increments the failure count" do + sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json( + sync_config_json.to_json + ) + record = [ + { "col1" => 1, "col2" => "first", "col3" => 1.1 } + ] + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_raise(StandardError, "Test error") + response = client.write(sync_config, record) + expect(response.tracking.failed).to eq(record.size) + expect(response.tracking.success).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("error") + expect(log_message.message).to include("request") + expect(log_message.message).to include("{\"request\":\"INSERT INTO table (col1, col2, col3) VALUES ('1', 'first', '1.1')\",\"response\":\"Test error\",\"level\":\"error\"}") + end + end + end + + describe "#meta_data" do + # change this to rollout validation for all connector rolling out + it "client class_name and meta name is same" do + meta_name = client.class.to_s.split("::")[-2] + expect(client.send(:meta_data)[:data][:name]).to eq(meta_name) + end + end +end From 108cc8208562e2b922933541a4878a186f8cd81e Mon Sep 17 00:00:00 2001 From: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> Date: Fri, 2 Aug 2024 05:25:51 -0400 Subject: [PATCH 07/74] chore(CE): update gem 0.6.0 (#278) --- .github/workflows/server-ci.yml | 8 ++++++ server/Dockerfile | 26 +++++++++++--------- server/Dockerfile.dev | 24 ++++++++++-------- server/Gemfile | 2 +- server/Gemfile.lock | 42 ++++++++++++++++++-------------- server/getoracleinstantclient.sh | 29 ++++++++++++++++++++++ 6 files changed, 90 insertions(+), 41 deletions(-) create mode 100644 server/getoracleinstantclient.sh diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 133e7399..176bacc6 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -50,6 +50,14 @@ jobs: sudo mv libduckdb/libduckdb.so /usr/local/lib sudo ldconfig /usr/local/lib + - name: Download and Install Oracle Instant Client + run: | + sudo apt-get install -y libaio1 alien + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-basic-19.6.0.0.0-1.x86_64.rpm + wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/x86_64/getPackage/oracle-instantclient19.6-devel-19.6.0.0.0-1.x86_64.rpm + sudo alien -i --scripts oracle-instantclient*.rpm + rm -f oracle-instantclient*.rpm + - name: Bundle Install run: bundle install working-directory: ./server diff --git a/server/Dockerfile b/server/Dockerfile index 3f9b9c4b..1615d389 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -20,6 +20,10 @@ FROM base as build RUN apt-get update -qq && \ apt-get install --no-install-recommends -y build-essential autoconf automake libtool git libpq-dev libvips pkg-config m4 perl libltdl-dev curl git wget unzip default-libmysqlclient-dev +# Copy and run the Oracle Instant Client installation script +COPY getoracleinstantclient.sh . +RUN chmod +x getoracleinstantclient.sh && ./getoracleinstantclient.sh + COPY getduckdb.sh . COPY gethttpfsextension.sh . @@ -72,17 +76,16 @@ ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} ARG TARGETARCH=amd64 RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "x86_64" ]; then \ - wget https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake-odbc-3.2.0.x86_64.deb -O snowflake-odbc.deb && \ - dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ + wget https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake-odbc-3.2.0.x86_64.deb -O snowflake-odbc.deb && \ + dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ elif [ "$TARGETARCH" = "arm64" ] || [ "$TARGETARCH" = "aarch64" ]; then \ - wget https://sfc-repo.snowflakecomputing.com/odbc/linuxaarch64/3.2.0/snowflake-odbc-3.2.0.aarch64.deb -O snowflake-odbc.deb && \ - dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ + wget https://sfc-repo.snowflakecomputing.com/odbc/linuxaarch64/3.2.0/snowflake-odbc-3.2.0.aarch64.deb -O snowflake-odbc.deb && \ + dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ else \ - echo "Unsupported architecture: $TARGETARCH" >&2; \ - exit 1; \ + echo "Unsupported architecture: $TARGETARCH" >&2; \ + exit 1; \ fi - RUN apt-get update -qq && \ apt-get install -y unzip @@ -90,14 +93,13 @@ RUN apt-get update -qq && \ apt-get install -y libsasl2-modules-gssapi-mit RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "x86_64" ]; then \ - wget --quiet https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.7.7/SimbaSparkODBC-2.7.7.1016-Debian-64bit.zip -O /tmp/databricks_odbc.zip && \ - unzip /tmp/databricks_odbc.zip -d /tmp && \ - dpkg -i /tmp/simbaspark_*.deb && \ - rm -rf /tmp/*; \ + wget --quiet https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.7.7/SimbaSparkODBC-2.7.7.1016-Debian-64bit.zip -O /tmp/databricks_odbc.zip && \ + unzip /tmp/databricks_odbc.zip -d /tmp && \ + dpkg -i /tmp/simbaspark_*.deb && \ + rm -rf /tmp/*; \ fi # ARM64 version of the Simba Spark ODBC driver is not currently available - # Change back to the root directory before copying the Rails app # Rails app lives here WORKDIR /rails diff --git a/server/Dockerfile.dev b/server/Dockerfile.dev index ae2e9ac1..c3d7fe36 100644 --- a/server/Dockerfile.dev +++ b/server/Dockerfile.dev @@ -9,6 +9,10 @@ FROM registry.docker.com/library/ruby:$RUBY_VERSION-slim as base RUN apt-get update -qq && \ apt-get install --no-install-recommends -y build-essential autoconf automake libtool git libpq-dev libvips pkg-config m4 perl libltdl-dev curl git wget unzip default-libmysqlclient-dev +# Copy and run the Oracle Instant Client installation script +COPY getoracleinstantclient.sh . +RUN chmod +x getoracleinstantclient.sh && ./getoracleinstantclient.sh + COPY getduckdb.sh . COPY gethttpfsextension.sh . @@ -78,14 +82,14 @@ ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} ARG TARGETARCH=amd64 RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "x86_64" ]; then \ - wget https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake-odbc-3.2.0.x86_64.deb -O snowflake-odbc.deb && \ - dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ + wget https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake-odbc-3.2.0.x86_64.deb -O snowflake-odbc.deb && \ + dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ elif [ "$TARGETARCH" = "arm64" ] || [ "$TARGETARCH" = "aarch64" ]; then \ - wget https://sfc-repo.snowflakecomputing.com/odbc/linuxaarch64/3.2.0/snowflake-odbc-3.2.0.aarch64.deb -O snowflake-odbc.deb && \ - dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ + wget https://sfc-repo.snowflakecomputing.com/odbc/linuxaarch64/3.2.0/snowflake-odbc-3.2.0.aarch64.deb -O snowflake-odbc.deb && \ + dpkg -i snowflake-odbc.deb || apt-get -y -f install; \ else \ - echo "Unsupported architecture: $TARGETARCH" >&2; \ - exit 1; \ + echo "Unsupported architecture: $TARGETARCH" >&2; \ + exit 1; \ fi RUN apt-get update -qq && \ @@ -95,10 +99,10 @@ RUN apt-get update -qq && \ apt-get install -y libsasl2-modules-gssapi-mit RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "x86_64" ]; then \ - wget --quiet https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.7.7/SimbaSparkODBC-2.7.7.1016-Debian-64bit.zip -O /tmp/databricks_odbc.zip && \ - unzip /tmp/databricks_odbc.zip -d /tmp && \ - dpkg -i /tmp/simbaspark_*.deb && \ - rm -rf /tmp/*; \ + wget --quiet https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.7.7/SimbaSparkODBC-2.7.7.1016-Debian-64bit.zip -O /tmp/databricks_odbc.zip && \ + unzip /tmp/databricks_odbc.zip -d /tmp && \ + dpkg -i /tmp/simbaspark_*.deb && \ + rm -rf /tmp/*; \ fi #ARM64 version of the Simba Spark ODBC driver is not currently available, diff --git a/server/Gemfile b/server/Gemfile index 35703cb6..6f715f6c 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -13,7 +13,7 @@ gem "interactor", "~> 3.0" gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git" -gem "multiwoven-integrations", "~> 0.5.2" +gem "multiwoven-integrations", "~> 0.6.0" gem "temporal-ruby", github: "coinbase/temporal-ruby" diff --git a/server/Gemfile.lock b/server/Gemfile.lock index f95ef77b..4cf79a94 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -108,7 +108,7 @@ GEM appsignal (3.7.5) rack ast (2.4.2) - async (2.14.1) + async (2.14.2) console (~> 1.25, >= 1.25.2) fiber-annotation io-event (~> 1.6, >= 1.6.5) @@ -123,8 +123,9 @@ GEM traces (>= 0.10) async-pool (0.7.0) async (>= 1.25) - async-websocket (0.27.0) + async-websocket (0.28.0) async-http (~> 0.54) + protocol-http (>= 0.28.1) protocol-rack (~> 0.5) protocol-websocket (~> 0.15) aws-eventstream (1.3.0) @@ -1661,7 +1662,7 @@ GEM activesupport concurrent-ruby (1.2.3) connection_pool (2.4.1) - console (1.25.2) + console (1.27.0) fiber-annotation fiber-local (~> 1.1) json @@ -1749,7 +1750,7 @@ GEM railties (>= 5.0.0) faker (3.2.3) i18n (>= 1.8.11, < 2) - faraday (2.10.0) + faraday (2.10.1) faraday-net_http (>= 2.0, < 3.2) logger faraday-follow_redirects (0.3.0) @@ -1759,9 +1760,12 @@ GEM hashie faraday-multipart (1.0.4) multipart-post (~> 2) - faraday-net_http (3.1.0) + faraday-net_http (3.1.1) net-http + ffi (1.17.0-aarch64-linux-gnu) ffi (1.17.0-arm64-darwin) + ffi (1.17.0-x86_64-darwin) + ffi (1.17.0-x86_64-linux-gnu) fiber-annotation (0.2.0) fiber-local (1.1.0) fiber-storage @@ -1774,20 +1778,20 @@ GEM addressable (~> 2.8) process_executer (~> 1.1) rchardet (~> 1.8) - gli (2.21.3) + gli (2.21.5) globalid (1.2.1) activesupport (>= 6.1) - google-apis-bigquery_v2 (0.72.0) + google-apis-bigquery_v2 (0.73.0) google-apis-core (>= 0.15.0, < 2.a) - google-apis-core (0.15.0) + google-apis-core (0.15.1) addressable (~> 2.5, >= 2.5.1) googleauth (~> 1.9) - httpclient (>= 2.8.1, < 3.a) + httpclient (>= 2.8.3, < 3.a) mini_mime (~> 1.0) + mutex_m representable (~> 3.0) retriable (>= 2.0, < 4.a) - rexml - google-apis-sheets_v4 (0.32.0) + google-apis-sheets_v4 (0.33.0) google-apis-core (>= 0.15.0, < 2.a) google-cloud-bigquery (1.50.0) bigdecimal (~> 3.0) @@ -1894,7 +1898,7 @@ GEM msgpack (1.7.2) multi_json (1.15.0) multipart-post (2.4.1) - multiwoven-integrations (0.5.2) + multiwoven-integrations (0.6.0) activesupport async-websocket aws-sdk-athena @@ -1915,6 +1919,7 @@ GEM rake restforce ruby-limiter + ruby-oci8 ruby-odbc rubyzip sequel @@ -1967,8 +1972,8 @@ GEM net-smtp premailer (~> 1.7, >= 1.7.9) process_executer (1.1.0) - protocol-hpack (1.4.3) - protocol-http (0.27.0) + protocol-hpack (1.5.0) + protocol-http (0.28.1) protocol-http1 (0.19.1) protocol-http (~> 0.22) protocol-http2 (0.18.0) @@ -2091,10 +2096,11 @@ GEM rubocop (>= 1.33.0, < 2.0) rubocop-ast (>= 1.30.0, < 2.0) ruby-limiter (2.3.0) + ruby-oci8 (2.2.12) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) rubyzip (2.3.2) - sequel (5.82.0) + sequel (5.83.0) bigdecimal shoulda-matchers (5.3.0) activesupport (>= 5.2.0) @@ -2117,9 +2123,9 @@ GEM faraday-multipart gli hashie - sorbet-runtime (0.5.11481) + sorbet-runtime (0.5.11504) stringio (3.1.0) - stripe (12.2.0) + stripe (12.4.0) strong_migrations (1.8.0) activerecord (>= 5.2) thor (1.3.0) @@ -2191,7 +2197,7 @@ DEPENDENCIES jwt kaminari liquid - multiwoven-integrations (~> 0.5.2) + multiwoven-integrations (~> 0.6.0) mysql2 newrelic_rpm parallel diff --git a/server/getoracleinstantclient.sh b/server/getoracleinstantclient.sh new file mode 100644 index 00000000..a7b693a6 --- /dev/null +++ b/server/getoracleinstantclient.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +MACHINE=`uname -m` + +case "$MACHINE" in + "x86_64" ) ARC=x86_64 ;; + "aarch64" ) ARC=aarch64 ;; + * ) echo "Unsupported architecture: $MACHINE" >&2; exit 1 ;; +esac + +# Download basic package +if ! wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/$ARC/getPackage/oracle-instantclient19.10-basic-19.10.0.0.0-1.$ARC.rpm; then + echo "Failed to download oracle-instantclient19.10-basic.rpm" >&2 + exit 1 +fi + +# Download devel package (repeat for devel package) +if ! wget http://yum.oracle.com/repo/OracleLinux/OL7/oracle/instantclient/$ARC/getPackage/oracle-instantclient19.10-devel-19.10.0.0.0-1.$ARC.rpm; then + echo "Failed to download oracle-instantclient19.10-devel.rpm" >&2 + exit 1 +fi + +# Install packages +apt-get update -qq && \ + apt-get install -y libaio1 alien && \ + alien -i --scripts oracle-instantclient19.10-basic-19.10.0.0.0-1.$ARC.rpm && \ + alien -i --scripts oracle-instantclient19.10-devel-19.10.0.0.0-1.$ARC.rpm && \ + rm -f oracle-instantclient19.10-basic-19.10.0.0.0-1.$ARC.rpm && \ + rm -f oracle-instantclient19.10-devel-19.10.0.0.0-1.$ARC.rpm \ No newline at end of file From a74d8411fa66862babc15dbef3f399eb3fe01b14 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:56:37 +0530 Subject: [PATCH 08/74] feat(CE): add oracle db source connector (#274) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- integrations/Gemfile.lock | 2 +- integrations/lib/multiwoven/integrations.rb | 1 + .../lib/multiwoven/integrations/rollout.rb | 3 +- .../integrations/source/oracle_db/client.rb | 127 +++++++++++++++ .../source/oracle_db/config/meta.json | 15 ++ .../source/oracle_db/config/spec.json | 47 ++++++ .../integrations/source/oracle_db/icon.svg | 4 + .../source/oracle_db/client_spec.rb | 153 ++++++++++++++++++ 8 files changed, 350 insertions(+), 2 deletions(-) create mode 100644 integrations/lib/multiwoven/integrations/source/oracle_db/client.rb create mode 100644 integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json create mode 100644 integrations/lib/multiwoven/integrations/source/oracle_db/config/spec.json create mode 100644 integrations/lib/multiwoven/integrations/source/oracle_db/icon.svg create mode 100644 integrations/spec/multiwoven/integrations/source/oracle_db/client_spec.rb diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 5843cd3a..23ac9eee 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.6.0) + multiwoven-integrations (0.7.0) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb index dd8636cb..4f80fab4 100644 --- a/integrations/lib/multiwoven/integrations.rb +++ b/integrations/lib/multiwoven/integrations.rb @@ -61,6 +61,7 @@ require_relative "integrations/source/clickhouse/client" require_relative "integrations/source/amazon_s3/client" require_relative "integrations/source/maria_db/client" +require_relative "integrations/source/oracle_db/client" # Destination require_relative "integrations/destination/klaviyo/client" diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 03ec5a0a..45d2b8b2 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.6.0" + VERSION = "0.7.0" ENABLED_SOURCES = %w[ Snowflake @@ -15,6 +15,7 @@ module Integrations Clickhouse AmazonS3 MariaDB + Oracle ].freeze ENABLED_DESTINATIONS = %w[ diff --git a/integrations/lib/multiwoven/integrations/source/oracle_db/client.rb b/integrations/lib/multiwoven/integrations/source/oracle_db/client.rb new file mode 100644 index 00000000..f5d787bc --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/oracle_db/client.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +module Multiwoven::Integrations::Source + module Oracle + include Multiwoven::Integrations::Core + class Client < SourceConnector + def check_connection(connection_config) + connection_config = connection_config.with_indifferent_access + create_connection(connection_config) + ConnectionStatus.new( + status: ConnectionStatusType["succeeded"] + ).to_multiwoven_message + rescue StandardError => e + ConnectionStatus.new( + status: ConnectionStatusType["failed"], message: e.message + ).to_multiwoven_message + end + + def discover(connection_config) + records = [] + connection_config = connection_config.with_indifferent_access + query = "SELECT table_name, column_name, data_type, nullable + FROM all_tab_columns + WHERE owner = '#{connection_config[:username].upcase}' + ORDER BY table_name, column_id" + conn = create_connection(connection_config) + cursor = conn.exec(query) + while (row = cursor.fetch) + records << row + end + catalog = Catalog.new(streams: create_streams(records)) + catalog.to_multiwoven_message + rescue StandardError => e + handle_exception( + "ORACLE:DISCOVER:EXCEPTION", + "error", + e + ) + end + + def read(sync_config) + connection_config = sync_config.source.connection_specification.with_indifferent_access + query = sync_config.model.query + db = create_connection(connection_config) + query(db, query) + rescue StandardError => e + handle_exception(e, { + context: "ORACLE:READ:EXCEPTION", + type: "error", + sync_id: sync_config.sync_id, + sync_run_id: sync_config.sync_run_id + }) + end + + private + + def create_connection(connection_config) + OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}") + end + + def create_streams(records) + group_by_table(records).map do |_, r| + Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns])) + end + end + + def query(connection, query) + records = [] + query = reformat_query(query) + cursor = connection.exec(query) + columns = cursor.get_col_names + while (row = cursor.fetch) + data_hash = columns.zip(row).to_h + records << RecordMessage.new(data: data_hash, emitted_at: Time.now.to_i).to_multiwoven_message + end + records + end + + def group_by_table(records) + result = {} + records.each_with_index do |entry, index| + table_name = entry[0] + column_data = { + column_name: entry[1], + data_type: entry[2], + is_nullable: entry[3] == "Y" + } + result[index] ||= {} + result[index][:tablename] = table_name + result[index][:columns] = [column_data] + end + result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries| + { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } } + end + end + + def reformat_query(sql_query) + offset = nil + limit = nil + + sql_query = sql_query.gsub(";", "") + + if sql_query.match?(/LIMIT (\d+)/i) + limit = sql_query.match(/LIMIT (\d+)/i)[1].to_i + sql_query.sub!(/LIMIT \d+/i, "") + end + + if sql_query.match?(/OFFSET (\d+)/i) + offset = sql_query.match(/OFFSET (\d+)/i)[1].to_i + sql_query.sub!(/OFFSET \d+/i, "") + end + + sql_query.strip! + + if offset && limit + "#{sql_query} OFFSET #{offset} ROWS FETCH NEXT #{limit} ROWS ONLY" + elsif offset + "#{sql_query} OFFSET #{offset} ROWS" + elsif limit + "#{sql_query} FETCH NEXT #{limit} ROWS ONLY" + else + sql_query + end + end + end + end +end diff --git a/integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json b/integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json new file mode 100644 index 00000000..833d36f0 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json @@ -0,0 +1,15 @@ +{ + "data": { + "name": "Oracle", + "title": "Oracle", + "connector_type": "source", + "category": "Database", + "documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle", + "github_issue_label": "source-oracle", + "icon": "icon.svg", + "license": "MIT", + "release_stage": "alpha", + "support_level": "community", + "tags": ["language:ruby", "multiwoven"] + } +} diff --git a/integrations/lib/multiwoven/integrations/source/oracle_db/config/spec.json b/integrations/lib/multiwoven/integrations/source/oracle_db/config/spec.json new file mode 100644 index 00000000..703791b7 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/oracle_db/config/spec.json @@ -0,0 +1,47 @@ +{ + "documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle", + "stream_type": "dynamic", + "connector_query_type": "raw_sql", + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Oracle", + "type": "object", + "required": ["host", "port", "sid", "username", "password"], + "properties": { + "host": { + "description": "The Oracle host.", + "examples": ["localhost"], + "type": "string", + "title": "Host", + "order": 0 + }, + "port": { + "description": "The Oracle port number.", + "examples": ["1521"], + "type": "string", + "title": "Port", + "order": 1 + }, + "sid": { + "description": "The name of your service in Oracle.", + "examples": ["ORCLPDB1"], + "type": "string", + "title": "SID", + "order": 2 + }, + "username": { + "description": "The username used to authenticate and connect.", + "type": "string", + "title": "Username", + "order": 3 + }, + "password": { + "description": "The password corresponding to the username used for authentication.", + "type": "string", + "multiwoven_secret": true, + "title": "Password", + "order": 4 + } + } + } +} \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/source/oracle_db/icon.svg b/integrations/lib/multiwoven/integrations/source/oracle_db/icon.svg new file mode 100644 index 00000000..3f4e051f --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/oracle_db/icon.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/integrations/spec/multiwoven/integrations/source/oracle_db/client_spec.rb b/integrations/spec/multiwoven/integrations/source/oracle_db/client_spec.rb new file mode 100644 index 00000000..51b9af14 --- /dev/null +++ b/integrations/spec/multiwoven/integrations/source/oracle_db/client_spec.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +RSpec.describe Multiwoven::Integrations::Source::Oracle::Client do + let(:client) { Multiwoven::Integrations::Source::Oracle::Client.new } + let(:sync_config) do + { + "source": { + "name": "OracleConnector", + "type": "source", + "connection_specification": { + "host": "localhost", + "port": "1521", + "servicename": "PDB1", + "username": "oracle_user", + "password": "oracle_password" + } + }, + "destination": { + "name": "DestinationConnectorName", + "type": "destination", + "connection_specification": { + "example_destination_key": "example_destination_value" + } + }, + "model": { + "name": "OracleDB Model", + "query": "SELECT col1, col2, col3 FROM test_table", + "query_type": "raw_sql", + "primary_key": "id" + }, + "stream": { + "name": "example_stream", "action": "create", + "json_schema": { "field1": "type1" }, + "supported_sync_modes": %w[full_refresh incremental], + "source_defined_cursor": true, + "default_cursor_field": ["field1"], + "source_defined_primary_key": [["field1"], ["field2"]], + "namespace": "exampleNamespace", + "url": "https://api.example.com/data", + "method": "GET" + }, + "sync_mode": "full_refresh", + "cursor_field": "timestamp", + "destination_sync_mode": "upsert", + "sync_id": "1" + } + end + + let(:oracle_connection) { instance_double(OCI8) } + let(:cursor) { instance_double("OCI8::Cursor") } + + describe "#check_connection" do + context "when the connection is successful" do + it "returns a succeeded connection status" do + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(true) + message = client.check_connection(sync_config[:source][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("succeeded") + expect(result.message).to be_nil + end + end + + context "when the connection fails" do + it "returns a failed connection status with an error message" do + allow(client).to receive(:create_connection).and_raise(StandardError, "Connection failed") + message = client.check_connection(sync_config[:source][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("failed") + expect(result.message).to include("Connection failed") + end + end + end + + # read and #discover tests for MariaDB + describe "#read" do + it "reads records successfully" do + s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json) + columns = %w[col1 col2 col3] + response = %w[1 First Row Text First Row Additional Text] + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(cursor) + allow(cursor).to receive(:get_col_names).and_return(columns, nil) + allow(cursor).to receive(:fetch).and_return(response, nil) + records = client.read(s_config) + expect(records).to be_an(Array) + expect(records).not_to be_empty + expect(records.first).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage) + end + + it "reads records successfully with limit" do + s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json) + s_config.limit = 100 + s_config.offset = 1 + columns = %w[col1 col2 col3] + response = %w[1 First Row Text First Row Additional Text] + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(cursor) + allow(cursor).to receive(:get_col_names).and_return(columns, nil) + allow(cursor).to receive(:fetch).and_return(response, nil) + records = client.read(s_config) + expect(records).to be_an(Array) + expect(records).not_to be_empty + expect(records.first).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage) + end + + it "read records failure" do + s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json) + s_config.sync_run_id = "2" + allow(client).to receive(:create_connection).and_raise(StandardError, "test error") + expect(client).to receive(:handle_exception).with( + an_instance_of(StandardError), { + context: "ORACLE:READ:EXCEPTION", + type: "error", + sync_id: "1", + sync_run_id: "2" + } + ) + client.read(s_config) + end + end + + describe "#discover" do + it "discovers schema successfully" do + response = %w[test_table col1 NUMBER Y] + allow(OCI8).to receive(:new).and_return(oracle_connection) + allow(oracle_connection).to receive(:exec).and_return(cursor) + allow(cursor).to receive(:fetch).and_return(response, nil) + message = client.discover(sync_config[:source][:connection_specification]) + expect(message.catalog).to be_an(Multiwoven::Integrations::Protocol::Catalog) + first_stream = message.catalog.streams.first + expect(first_stream).to be_a(Multiwoven::Integrations::Protocol::Stream) + expect(first_stream.name).to eq("test_table") + expect(first_stream.json_schema).to be_an(Hash) + expect(first_stream.json_schema["type"]).to eq("object") + expect(first_stream.json_schema["properties"]).to eq({ "col1" => { "type" => "string" } }) + end + end + + describe "#meta_data" do + # change this to rollout validation for all connector rolling out + it "client class_name and meta name is same" do + meta_name = client.class.to_s.split("::")[-2] + expect(client.send(:meta_data)[:data][:name]).to eq(meta_name) + end + end + + describe "method definition" do + it "defines a private #query method" do + expect(described_class.private_instance_methods).to include(:query) + end + end +end From a03b1138223ab39d6135784522f599e4aee5c6d1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:14:18 +0530 Subject: [PATCH 09/74] chore(CE): change name to databricks datawarehouse (#273) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- integrations/Gemfile.lock | 2 +- .../destination/databricks_lakehouse/config/meta.json | 4 ++-- .../destination/databricks_lakehouse/config/spec.json | 2 +- integrations/lib/multiwoven/integrations/rollout.rb | 2 +- .../integrations/source/databricks/config/spec.json | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 23ac9eee..846de3fe 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.7.0) + multiwoven-integrations (0.7.1) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json b/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json index 6849247c..34e949ea 100644 --- a/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json +++ b/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json @@ -1,9 +1,9 @@ { "data": { "name": "DatabricksLakehouse", - "title": "Databricks Lakehouse", + "title": "Databricks Datawarehouse", "connector_type": "destination", - "category": "Marketing Automation", + "category": "Database", "documentation_url": "https://docs.multiwoven.com/destinations/databricks_lakehouse", "github_issue_label": "destination-databricks-lakehouse", "icon": "icon.svg", diff --git a/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/spec.json b/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/spec.json index bbd4f497..5693add1 100644 --- a/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/spec.json +++ b/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/spec.json @@ -3,7 +3,7 @@ "stream_type": "static", "connection_specification": { "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Databricks Lakehouse", + "title": "Databricks Datawarehouse", "type": "object", "required": ["host", "api_token", "warehouse_id", "catalog", "schema"], "properties": { diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 45d2b8b2..dd0b7129 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.7.0" + VERSION = "0.7.1" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/lib/multiwoven/integrations/source/databricks/config/spec.json b/integrations/lib/multiwoven/integrations/source/databricks/config/spec.json index e446ffe3..9d5ff33d 100644 --- a/integrations/lib/multiwoven/integrations/source/databricks/config/spec.json +++ b/integrations/lib/multiwoven/integrations/source/databricks/config/spec.json @@ -4,7 +4,7 @@ "connector_query_type": "raw_sql", "connection_specification": { "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Databricks", + "title": "Databricks Datawarehouse", "type": "object", "required": ["host", "port", "http_path", "catalog", "schema"], "properties": { From 63df2a50d1d4e56c3c94e1cd03494958704b090b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:31:39 +0530 Subject: [PATCH 10/74] chore(CE): update integration gem 0.7.1 (#272) --- server/Gemfile | 2 +- server/Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/Gemfile b/server/Gemfile index 6f715f6c..5dc78e2c 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -13,7 +13,7 @@ gem "interactor", "~> 3.0" gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git" -gem "multiwoven-integrations", "~> 0.6.0" +gem "multiwoven-integrations", "~> 0.7.1" gem "temporal-ruby", github: "coinbase/temporal-ruby" diff --git a/server/Gemfile.lock b/server/Gemfile.lock index 4cf79a94..fd46ce10 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -1898,7 +1898,7 @@ GEM msgpack (1.7.2) multi_json (1.15.0) multipart-post (2.4.1) - multiwoven-integrations (0.6.0) + multiwoven-integrations (0.7.1) activesupport async-websocket aws-sdk-athena @@ -2197,7 +2197,7 @@ DEPENDENCIES jwt kaminari liquid - multiwoven-integrations (~> 0.6.0) + multiwoven-integrations (~> 0.7.1) mysql2 newrelic_rpm parallel From f4add1b524d2ad0c537a402e4a962dc7bc2bfd9c Mon Sep 17 00:00:00 2001 From: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> Date: Fri, 2 Aug 2024 16:22:47 -0400 Subject: [PATCH 11/74] chore(CE): update README (#358) --- integrations/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integrations/README.md b/integrations/README.md index 19c95be9..b11c918b 100644 --- a/integrations/README.md +++ b/integrations/README.md @@ -59,6 +59,10 @@ Before you begin the installation, ensure you have the following dependencies in - Command: `brew install openssl@3` - Description: Essential for secure communication. +- **Oracle Instant Client** + - Download Link: https://www.oracle.com/database/technologies/instant-client/downloads.html + - Description: Required for database interactions. + ### Installation From d36ddcb8839b5df3c4416986135b3da6c6871760 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 10:05:25 +0530 Subject: [PATCH 12/74] fix(CE): lazy init temporal after puma worker boot (#239) --- server/app/temporal/cli/worker | 2 + server/config/initializers/logging.rb | 5 ++- server/config/initializers/temporal.rb | 57 +++++++++++++++----------- server/config/puma.rb | 4 ++ 4 files changed, 42 insertions(+), 26 deletions(-) diff --git a/server/app/temporal/cli/worker b/server/app/temporal/cli/worker index f219bcc8..e472520e 100755 --- a/server/app/temporal/cli/worker +++ b/server/app/temporal/cli/worker @@ -3,6 +3,8 @@ require_relative "setup" +TemporalService.setup + worker = Temporal::Worker.new worker.register_workflow(Workflows::SyncWorkflow) diff --git a/server/config/initializers/logging.rb b/server/config/initializers/logging.rb index 285d5c13..7b8e19e8 100644 --- a/server/config/initializers/logging.rb +++ b/server/config/initializers/logging.rb @@ -3,4 +3,7 @@ if ENV["APPSIGNAL_PUSH_API_KEY"] appsignal_logger = Appsignal::Logger.new("rails") Rails.logger.broadcast_to(appsignal_logger) -end \ No newline at end of file +end + +Rails.logger.info "GRPC_ENABLE_FORK_SUPPORT: #{ENV['GRPC_ENABLE_FORK_SUPPORT']}" +Rails.logger.info "ALLOWED_HOSTS: #{ENV['ALLOWED_HOST']}" \ No newline at end of file diff --git a/server/config/initializers/temporal.rb b/server/config/initializers/temporal.rb index 88b88a9b..91ac5d0c 100644 --- a/server/config/initializers/temporal.rb +++ b/server/config/initializers/temporal.rb @@ -1,31 +1,38 @@ -require 'temporal' -require 'temporal/metrics_adapters/log' +# frozen_string_literal: tr -metrics_logger = Logger.new(STDOUT, progname: 'metrics') +require Rails.root.join("lib/utils/exception_reporter") -client_key_path = ENV.fetch('TEMPORAL_ROOT_CERT', 'app/temporal/cli/client.key') -client_cert_path = ENV.fetch('TEMPORAL_CLIENT_KEY', 'app/temporal/cli/client.pem') - -if File.exist?(client_key_path) && File.exist?(client_cert_path) - client_key = File.read(client_key_path) - client_cert = File.read(client_cert_path) +Multiwoven::Integrations::Service.new do |config| + config.logger = Rails.logger + config.exception_reporter = Utils::ExceptionReporter end -Temporal.configure do |config| - config.host = ENV.fetch('TEMPORAL_HOST', 'localhost') - config.port = ENV.fetch('TEMPORAL_PORT', 7233).to_i - config.namespace = ENV.fetch('TEMPORAL_NAMESPACE', 'multiwoven-dev') - config.task_queue = ENV.fetch('TEMPORAL_TASK_QUEUE', 'sync-dev') - config.metrics_adapter = Temporal::MetricsAdapters::Log.new(metrics_logger) - if client_key && client_cert - config.credentials = GRPC::Core::ChannelCredentials.new( - nil, - client_key, - client_cert - ) +module TemporalService + def self.setup + require "temporal" + require "temporal/metrics_adapters/log" + metrics_logger = Logger.new($stdout, progname: "metrics") + client_key_path = ENV.fetch("TEMPORAL_ROOT_CERT", "app/temporal/cli/client.key") + client_cert_path = ENV.fetch("TEMPORAL_CLIENT_KEY", "app/temporal/cli/client.pem") + + if File.exist?(client_key_path) && File.exist?(client_cert_path) + client_key = File.read(client_key_path) + client_cert = File.read(client_cert_path) + end + + Temporal.configure do |config| + config.host = ENV.fetch("TEMPORAL_HOST", "localhost") + config.port = ENV.fetch("TEMPORAL_PORT", 7233).to_i + config.namespace = ENV.fetch("TEMPORAL_NAMESPACE", "multiwoven-dev") + config.task_queue = ENV.fetch("TEMPORAL_TASK_QUEUE", "sync-dev") + config.metrics_adapter = Temporal::MetricsAdapters::Log.new(metrics_logger) + if client_key && client_cert + config.credentials = GRPC::Core::ChannelCredentials.new( + nil, + client_key, + client_cert + ) + end + end end end - -Multiwoven::Integrations::Service.new do |config| - config.logger = Rails.logger -end \ No newline at end of file diff --git a/server/config/puma.rb b/server/config/puma.rb index afa809b4..3d38fb16 100644 --- a/server/config/puma.rb +++ b/server/config/puma.rb @@ -15,6 +15,10 @@ if ENV["RAILS_ENV"] == "production" require "concurrent-ruby" worker_count = Integer(ENV.fetch("WEB_CONCURRENCY") { Concurrent.physical_processor_count }) + on_worker_boot do + Rails.logger.info("Initializing temporal client") + TemporalService.setup + end workers worker_count if worker_count > 1 end From b475a2da8c99509dbbcf1418ec867213bef167a3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 10:06:33 +0530 Subject: [PATCH 13/74] fix(CE): logger issue in sync (#279) Co-authored-by: afthab vp --- .../reverse_etl/extractors/full_refresh.rb | 16 +++++------ server/lib/reverse_etl/loaders/standard.rb | 28 +++++++++---------- .../reverse_etl/transformers/user_mapping.rb | 10 +++---- .../lib/reverse_etl/loaders/standard_spec.rb | 12 ++++++++ 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/server/lib/reverse_etl/extractors/full_refresh.rb b/server/lib/reverse_etl/extractors/full_refresh.rb index e80e0ec8..6baaa032 100644 --- a/server/lib/reverse_etl/extractors/full_refresh.rb +++ b/server/lib/reverse_etl/extractors/full_refresh.rb @@ -45,10 +45,10 @@ def process_records(records, sync_run, model) log_mismatch_error(records, sync_records_to_save, result.rows.flatten, sync_run) rescue StandardError => e - Utils::ExceptionReporter.report(e, { - sync_id: sync_run.sync_id, - sync_run_id: sync_run.id - }) + # Utils::ExceptionReporter.report(e, { + # sync_id: sync_run.sync_id, + # sync_run_id: sync_run.id + # }) log_error("#{e.message}. Sync ID: #{sync_run.sync_id}, Sync Run ID: #{sync_run.id}.") end @@ -79,10 +79,10 @@ def build_sync_record(message, sync_run, model) record: record_data } rescue StandardError => e - Utils::ExceptionReporter.report(e, { - sync_id: sync_run.sync_id, - sync_run_id: sync_run.id - }) + # Utils::ExceptionReporter.report(e, { + # sync_id: sync_run.sync_id, + # sync_run_id: sync_run.id + # }) error_message = "#{e.message}. Sync ID: #{sync_run.sync_id}, Sync Run ID: #{sync_run.id}. Record data: #{record_data.to_json}" log_error(error_message) diff --git a/server/lib/reverse_etl/loaders/standard.rb b/server/lib/reverse_etl/loaders/standard.rb index 897cd2b4..e209c38c 100644 --- a/server/lib/reverse_etl/loaders/standard.rb +++ b/server/lib/reverse_etl/loaders/standard.rb @@ -41,11 +41,16 @@ def process_individual_records(sync_run, sync, sync_config, activity) rescue Activities::LoaderActivity::FullRefreshFailed raise rescue StandardError => e - Utils::ExceptionReporter.report(e, { - sync_run_id: sync_run.id, - sync_id: sync.id - }) - Rails.logger(e) + # Utils::ExceptionReporter.report(e, { + # sync_run_id: sync_run.id, + # sync_id: sync.id + # }) + Rails.logger.error({ + error_message: e.message, + sync_run_id: sync_run.id, + sync_id: sync_run.sync_id, + stack_trace: Rails.backtrace_cleaner.clean(e.backtrace) + }.to_s) end heartbeat(activity, sync_run) @@ -72,15 +77,10 @@ def process_batch_records(sync_run, sync, sync_config, activity) end rescue Activities::LoaderActivity::FullRefreshFailed raise - rescue StandardError => e - Utils::ExceptionReporter.report(e, { - sync_run_id: sync_run.id - }) - Rails.logger.error({ - error_message: e.message, - sync_run_id: sync_run.id, - stack_trace: Rails.backtrace_cleaner.clean(e.backtrace) - }.to_s) + rescue StandardError + # Utils::ExceptionReporter.report(e, { + # sync_run_id: sync_run.id + # }) end update_sync_records_status(sync_run, successfull_sync_records, failed_sync_records) heartbeat(activity, sync_run) diff --git a/server/lib/reverse_etl/transformers/user_mapping.rb b/server/lib/reverse_etl/transformers/user_mapping.rb index dd0e016f..1e1ba54e 100644 --- a/server/lib/reverse_etl/transformers/user_mapping.rb +++ b/server/lib/reverse_etl/transformers/user_mapping.rb @@ -18,13 +18,13 @@ def transform(sync, sync_record) @destination_data rescue StandardError => e - Utils::ExceptionReporter.report(e, { - sync_id: sync.id, - sync_record_id: sync_record.id - }) + # Utils::ExceptionReporter.report(e, { + # sync_id: sync.id, + # sync_record_id: sync_record.id + # }) Rails.logger.error({ error_message: e.message, - sync_run_id: sync_run.id, + sync_id: sync.id, stack_trace: Rails.backtrace_cleaner.clean(e.backtrace) }.to_s) end diff --git a/server/spec/lib/reverse_etl/loaders/standard_spec.rb b/server/spec/lib/reverse_etl/loaders/standard_spec.rb index 79741b7e..439aa770 100644 --- a/server/spec/lib/reverse_etl/loaders/standard_spec.rb +++ b/server/spec/lib/reverse_etl/loaders/standard_spec.rb @@ -173,6 +173,18 @@ allow(activity).to receive(:heartbeat).and_return(activity) end + it "calls process_individual_records throw standard error" do + sync_config = sync_individual.to_protocol + sync_config.sync_run_id = sync_run_individual.id.to_s + + allow(sync_individual.destination.connector_client).to receive(:new).and_return(client) + allow(client).to receive(:write).with(sync_config, [transform], + "destination_insert").and_raise(StandardError.new("write error")) + expect(subject).to receive(:heartbeat).once.with(activity, sync_run_individual) + expect(sync_run_individual).to have_state(:queued) + subject.write(sync_run_individual.id, activity) + end + it "calls process_individual_records method" do sync_config = sync_individual.to_protocol sync_config.sync_run_id = sync_run_individual.id.to_s From 54fb0cfa73f3888cb8eebf8575a78e9c1660b176 Mon Sep 17 00:00:00 2001 From: RafaelOAiSquared <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Mon, 5 Aug 2024 23:28:34 -0400 Subject: [PATCH 14/74] Multiwoven release v0.19.0 (#283) Co-authored-by: github-actions --- release-notes.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/release-notes.md b/release-notes.md index 7f523226..6cece38a 100644 --- a/release-notes.md +++ b/release-notes.md @@ -2,22 +2,30 @@ All notable changes to this project will be documented in this file. -## [0.18.0] - 2024-07-26 +## [0.19.0] - 2024-08-05 ### 🚀 Features -- *(CE)* Add description to toast -- *(CE)* Added search filter to table selector method +- *(CE)* Add sync run type column to sync run records table +- *(CE)* Test sync extractor changes (#270) +- *(CE)* Add oracle db destination connector (#277) +- *(CE)* Add oracle db source connector (#274) ### 🐛 Bug Fixes -- *(CE)* Update manual sync delete API call -- *(CE)* Show server error if data is fetched and no data -- *(CE)* Heartbeat timeout actions in extractor (#262) -- *(CE)* Fix gem (#250) +- *(CE)* Lazy init temporal after puma worker boot (#239) +- *(CE)* Logger issue in sync (#279) + +### 🚜 Refactor + +- *(CE)* Wrapped main layout with protector ### ⚙️ Miscellaneous Tasks -- *(CE)* Added sync_run_type column in sync run (#263) +- *(CE)* Verify user mail in signup using devise (#265) +- *(CE)* Update server gem 0.5.2 (#275) +- *(CE)* Update gem 0.6.0 (#278) +- *(CE)* Change name to databricks datawarehouse (#273) +- *(CE)* Update integration gem 0.7.1 (#272) From 2ce482abcfda4cb4630fb966004d7358f44e6ac3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 17:35:32 +0530 Subject: [PATCH 15/74] feat(CE): connector placeholder image --- ui/src/assets/icons/connector-placeholder.svg | 10 +++++++ ui/src/components/EntityItem/EntityItem.tsx | 10 ++++++- .../SelectDestinations/SelectDestinations.tsx | 27 +++---------------- .../SelectDataSourcesForm.tsx | 27 +++---------------- ui/src/views/Models/ViewModel/ViewModel.tsx | 2 +- 5 files changed, 26 insertions(+), 50 deletions(-) create mode 100644 ui/src/assets/icons/connector-placeholder.svg diff --git a/ui/src/assets/icons/connector-placeholder.svg b/ui/src/assets/icons/connector-placeholder.svg new file mode 100644 index 00000000..a6a80897 --- /dev/null +++ b/ui/src/assets/icons/connector-placeholder.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/ui/src/components/EntityItem/EntityItem.tsx b/ui/src/components/EntityItem/EntityItem.tsx index 44e2eb20..e610a054 100644 --- a/ui/src/components/EntityItem/EntityItem.tsx +++ b/ui/src/components/EntityItem/EntityItem.tsx @@ -1,4 +1,5 @@ import { Box, Image, Text } from '@chakra-ui/react'; +import connectorPlaceholderIcon from '@/assets/icons/connector-placeholder.svg'; type EntityItem = { icon: string; @@ -21,7 +22,14 @@ const EntityItem = ({ icon, name }: EntityItem): JSX.Element => { justifyContent='center' alignItems='center' > - + {name} diff --git a/ui/src/views/Connectors/Destinations/DestinationsForm/SelectDestinations/SelectDestinations.tsx b/ui/src/views/Connectors/Destinations/DestinationsForm/SelectDestinations/SelectDestinations.tsx index 2513d9a6..7d77ee78 100644 --- a/ui/src/views/Connectors/Destinations/DestinationsForm/SelectDestinations/SelectDestinations.tsx +++ b/ui/src/views/Connectors/Destinations/DestinationsForm/SelectDestinations/SelectDestinations.tsx @@ -1,12 +1,13 @@ import { getConnectorsDefintions, ConnectorsDefinationApiResponse } from '@/services/connectors'; import { getDestinationCategories } from '@/views/Connectors/helpers'; import { useContext, useState } from 'react'; -import { Box, Grid, Image, Text, Wrap } from '@chakra-ui/react'; +import { Box, Grid, Text, Wrap } from '@chakra-ui/react'; import ContentContainer from '@/components/ContentContainer'; import { ALL_DESTINATIONS_CATEGORY } from '@/views/Connectors/constant'; import { Connector } from '@/views/Connectors/types'; import { SteppedFormContext } from '@/components/SteppedForm/SteppedForm'; import useQueryWrapper from '@/hooks/useQueryWrapper'; +import EntityItem from '@/components/EntityItem'; const SelectDestinations = (): JSX.Element => { const { stepInfo, handleMoveForward } = useContext(SteppedFormContext); @@ -84,29 +85,7 @@ const SelectDestinations = (): JSX.Element => { height='56px' onClick={() => onDestinationSelect(connector)} > - - - - - {connector.title} - + ) : null, )} diff --git a/ui/src/views/Connectors/Sources/SourcesForm/SelectDataSourcesForm/SelectDataSourcesForm.tsx b/ui/src/views/Connectors/Sources/SourcesForm/SelectDataSourcesForm/SelectDataSourcesForm.tsx index 02f11704..3a0b5daa 100644 --- a/ui/src/views/Connectors/Sources/SourcesForm/SelectDataSourcesForm/SelectDataSourcesForm.tsx +++ b/ui/src/views/Connectors/Sources/SourcesForm/SelectDataSourcesForm/SelectDataSourcesForm.tsx @@ -1,10 +1,11 @@ import { useContext } from 'react'; -import { Box, Image, Text } from '@chakra-ui/react'; +import { Box } from '@chakra-ui/react'; import { SteppedFormContext } from '@/components/SteppedForm/SteppedForm'; import { getConnectorsDefintions, ConnectorsDefinationApiResponse } from '@/services/connectors'; import { DatasourceType } from '@/views/Connectors/types'; import ContentContainer from '@/components/ContentContainer'; import useQueryWrapper from '@/hooks/useQueryWrapper'; +import EntityItem from '@/components/EntityItem'; const SelectDataSourcesForm = (): JSX.Element => { const { stepInfo, handleMoveForward } = useContext(SteppedFormContext); @@ -54,29 +55,7 @@ const SelectDataSourcesForm = (): JSX.Element => { height='56px' onClick={() => handleOnClick(datasource)} > - - - - - {datasource.title} - + ))} diff --git a/ui/src/views/Models/ViewModel/ViewModel.tsx b/ui/src/views/Models/ViewModel/ViewModel.tsx index 960b522c..0db79e73 100644 --- a/ui/src/views/Models/ViewModel/ViewModel.tsx +++ b/ui/src/views/Models/ViewModel/ViewModel.tsx @@ -165,7 +165,7 @@ const ViewModel = (): JSX.Element => { borderColor='gray.400' > From 82227945b7d7f3fc3292ca63204589ccf4463980 Mon Sep 17 00:00:00 2001 From: Sumit Dhanania Date: Fri, 9 Aug 2024 16:40:26 +0530 Subject: [PATCH 16/74] refactor(CE): remove account verify route --- ui/src/routes/main.tsx | 10 --- ui/src/views/AccountVerify/AccountVerify.tsx | 90 -------------------- ui/src/views/AccountVerify/index.ts | 1 - 3 files changed, 101 deletions(-) delete mode 100644 ui/src/views/AccountVerify/AccountVerify.tsx delete mode 100644 ui/src/views/AccountVerify/index.ts diff --git a/ui/src/routes/main.tsx b/ui/src/routes/main.tsx index 7a16fc8c..f1563519 100644 --- a/ui/src/routes/main.tsx +++ b/ui/src/routes/main.tsx @@ -3,7 +3,6 @@ const AboutUs = lazy(() => import('@/views/AboutUs')); const Dashboard = lazy(() => import('@/views/Dashboard')); const SignIn = lazy(() => import('@/views/Authentication/SignIn')); const SignUp = lazy(() => import('@/views/Authentication/SignUp')); -const AccountVerify = lazy(() => import('@/views/AccountVerify')); const Models = lazy(() => import('@/views/Models')); const SetupConnectors = lazy(() => import('@/views/Connectors/SetupConnectors')); @@ -103,13 +102,4 @@ export const AUTH_ROUTES: MAIN_PAGE_ROUTES_ITEM[] = [ ), }, - { - name: 'Account Verify', - url: '/account-verify', - component: ( - - - - ), - }, ]; diff --git a/ui/src/views/AccountVerify/AccountVerify.tsx b/ui/src/views/AccountVerify/AccountVerify.tsx deleted file mode 100644 index 8525b369..00000000 --- a/ui/src/views/AccountVerify/AccountVerify.tsx +++ /dev/null @@ -1,90 +0,0 @@ -import { Formik, Form, ErrorMessage } from 'formik'; -import * as Yup from 'yup'; -import { useNavigate } from 'react-router-dom'; -import { - Box, - Button, - FormControl, - Heading, - Input, - Container, - Stack, - HStack, - FormLabel, -} from '@chakra-ui/react'; -import MultiwovenIcon from '@/assets/images/icon.svg'; -import { useState } from 'react'; -import { accountVerify } from '@/services/common'; - -const AccountVerifySchema = Yup.object().shape({ - code: Yup.string().required('Code is required'), -}); - -const AccountVerify = (): JSX.Element => { - const [submitting, setSubmitting] = useState(false); - const navigate = useNavigate(); - - const handleSubmit = async (values: any) => { - setSubmitting(true); - const data = { - email: sessionStorage.getItem('userEmail'), - confirmation_code: values.code, - }; - const result = await accountVerify(data); - if (result.success) { - setSubmitting(false); - navigate('/login'); - } else { - setSubmitting(false); - } - }; - return ( - handleSubmit(values)} - validationSchema={AccountVerifySchema} - > - {({ getFieldProps, touched, errors }) => ( -
- - - - - - Verify your account - - - - - - - Verification Code - - - - - - - - - - - - - - )} - - ); -}; - -export default AccountVerify; diff --git a/ui/src/views/AccountVerify/index.ts b/ui/src/views/AccountVerify/index.ts deleted file mode 100644 index 9e917c81..00000000 --- a/ui/src/views/AccountVerify/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from './AccountVerify'; From f0bfaaddcff6d96ff76dfdc1781ea6a2d57f7e6e Mon Sep 17 00:00:00 2001 From: TivonB-AI2 Date: Mon, 12 Aug 2024 12:16:12 -0400 Subject: [PATCH 17/74] chore(CE): add request response log for Slack --- integrations/Gemfile.lock | 2 +- .../integrations/destination/slack/client.rb | 16 +++++++--------- .../lib/multiwoven/integrations/rollout.rb | 2 +- .../destination/slack/client_spec.rb | 12 ++++++++++++ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 846de3fe..3819c212 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.7.1) + multiwoven-integrations (0.7.4) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/destination/slack/client.rb b/integrations/lib/multiwoven/integrations/destination/slack/client.rb index 5cf85884..1dd464fd 100644 --- a/integrations/lib/multiwoven/integrations/destination/slack/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/slack/client.rb @@ -57,11 +57,13 @@ def configure_slack(api_token) end def process_records(records, stream) + log_message_array = [] write_success = 0 write_failure = 0 records.each do |record_object| - process_record(stream, record_object.with_indifferent_access) + request, response = *process_record(stream, record_object.with_indifferent_access) write_success += 1 + log_message_array << log_request_response("info", request, response) rescue StandardError => e write_failure += 1 handle_exception(e, { @@ -70,8 +72,9 @@ def process_records(records, stream) sync_id: @sync_config.sync_id, sync_run_id: @sync_config.sync_run_id }) + log_message_array << log_request_response("error", request, e.message) end - tracking_message(write_success, write_failure) + tracking_message(write_success, write_failure, log_message_array) end def process_record(stream, record) @@ -80,7 +83,8 @@ def process_record(stream, record) def send_data_to_slack(stream_name, record = {}) args = build_args(stream_name, record) - @client.send(stream_name, **args) + response = @client.send(stream_name, **args) + [args, response] end def build_args(stream_name, record) @@ -114,12 +118,6 @@ def failure_status(error) def load_catalog read_json(CATALOG_SPEC_PATH) end - - def tracking_message(success, failure) - Multiwoven::Integrations::Protocol::TrackingMessage.new( - success: success, failed: failure - ).to_multiwoven_message - end end end end diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index dd0b7129..9f9792f4 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.7.1" + VERSION = "0.7.4" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/spec/multiwoven/integrations/destination/slack/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/slack/client_spec.rb index 0278a196..491f78ce 100644 --- a/integrations/spec/multiwoven/integrations/destination/slack/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/destination/slack/client_spec.rb @@ -125,6 +125,12 @@ expect(response.tracking.success).to eq(records.size) expect(response.tracking.failed).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end @@ -142,6 +148,12 @@ expect(response.tracking.failed).to eq(records.size) expect(response.tracking.success).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("error") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end end From fbd2c5763cdbc75a185c72e6baf8c00cabd4fb27 Mon Sep 17 00:00:00 2001 From: TivonB-AI2 Date: Mon, 12 Aug 2024 12:46:50 -0400 Subject: [PATCH 18/74] chore(CE): add request response log for Airtable --- integrations/Gemfile.lock | 2 +- .../integrations/destination/airtable/client.rb | 13 ++++++------- .../lib/multiwoven/integrations/rollout.rb | 2 +- .../destination/airtable/client_spec.rb | 16 ++++++++++++++-- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 846de3fe..977b84c5 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.7.1) + multiwoven-integrations (0.7.7) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/destination/airtable/client.rb b/integrations/lib/multiwoven/integrations/destination/airtable/client.rb index 5ff4d570..8d89ff88 100644 --- a/integrations/lib/multiwoven/integrations/destination/airtable/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/airtable/client.rb @@ -59,10 +59,12 @@ def write(sync_config, records, _action = "create") connection_config = sync_config.destination.connection_specification.with_indifferent_access api_key = connection_config[:api_key] url = sync_config.stream.url + log_message_array = [] write_success = 0 write_failure = 0 records.each_slice(MAX_CHUNK_SIZE) do |chunk| payload = create_payload(chunk) + args = [sync_config.stream.request_method, url, payload] response = Multiwoven::Integrations::Core::HttpClient.request( url, sync_config.stream.request_method, @@ -74,6 +76,7 @@ def write(sync_config, records, _action = "create") else write_failure += chunk.size end + log_message_array << log_request_response("info", args, response) rescue StandardError => e handle_exception(e, { context: "AIRTABLE:RECORD:WRITE:EXCEPTION", @@ -82,13 +85,9 @@ def write(sync_config, records, _action = "create") sync_run_id: sync_config.sync_run_id }) write_failure += chunk.size + log_message_array << log_request_response("error", args, e.message) end - - tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new( - success: write_success, - failed: write_failure - ) - tracker.to_multiwoven_message + tracking_message(write_success, write_failure, log_message_array) rescue StandardError => e handle_exception(e, { context: "AIRTABLE:RECORD:WRITE:EXCEPTION", @@ -119,7 +118,7 @@ def auth_headers(access_token) end def base_id_exists?(bases, base_id) - return if extract_data(bases).any? { |base| base["id"] == base_id } + return if extract_bases(bases).any? { |base| base["id"] == base_id } raise ArgumentError, "base_id not found" end diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index dd0b7129..0b16f478 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.7.1" + VERSION = "0.7.7" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/spec/multiwoven/integrations/destination/airtable/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/airtable/client_spec.rb index 4bd23143..e3b3b411 100644 --- a/integrations/spec/multiwoven/integrations/destination/airtable/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/destination/airtable/client_spec.rb @@ -79,7 +79,7 @@ it "returns a successful connection status if the request is successful" do allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request).and_return(success_response) - allow(client).to receive(:extract_data).with(success_response).and_return([{ "id" => "app43WSzJbarW7bTX" }]) + allow(client).to receive(:extract_bases).with(success_response).and_return([{ "id" => "app43WSzJbarW7bTX" }]) message = client.check_connection(connection_config) expect(message).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage) result = message.connection_status @@ -88,7 +88,7 @@ it "raises an error if the base idis not found in the response" do allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request).and_return(success_response) - allow(client).to receive(:extract_data).with(success_response).and_return([{ "id" => "invalid" }]) + allow(client).to receive(:extract_bases).with(success_response).and_return([{ "id" => "invalid" }]) message = client.check_connection(connection_config) expect(message).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage) result = message.connection_status @@ -156,6 +156,12 @@ message = client.write(sync_config, records) expect(message.tracking.success).to eq(2) expect(message.tracking.failed).to eq(0) + log_message = message.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end it "increments the failure count" do @@ -164,6 +170,12 @@ message = client.write(sync_config, records) expect(message.tracking.success).to eq(0) expect(message.tracking.failed).to eq(2) + log_message = message.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end From 63f09c3fb6c333a1661e28e71785ee88383c8cf2 Mon Sep 17 00:00:00 2001 From: TivonB-AI2 Date: Mon, 12 Aug 2024 12:56:43 -0400 Subject: [PATCH 19/74] chore(CE): add request response log for HTTP --- integrations/Gemfile.lock | 2 +- .../integrations/destination/http/client.rb | 11 +++++------ integrations/lib/multiwoven/integrations/rollout.rb | 2 +- .../integrations/destination/http/client_spec.rb | 12 ++++++++++++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 846de3fe..167906bd 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.7.1) + multiwoven-integrations (0.7.8) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/destination/http/client.rb b/integrations/lib/multiwoven/integrations/destination/http/client.rb index 688786aa..80b695d5 100644 --- a/integrations/lib/multiwoven/integrations/destination/http/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/http/client.rb @@ -45,10 +45,12 @@ def write(sync_config, records, _action = "create") connection_config = sync_config.destination.connection_specification.with_indifferent_access url = connection_config[:destination_url] headers = connection_config[:headers] + log_message_array = [] write_success = 0 write_failure = 0 records.each_slice(MAX_CHUNK_SIZE) do |chunk| payload = create_payload(chunk) + args = [sync_config.stream.request_method, url, payload] response = Multiwoven::Integrations::Core::HttpClient.request( url, sync_config.stream.request_method, @@ -60,6 +62,7 @@ def write(sync_config, records, _action = "create") else write_failure += chunk.size end + log_message_array << log_request_response("info", args, response) rescue StandardError => e handle_exception(e, { context: "HTTP:RECORD:WRITE:EXCEPTION", @@ -68,13 +71,9 @@ def write(sync_config, records, _action = "create") sync_run_id: sync_config.sync_run_id }) write_failure += chunk.size + log_message_array << log_request_response("error", args, e.message) end - - tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new( - success: write_success, - failed: write_failure - ) - tracker.to_multiwoven_message + tracking_message(write_success, write_failure, log_message_array) rescue StandardError => e handle_exception(e, { context: "HTTP:RECORD:WRITE:EXCEPTION", diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index dd0b7129..e48822de 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.7.1" + VERSION = "0.7.8" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/spec/multiwoven/integrations/destination/http/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/http/client_spec.rb index 557291af..2d85944a 100644 --- a/integrations/spec/multiwoven/integrations/destination/http/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/destination/http/client_spec.rb @@ -114,6 +114,12 @@ response = client.write(sync_config, records) expect(response.tracking.success).to eq(records.size) expect(response.tracking.failed).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end @@ -130,6 +136,12 @@ response = client.write(sync_config, records) expect(response.tracking.failed).to eq(records.size) expect(response.tracking.success).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end end From 86f3e9f9a5969f85ac2d9a56dd6815240d1923b3 Mon Sep 17 00:00:00 2001 From: TivonB-AI2 Date: Mon, 12 Aug 2024 13:07:37 -0400 Subject: [PATCH 20/74] chore(CE): add request response log for Klaviyo --- integrations/Gemfile.lock | 2 +- .../integrations/destination/klaviyo/client.rb | 10 +++++----- integrations/lib/multiwoven/integrations/rollout.rb | 2 +- .../integrations/destination/klaviyo/client_spec.rb | 12 ++++++++++++ 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 846de3fe..71ed2acb 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.7.1) + multiwoven-integrations (0.7.9) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/destination/klaviyo/client.rb b/integrations/lib/multiwoven/integrations/destination/klaviyo/client.rb index 744817b2..78b649e4 100644 --- a/integrations/lib/multiwoven/integrations/destination/klaviyo/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/klaviyo/client.rb @@ -38,6 +38,7 @@ def write(sync_config, records, _action = "destination_insert") request_method = sync_config.stream.request_method + log_message_array = [] write_success = 0 write_failure = 0 records.each do |record| @@ -45,6 +46,7 @@ def write(sync_config, records, _action = "destination_insert") # Add hardcode values into payload record["data"] ||= {} record["data"]["type"] = sync_config.stream.name + args = [request_method, url, record] response = Multiwoven::Integrations::Core::HttpClient.request( url, @@ -57,6 +59,7 @@ def write(sync_config, records, _action = "destination_insert") else write_failure += 1 end + log_message_array << log_request_response("info", args, response) rescue StandardError => e handle_exception(e, { context: "KLAVIYO:RECORD:WRITE:FAILURE", @@ -65,12 +68,9 @@ def write(sync_config, records, _action = "destination_insert") sync_run_id: sync_config.sync_run_id }) write_failure += 1 + log_message_array << log_request_response("error", args, e.message) end - tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new( - success: write_success, - failed: write_failure - ) - tracker.to_multiwoven_message + tracking_message(write_success, write_failure, log_message_array) rescue StandardError => e # TODO: Handle rate limiting seperately handle_exception(e, { diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index dd0b7129..62c6d503 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.7.1" + VERSION = "0.7.9" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/spec/multiwoven/integrations/destination/klaviyo/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/klaviyo/client_spec.rb index e28bb6df..499e4a28 100644 --- a/integrations/spec/multiwoven/integrations/destination/klaviyo/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/destination/klaviyo/client_spec.rb @@ -193,6 +193,12 @@ expect(tracker.success).to eq(records.count) expect(tracker.failed).to eq(0) + log_message = message.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end @@ -208,6 +214,12 @@ tracker = message.tracking expect(tracker.failed).to eq(records.count) expect(tracker.success).to eq(0) + log_message = message.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("error") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end it "returns log message" do From 5ac0b638f4a7810827904cf49090b6ecebfb734f Mon Sep 17 00:00:00 2001 From: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> Date: Mon, 12 Aug 2024 13:22:32 -0400 Subject: [PATCH 21/74] chore(CE): add request response log for MariaDB (#286) --- integrations/Gemfile.lock | 2 +- .../integrations/destination/maria_db/client.rb | 11 ++++------- integrations/lib/multiwoven/integrations/rollout.rb | 2 +- .../integrations/destination/maria_db/client_spec.rb | 12 ++++++++++++ 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 846de3fe..13f1c643 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.7.1) + multiwoven-integrations (0.7.2) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/destination/maria_db/client.rb b/integrations/lib/multiwoven/integrations/destination/maria_db/client.rb index ccb74107..955f5371 100644 --- a/integrations/lib/multiwoven/integrations/destination/maria_db/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/maria_db/client.rb @@ -41,6 +41,7 @@ def write(sync_config, records, action = "destination_insert") primary_key = sync_config.model.primary_key db = create_connection(connection_config) + log_message_array = [] write_success = 0 write_failure = 0 @@ -50,6 +51,7 @@ def write(sync_config, records, action = "destination_insert") begin db.run(query) write_success += 1 + log_message_array << log_request_response("info", query, "Successful") rescue StandardError => e handle_exception(e, { context: "MARIA:DB:RECORD:WRITE:EXCEPTION", @@ -58,9 +60,10 @@ def write(sync_config, records, action = "destination_insert") sync_run_id: sync_config.sync_run_id }) write_failure += 1 + log_message_array << log_request_response("error", query, e.message) end end - tracking_message(write_success, write_failure) + tracking_message(write_success, write_failure, log_message_array) rescue StandardError => e handle_exception(e, { context: "MARIA:DB:RECORD:WRITE:EXCEPTION", @@ -106,12 +109,6 @@ def group_by_table(records) { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } } end end - - def tracking_message(success, failure) - Multiwoven::Integrations::Protocol::TrackingMessage.new( - success: success, failed: failure - ).to_multiwoven_message - end end end end diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index dd0b7129..779161a8 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.7.1" + VERSION = "0.7.2" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/spec/multiwoven/integrations/destination/maria_db/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/maria_db/client_spec.rb index fa8e4b0e..81efbf43 100644 --- a/integrations/spec/multiwoven/integrations/destination/maria_db/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/destination/maria_db/client_spec.rb @@ -114,6 +114,12 @@ response = client.write(sync_config, records) expect(response.tracking.success).to eq(records.size) expect(response.tracking.failed).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end @@ -135,6 +141,12 @@ response = client.write(sync_config, records) expect(response.tracking.failed).to eq(records.size) expect(response.tracking.success).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("error") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end end From 3a26b63ead8956de8a101eb63f99d0faeb395a81 Mon Sep 17 00:00:00 2001 From: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> Date: Mon, 12 Aug 2024 14:34:30 -0400 Subject: [PATCH 22/74] chore(CE): add request response log for Google Sheets (#287) --- integrations/Gemfile.lock | 2 +- .../destination/google_sheets/client.rb | 17 +++++++---------- .../lib/multiwoven/integrations/rollout.rb | 2 +- .../destination/google_sheets/client_spec.rb | 17 +++++++++++++---- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 13f1c643..87de8f18 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.7.2) + multiwoven-integrations (0.7.3) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/destination/google_sheets/client.rb b/integrations/lib/multiwoven/integrations/destination/google_sheets/client.rb index 337b306a..b5c3614e 100644 --- a/integrations/lib/multiwoven/integrations/destination/google_sheets/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/google_sheets/client.rb @@ -153,13 +153,15 @@ def extract_spreadsheet_id(link) # Batch has a limit of sending 2MB data. So creating a chunk of records to meet that limit def process_record_chunks(records, sync_config) + log_message_array = [] write_success = 0 write_failure = 0 records.each_slice(MAX_CHUNK_SIZE) do |chunk| values = prepare_chunk_values(chunk, sync_config.stream) - update_sheet_values(values, sync_config.stream.name) + request, response = *update_sheet_values(values, sync_config.stream.name) write_success += values.size + log_message_array << log_request_response("info", request, response) rescue StandardError => e handle_exception(e, { context: "GOOGLE_SHEETS:RECORD:WRITE:EXCEPTION", @@ -168,9 +170,9 @@ def process_record_chunks(records, sync_config) sync_run_id: sync_config.sync_run_id }) write_failure += chunk.size + log_message_array << log_request_response("error", request, e.message) end - - tracking_message(write_success, write_failure) + tracking_message(write_success, write_failure, log_message_array) end # We need to format the data to adhere to google sheets API format. This converts the sync mapped data to 2D array format expected by google sheets API @@ -199,19 +201,14 @@ def update_sheet_values(values, stream_name) ) # TODO: Remove & this is added for the test to pass we need - @client&.batch_update_values(@spreadsheet_id, batch_update_request) + response = @client&.batch_update_values(@spreadsheet_id, batch_update_request) + [batch_update_request, response] end def load_catalog read_json(CATALOG_SPEC_PATH) end - def tracking_message(success, failure) - Multiwoven::Integrations::Protocol::TrackingMessage.new( - success: success, failed: failure - ).to_multiwoven_message - end - def delete_extra_sheets(sheet_ids) # Leave one sheet intact as a spreadsheet must have at least one sheet. # Delete all other sheets. diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 779161a8..771ff5af 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.7.2" + VERSION = "0.7.3" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/spec/multiwoven/integrations/destination/google_sheets/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/google_sheets/client_spec.rb index 7dcca275..a65af061 100644 --- a/integrations/spec/multiwoven/integrations/destination/google_sheets/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/destination/google_sheets/client_spec.rb @@ -245,15 +245,18 @@ expect(response.tracking.success).to eq(records.size) expect(response.tracking.failed).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end context "when the write operation fails" do before do - batch_update_request = instance_double(Google::Apis::SheetsV4::BatchUpdateValuesRequest) - allow(google_sheets_service).to receive(:batch_update_values) - .with(@spreadsheet_id, batch_update_request) - .and_raise(Google::Apis::ClientError.new("Invalid request")) + allow(@client).to receive(:update_sheet_values).and_raise(StandardError.new("Failed to update_sheet_values")) end it "increments the failure count" do @@ -261,6 +264,12 @@ expect(response.tracking.failed).to eq(records.size) expect(response.tracking.success).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("error") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end end From c5961357ab357b44cbd80dccfe3d188640c45900 Mon Sep 17 00:00:00 2001 From: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> Date: Mon, 12 Aug 2024 14:44:43 -0400 Subject: [PATCH 23/74] chore(CE): add request response log for Iterable (#289) --- integrations/Gemfile.lock | 2 +- .../destination/iterable/client.rb | 24 +++++++++---------- .../lib/multiwoven/integrations/rollout.rb | 2 +- .../destination/iterable/client_spec.rb | 16 +++++++++++-- 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 3819c212..f64a5eba 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.7.4) + multiwoven-integrations (0.7.5) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/destination/iterable/client.rb b/integrations/lib/multiwoven/integrations/destination/iterable/client.rb index 8b55bd1c..0e3cb66e 100644 --- a/integrations/lib/multiwoven/integrations/destination/iterable/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/iterable/client.rb @@ -54,31 +54,35 @@ def initialize_client(connection_config) end def process_records(records, stream) + log_message_array = [] write_success = 0 write_failure = 0 records.each do |record_object| record = extract_data(record_object, stream.json_schema[:properties]) - response = process_stream(record, stream) + request, response = *process_stream(record, stream) if response.success? write_success += 1 else write_failure += 1 end + log_message_array << log_request_response("info", request, response.body) rescue StandardError => e handle_exception("ITERABLE:WRITE:EXCEPTION", "error", e) write_failure += 1 + log_message_array << log_request_response("error", request, e.message) end - tracking_message(write_success, write_failure) + tracking_message(write_success, write_failure, log_message_array) end def process_stream(record, stream) klass = ::Iterable.const_get(stream.name).new(*initialize_params(stream, record)) item_attrs = initialize_attribute(stream, record) - if stream.name == "CatalogItems" - klass.send(@action, item_attrs) - else - klass.send(@action) - end + response = if stream.name == "CatalogItems" + klass.send("create", item_attrs) + else + klass.send("create") + end + [item_attrs, response] end def initialize_params(stream, record) @@ -97,12 +101,6 @@ def initialize_attribute(stream, record) end end - def tracking_message(success, failure) - Multiwoven::Integrations::Protocol::TrackingMessage.new( - success: success, failed: failure - ).to_multiwoven_message - end - def load_catalog read_json(CATALOG_SPEC_PATH) end diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 9f9792f4..d6d174fb 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.7.4" + VERSION = "0.7.5" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/spec/multiwoven/integrations/destination/iterable/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/iterable/client_spec.rb index 572b5e68..6c562a24 100644 --- a/integrations/spec/multiwoven/integrations/destination/iterable/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/destination/iterable/client_spec.rb @@ -107,7 +107,7 @@ describe "#write" do context "when the write operation is successful" do before do - allow_any_instance_of(::Iterable::CatalogItems).to receive(:create).and_return(double(success?: true)) + allow_any_instance_of(::Iterable::CatalogItems).to receive(:create).and_return(double(success?: true, body: 1)) end it "increments the success count" do @@ -117,12 +117,18 @@ response = client.write(sync_config, records) expect(response.tracking.success).to eq(records.size) expect(response.tracking.failed).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end context "when the write operation fails" do before do - allow_any_instance_of(::Iterable::CatalogItems).to receive(:create).and_return(double(success?: false)) + allow_any_instance_of(::Iterable::CatalogItems).to receive(:create).and_return(double(success?: false, body: 1)) end it "increments the failure count" do sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json( @@ -131,6 +137,12 @@ response = client.write(sync_config, records) expect(response.tracking.failed).to eq(records.size) expect(response.tracking.success).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end end From 5602bf1533eaf8ba4fa86dbea96996ebbe668ab0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 13 Aug 2024 00:15:48 +0530 Subject: [PATCH 24/74] refactor(CE): Connector creation process --- ui/src/components/FormFooter/FormFooter.tsx | 92 +++++--- .../Activate/Syncs/EditSync/EditSync.tsx | 4 +- .../ConfigureSyncs/ConfigureSyncs.tsx | 4 +- .../SyncForm/FinaliseSync/FinaliseSync.tsx | 4 +- .../SelectDestination/SelectDestination.tsx | 4 +- .../ConnectorConfigForm.tsx} | 24 +- .../Connectors/ConnectorConfigForm/index.ts | 1 + .../ConnectorConnectionTest.tsx | 208 ++++++++++-------- .../ConnectorConnectionTestProgress.tsx | 103 +++++++++ .../ConnectorConnectionTest/index.ts | 1 + .../ConnectorFinaliseForm.tsx} | 60 +++-- .../Connectors/ConnectorFinaliseForm/index.ts | 1 + .../DestinationConfigForm.tsx | 67 ------ .../DestinationConfigForm/index.ts | 1 - .../DestinationConnectionTest.tsx | 129 ----------- .../DestinationConnectionTest/index.ts | 0 .../DestinationFinalizeForm/index.ts | 1 - .../DestinationsForm/DestinationsForm.tsx | 12 +- .../EditDestinations/EditDestinations.tsx | 4 +- .../Sources/EditSource/EditSource.tsx | 4 +- .../SourcesForm/SourceConfigForm/index.ts | 1 - .../SourceConnectionTest.tsx | 122 ---------- .../SourcesForm/SourceConnectionTest/index.ts | 1 - .../SourceFinalizeForm/SourceFinalizeForm.tsx | 4 +- .../SourcesForm/SourceFinalizeForm/index.ts | 1 - .../SourcesForm/SourceFormFooter/index.ts | 1 - .../Sources/SourcesForm/SourcesForm.tsx | 12 +- ui/src/views/Connectors/constant.ts | 17 +- ui/src/views/Connectors/types.ts | 4 +- .../DefineModel/DefineSQL/DefineSQL.tsx | 6 +- .../TableSelector/TableSelector.tsx | 6 +- .../FinalizeModel/FinalizeModel.tsx | 4 +- .../ModelMethod/SelectModelMethod.tsx | 4 +- ui/src/views/Settings/Workspace.tsx | 4 +- 34 files changed, 375 insertions(+), 536 deletions(-) rename ui/src/views/Connectors/{Sources/SourcesForm/SourceConfigForm/SourceConfigForm.tsx => ConnectorConfigForm/ConnectorConfigForm.tsx} (73%) create mode 100644 ui/src/views/Connectors/ConnectorConfigForm/index.ts create mode 100644 ui/src/views/Connectors/ConnectorConnectionTest/ConnectorConnectionTestProgress.tsx create mode 100644 ui/src/views/Connectors/ConnectorConnectionTest/index.ts rename ui/src/views/Connectors/{Destinations/DestinationsForm/DestinationFinalizeForm/DestinationFinalizeForm.tsx => ConnectorFinaliseForm/ConnectorFinaliseForm.tsx} (72%) create mode 100644 ui/src/views/Connectors/ConnectorFinaliseForm/index.ts delete mode 100644 ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConfigForm/DestinationConfigForm.tsx delete mode 100644 ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConfigForm/index.ts delete mode 100644 ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConnectionTest/DestinationConnectionTest.tsx delete mode 100644 ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConnectionTest/index.ts delete mode 100644 ui/src/views/Connectors/Destinations/DestinationsForm/DestinationFinalizeForm/index.ts delete mode 100644 ui/src/views/Connectors/Sources/SourcesForm/SourceConfigForm/index.ts delete mode 100644 ui/src/views/Connectors/Sources/SourcesForm/SourceConnectionTest/SourceConnectionTest.tsx delete mode 100644 ui/src/views/Connectors/Sources/SourcesForm/SourceConnectionTest/index.ts delete mode 100644 ui/src/views/Connectors/Sources/SourcesForm/SourceFinalizeForm/index.ts delete mode 100644 ui/src/views/Connectors/Sources/SourcesForm/SourceFormFooter/index.ts diff --git a/ui/src/components/FormFooter/FormFooter.tsx b/ui/src/components/FormFooter/FormFooter.tsx index 11343f12..1a8db00d 100644 --- a/ui/src/components/FormFooter/FormFooter.tsx +++ b/ui/src/components/FormFooter/FormFooter.tsx @@ -1,18 +1,23 @@ import { useUiConfig } from '@/utils/hooks'; import { Box, Button, ButtonGroup, Icon, Text } from '@chakra-ui/react'; import { useEffect, useState } from 'react'; -import { FiBookOpen, FiHeadphones } from 'react-icons/fi'; +import { FiBookOpen, FiSlack } from 'react-icons/fi'; import { Link, useNavigate } from 'react-router-dom'; -type SourceFormFooterProps = { +type FormFooterProps = { ctaName: string; + secondaryCtaText?: string; ctaType?: 'button' | 'reset' | 'submit' | undefined; onCtaClick?: undefined | (() => void); isCtaDisabled?: boolean; isCtaLoading?: boolean; isBackRequired?: boolean; + isContinueCtaRequired?: boolean; + isDocumentsSectionRequired?: boolean; isAlignToContentContainer?: boolean; extra?: JSX.Element; + navigateToListScreen?: boolean; + listScreenUrl?: string; }; const FormFooter = ({ @@ -22,9 +27,14 @@ const FormFooter = ({ onCtaClick, isBackRequired, extra, + listScreenUrl, isCtaLoading = false, isCtaDisabled = false, -}: SourceFormFooterProps): JSX.Element => { + isContinueCtaRequired = false, + isDocumentsSectionRequired = false, + secondaryCtaText = 'Back', + navigateToListScreen = false, +}: FormFooterProps): JSX.Element => { const [leftOffset, setLeftOffet] = useState(0); const { maxContentWidth } = useUiConfig(); const navigate = useNavigate(); @@ -42,9 +52,9 @@ const FormFooter = ({ left={leftOffset} right='0' borderWidth='thin' + borderColor='gray.400' bottom='0' backgroundColor='gray.100' - padding='30px' display='flex' justifyContent='center' minHeight='80px' @@ -56,48 +66,62 @@ const FormFooter = ({ display='flex' justifyContent='space-between' alignItems='center' + paddingX='30px' > - - - - - - Read Documentation - - - - - - - - Contact Support - - - + + {isDocumentsSectionRequired ? ( + <> + + + + + Read Documentation + + + + + + + + Contact Support + + + + + ) : null} {extra} {isBackRequired ? ( + ) : null} + {isContinueCtaRequired ? ( + ) : null} - diff --git a/ui/src/views/Activate/Syncs/EditSync/EditSync.tsx b/ui/src/views/Activate/Syncs/EditSync/EditSync.tsx index 57306b05..39590115 100644 --- a/ui/src/views/Activate/Syncs/EditSync/EditSync.tsx +++ b/ui/src/views/Activate/Syncs/EditSync/EditSync.tsx @@ -20,7 +20,7 @@ import { } from '@/views/Activate/Syncs/types'; import ScheduleForm from './ScheduleForm'; import { FormikProps, useFormik } from 'formik'; -import SourceFormFooter from '@/views/Connectors/Sources/SourcesForm/SourceFormFooter'; +import FormFooter from '@/components/FormFooter'; import { FieldMap as FieldMapType } from '@/views/Activate/Syncs/types'; import MapCustomFields from '../SyncForm/ConfigureSyncs/MapCustomFields'; import { useStore } from '@/stores'; @@ -264,7 +264,7 @@ const EditSync = (): JSX.Element | null => { ) : null} - )} - { - - + )} diff --git a/ui/src/views/Connectors/Sources/SourcesForm/SourceConfigForm/SourceConfigForm.tsx b/ui/src/views/Connectors/ConnectorConfigForm/ConnectorConfigForm.tsx similarity index 73% rename from ui/src/views/Connectors/Sources/SourcesForm/SourceConfigForm/SourceConfigForm.tsx rename to ui/src/views/Connectors/ConnectorConfigForm/ConnectorConfigForm.tsx index 90c30e56..c1996394 100644 --- a/ui/src/views/Connectors/Sources/SourcesForm/SourceConfigForm/SourceConfigForm.tsx +++ b/ui/src/views/Connectors/ConnectorConfigForm/ConnectorConfigForm.tsx @@ -5,7 +5,7 @@ import { getConnectorDefinition } from '@/services/connectors'; import { useContext } from 'react'; import { Box } from '@chakra-ui/react'; -import SourceFormFooter from '@/views/Connectors/Sources/SourcesForm/SourceFormFooter'; +import FormFooter from '@/components/FormFooter'; import Loader from '@/components/Loader'; import { processFormData } from '@/views/Connectors/helpers'; @@ -14,19 +14,23 @@ import { generateUiSchema } from '@/utils/generateUiSchema'; import JSONSchemaForm from '@/components/JSONSchemaForm'; import { useStore } from '@/stores'; -const SourceConfigForm = (): JSX.Element | null => { +const ConnectorConfigForm = ({ connectorType }: { connectorType: string }): JSX.Element | null => { const { state, stepInfo, handleMoveForward } = useContext(SteppedFormContext); const { forms } = state; - const selectedDataSource = forms.find(({ stepKey }) => stepKey === 'datasource'); - const datasource = selectedDataSource?.data?.datasource as string; + const selectedConnector = forms.find( + ({ stepKey }) => stepKey === (connectorType === 'source' ? 'datasource' : connectorType), + ); + const connector = selectedConnector?.data?.[ + connectorType === 'source' ? 'datasource' : connectorType + ] as string; const activeWorkspaceId = useStore((state) => state.workspaceId); - if (!datasource) return null; + if (!connector) return null; const { data, isLoading } = useQuery({ - queryKey: ['connector_definition', datasource, activeWorkspaceId], - queryFn: () => getConnectorDefinition('source', datasource), - enabled: !!datasource && activeWorkspaceId > 0, + queryKey: ['connector_definition', connector, activeWorkspaceId], + queryFn: () => getConnectorDefinition(connectorType, connector), + enabled: !!connector && activeWorkspaceId > 0, refetchOnMount: false, refetchOnWindowFocus: false, }); @@ -52,7 +56,7 @@ const SourceConfigForm = (): JSX.Element | null => { uiSchema={generatedSchema} onSubmit={(formData: FormData) => handleFormSubmit(formData)} > - { ); }; -export default SourceConfigForm; +export default ConnectorConfigForm; diff --git a/ui/src/views/Connectors/ConnectorConfigForm/index.ts b/ui/src/views/Connectors/ConnectorConfigForm/index.ts new file mode 100644 index 00000000..cd945034 --- /dev/null +++ b/ui/src/views/Connectors/ConnectorConfigForm/index.ts @@ -0,0 +1 @@ +export { default } from './ConnectorConfigForm'; diff --git a/ui/src/views/Connectors/ConnectorConnectionTest/ConnectorConnectionTest.tsx b/ui/src/views/Connectors/ConnectorConnectionTest/ConnectorConnectionTest.tsx index d2dd3a47..f939ad2d 100644 --- a/ui/src/views/Connectors/ConnectorConnectionTest/ConnectorConnectionTest.tsx +++ b/ui/src/views/Connectors/ConnectorConnectionTest/ConnectorConnectionTest.tsx @@ -1,102 +1,128 @@ -import { Box, Icon, Spinner, Text } from '@chakra-ui/react'; -import { CONNECTION_STATUS } from '@/views/Connectors/constant'; -import { FiAlertOctagon, FiCheck } from 'react-icons/fi'; -import { TestConnectionResponse } from '../types'; +import { useContext, useMemo } from 'react'; +import { useQuery } from '@tanstack/react-query'; +import { Alert, AlertDescription, AlertTitle, Box, Button } from '@chakra-ui/react'; +import { SteppedFormContext } from '@/components/SteppedForm/SteppedForm'; +import { getConnectionStatus } from '@/services/connectors'; +import { processConnectorConfigData } from '@/views/Connectors/helpers'; +import { ConnectorTypes, TestConnectionPayload } from '@/views/Connectors/types'; +import FormFooter from '@/components/FormFooter'; +import ContentContainer from '@/components/ContentContainer'; +import { useStore } from '@/stores'; +import ConnectorConnectionTestProgress from '@/views/Connectors/ConnectorConnectionTest/ConnectorConnectionTestProgress'; -const STATUS_COLOR_MAP = { - success: 'green.400', - failed: 'red.400', - loading: 'gray.800', -}; - -const STATUS_TEXT_COLOR = { - success: 'success.500', - failed: 'error.500', - loading: 'black.100', -}; +const CONNECT_TO_SOURCES_KEY = 'connectToSources'; +const CONNECT_TO_DESTINATION_KEY = 'destinationConfig'; const ConnectorConnectionTest = ({ - connectionResponse, - isFetching, - connectorConfig, - selectedConnectorSource, + connectorType, }: { - connectionResponse?: TestConnectionResponse; - isFetching: boolean; - connectorConfig: unknown; - selectedConnectorSource: string; -}) => { - return ( - <> - {CONNECTION_STATUS.map(({ status }) => { - const statusMetaInfo = status({ - data: connectionResponse, - isLoading: isFetching, - configFormData: connectorConfig, - datasource: selectedConnectorSource, - }); + connectorType: ConnectorTypes; +}): JSX.Element | null => { + const activeWorkspaceId = useStore((state) => state.workspaceId); + const { state, stepInfo, handleMoveForward } = useContext(SteppedFormContext); + const { forms } = state; - return ( - - - {statusMetaInfo.status === 'loading' ? ( - - ) : null} - {statusMetaInfo.status === 'success' ? ( - - - - ) : null} + const connectorKey = connectorType === 'source' ? 'datasource' : connectorType; + const selectedConnector = forms.find(({ stepKey }) => stepKey === connectorKey)?.data?.[ + connectorKey + ] as string; - {statusMetaInfo.status === 'failed' ? ( - - - - ) : null} - - - stepKey === configKey); + const connectorConfig = connectorConfigForm?.data?.[configKey]; + + const processedConnectorConfig = useMemo( + () => processConnectorConfigData(connectorConfig, selectedConnector, connectorType), + [connectorConfig, selectedConnector, connectorType], + ); + + const { + data: connectionResponse, + refetch: retryConnectorConnection, + isFetching, + } = useQuery({ + queryKey: ['connector_definition', 'test-connection', connectorType, activeWorkspaceId], + queryFn: () => getConnectionStatus(processedConnectorConfig as TestConnectionPayload), + enabled: !!processedConnectorConfig && activeWorkspaceId > 0, + refetchOnMount: true, + refetchOnWindowFocus: false, + }); + + const isConnectionFailed = connectionResponse?.connection_status.status !== 'succeeded'; + + const handleContinueClick = () => { + handleMoveForward(stepInfo?.formKey as string, processedConnectorConfig); + }; + + return ( + + + + + + {isConnectionFailed && ( + + )} - ); - })} - + {!isFetching && ( + + + + {isConnectionFailed + ? 'Could not open a connection to remote host' + : 'Connected successfully!'} + + + {isConnectionFailed + ? connectionResponse?.connection_status.message + : `All tests passed. Continue to finish setting up your ${selectedConnector} Source`} + + + + )} + + + + ); }; diff --git a/ui/src/views/Connectors/ConnectorConnectionTest/ConnectorConnectionTestProgress.tsx b/ui/src/views/Connectors/ConnectorConnectionTest/ConnectorConnectionTestProgress.tsx new file mode 100644 index 00000000..d2dd3a47 --- /dev/null +++ b/ui/src/views/Connectors/ConnectorConnectionTest/ConnectorConnectionTestProgress.tsx @@ -0,0 +1,103 @@ +import { Box, Icon, Spinner, Text } from '@chakra-ui/react'; +import { CONNECTION_STATUS } from '@/views/Connectors/constant'; +import { FiAlertOctagon, FiCheck } from 'react-icons/fi'; +import { TestConnectionResponse } from '../types'; + +const STATUS_COLOR_MAP = { + success: 'green.400', + failed: 'red.400', + loading: 'gray.800', +}; + +const STATUS_TEXT_COLOR = { + success: 'success.500', + failed: 'error.500', + loading: 'black.100', +}; + +const ConnectorConnectionTest = ({ + connectionResponse, + isFetching, + connectorConfig, + selectedConnectorSource, +}: { + connectionResponse?: TestConnectionResponse; + isFetching: boolean; + connectorConfig: unknown; + selectedConnectorSource: string; +}) => { + return ( + <> + {CONNECTION_STATUS.map(({ status }) => { + const statusMetaInfo = status({ + data: connectionResponse, + isLoading: isFetching, + configFormData: connectorConfig, + datasource: selectedConnectorSource, + }); + + return ( + + + {statusMetaInfo.status === 'loading' ? ( + + ) : null} + {statusMetaInfo.status === 'success' ? ( + + + + ) : null} + + {statusMetaInfo.status === 'failed' ? ( + + + + ) : null} + + + + {statusMetaInfo.text} + + + + ); + })} + + ); +}; + +export default ConnectorConnectionTest; diff --git a/ui/src/views/Connectors/ConnectorConnectionTest/index.ts b/ui/src/views/Connectors/ConnectorConnectionTest/index.ts new file mode 100644 index 00000000..2c2da631 --- /dev/null +++ b/ui/src/views/Connectors/ConnectorConnectionTest/index.ts @@ -0,0 +1 @@ +export { default } from './ConnectorConnectionTest'; diff --git a/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationFinalizeForm/DestinationFinalizeForm.tsx b/ui/src/views/Connectors/ConnectorFinaliseForm/ConnectorFinaliseForm.tsx similarity index 72% rename from ui/src/views/Connectors/Destinations/DestinationsForm/DestinationFinalizeForm/DestinationFinalizeForm.tsx rename to ui/src/views/Connectors/ConnectorFinaliseForm/ConnectorFinaliseForm.tsx index 1605417c..fe7f6fdf 100644 --- a/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationFinalizeForm/DestinationFinalizeForm.tsx +++ b/ui/src/views/Connectors/ConnectorFinaliseForm/ConnectorFinaliseForm.tsx @@ -2,20 +2,28 @@ import { Box, Input, Text, Textarea } from '@chakra-ui/react'; import { useFormik } from 'formik'; import { useContext, useState } from 'react'; import { SteppedFormContext } from '@/components/SteppedForm/SteppedForm'; -import { CreateConnectorPayload, TestConnectionPayload } from '@/views/Connectors/types'; +import { + CreateConnectorPayload, + TestConnectionPayload, + ConnectorTypes, +} from '@/views/Connectors/types'; import { useNavigate } from 'react-router-dom'; import { createNewConnector } from '@/services/connectors'; import { useQueryClient } from '@tanstack/react-query'; -import { DESTINATIONS_LIST_QUERY_KEY } from '@/views/Connectors/constant'; import { useUiConfig } from '@/utils/hooks'; -import SourceFormFooter from '@/views/Connectors/Sources/SourcesForm/SourceFormFooter'; +import FormFooter from '@/components/FormFooter'; import ContentContainer from '@/components/ContentContainer'; import { CustomToastStatus } from '@/components/Toast/index'; import useCustomToast from '@/hooks/useCustomToast'; const finalDestinationConfigFormKey = 'testDestination'; +const finalDataSourceFormKey = 'testSource'; -const DestinationFinalizeForm = (): JSX.Element | null => { +const ConnectorFinaliseForm = ({ + connectorType, +}: { + connectorType: ConnectorTypes; +}): JSX.Element | null => { const [isLoading, setIsLoading] = useState(false); const { maxContentWidth } = useUiConfig(); const { state } = useContext(SteppedFormContext); @@ -23,15 +31,21 @@ const DestinationFinalizeForm = (): JSX.Element | null => { const showToast = useCustomToast(); const navigate = useNavigate(); const queryClient = useQueryClient(); - const finalDestinationConfigForm = forms.find( - ({ stepKey }) => stepKey === finalDestinationConfigFormKey, - )?.data?.[finalDestinationConfigFormKey] as TestConnectionPayload | undefined; - if (!finalDestinationConfigForm) return null; + const CONNECTOR_TYPE_TITLE = connectorType === 'source' ? 'Source' : 'Destination'; + + const configKey = + connectorType === 'source' ? finalDataSourceFormKey : finalDestinationConfigFormKey; + + const finalConnectorConfigForm = forms.find(({ stepKey }) => stepKey === configKey)?.data?.[ + configKey + ] as TestConnectionPayload | undefined; + + if (!finalConnectorConfigForm) return null; const formik = useFormik({ initialValues: { - connector_name: finalDestinationConfigForm.name, + connector_name: finalConnectorConfigForm.name, description: '', }, onSubmit: async (formData) => { @@ -39,10 +53,10 @@ const DestinationFinalizeForm = (): JSX.Element | null => { try { const payload: CreateConnectorPayload = { connector: { - configuration: finalDestinationConfigForm.connection_spec, + configuration: finalConnectorConfigForm.connection_spec, name: formData.connector_name, - connector_type: 'destination', - connector_name: finalDestinationConfigForm.name, + connector_type: connectorType, + connector_name: finalConnectorConfigForm.name, description: formData.description, }, }; @@ -50,16 +64,20 @@ const DestinationFinalizeForm = (): JSX.Element | null => { const createConnectorResponse = await createNewConnector(payload); if (createConnectorResponse?.data) { queryClient.removeQueries({ - queryKey: DESTINATIONS_LIST_QUERY_KEY, + queryKey: ['connectors', connectorType], }); showToast({ status: CustomToastStatus.Success, title: 'Success!!', - description: 'Destination created successfully!', + description: `${CONNECTOR_TYPE_TITLE} created successfully!`, position: 'bottom-right', }); - navigate('/setup/destinations'); + if (connectorType === 'source') { + navigate('/setup/sources'); + } else { + navigate('/setup/destinations'); + } } else { throw new Error(); } @@ -67,7 +85,7 @@ const DestinationFinalizeForm = (): JSX.Element | null => { showToast({ status: CustomToastStatus.Error, title: 'An error occurred.', - description: 'Something went wrong while creating the Destination.', + description: `Something went wrong while creating the ${CONNECTOR_TYPE_TITLE}.`, position: 'bottom-right', isClosable: true, }); @@ -84,16 +102,16 @@ const DestinationFinalizeForm = (): JSX.Element | null => {
- Finalize settings for this Destination + {`Finalize settings for this ${CONNECTOR_TYPE_TITLE}`} - Destination Name + {`${CONNECTOR_TYPE_TITLE} Name`} { /> - { ); }; -export default DestinationFinalizeForm; +export default ConnectorFinaliseForm; diff --git a/ui/src/views/Connectors/ConnectorFinaliseForm/index.ts b/ui/src/views/Connectors/ConnectorFinaliseForm/index.ts new file mode 100644 index 00000000..138d36cb --- /dev/null +++ b/ui/src/views/Connectors/ConnectorFinaliseForm/index.ts @@ -0,0 +1 @@ +export { default } from './ConnectorFinaliseForm'; diff --git a/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConfigForm/DestinationConfigForm.tsx b/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConfigForm/DestinationConfigForm.tsx deleted file mode 100644 index 5379c4db..00000000 --- a/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConfigForm/DestinationConfigForm.tsx +++ /dev/null @@ -1,67 +0,0 @@ -import { SteppedFormContext } from '@/components/SteppedForm/SteppedForm'; -import { getConnectorDefinition } from '@/services/connectors'; -import { Box } from '@chakra-ui/react'; -import { useQuery } from '@tanstack/react-query'; -import { useContext } from 'react'; - -import Loader from '@/components/Loader'; -import ContentContainer from '@/components/ContentContainer'; -import SourceFormFooter from '@/views/Connectors/Sources/SourcesForm/SourceFormFooter'; -import JSONSchemaForm from '@/components/JSONSchemaForm'; -import { generateUiSchema } from '@/utils/generateUiSchema'; -import { useStore } from '@/stores'; -import { processFormData } from '@/views/Connectors/helpers'; - -const DestinationConfigForm = (): JSX.Element | null => { - const { state, stepInfo, handleMoveForward } = useContext(SteppedFormContext); - const { forms } = state; - const selectedDestination = forms.find(({ stepKey }) => stepKey === 'destination'); - const activeWorkspaceId = useStore((state) => state.workspaceId); - - const destination = selectedDestination?.data?.destination as string; - if (!destination) return null; - - const { data, isLoading } = useQuery({ - queryKey: ['connector_definition', destination, activeWorkspaceId], - queryFn: () => getConnectorDefinition('destination', destination), - enabled: !!destination && activeWorkspaceId > 0, - refetchOnMount: false, - refetchOnWindowFocus: false, - }); - - if (isLoading) return ; - - const connectorSchema = data?.data?.connector_spec?.connection_specification; - if (!connectorSchema) return null; - - const handleFormSubmit = async (formData: FormData) => { - const processedFormData = processFormData(formData); - handleMoveForward(stepInfo?.formKey as string, processedFormData); - }; - - const generatedSchema = generateUiSchema(connectorSchema); - - return ( - - - - handleFormSubmit(formData)} - > - - - - - - ); -}; - -export default DestinationConfigForm; diff --git a/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConfigForm/index.ts b/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConfigForm/index.ts deleted file mode 100644 index 33a51ce2..00000000 --- a/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConfigForm/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from './DestinationConfigForm'; diff --git a/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConnectionTest/DestinationConnectionTest.tsx b/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConnectionTest/DestinationConnectionTest.tsx deleted file mode 100644 index 30e57f5d..00000000 --- a/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConnectionTest/DestinationConnectionTest.tsx +++ /dev/null @@ -1,129 +0,0 @@ -import { SteppedFormContext } from '@/components/SteppedForm/SteppedForm'; -import { getConnectionStatus } from '@/services/connectors'; -import { processConnectorConfigData } from '@/views/Connectors/helpers'; -import { TestConnectionPayload } from '@/views/Connectors/types'; -import { Alert, AlertDescription, AlertTitle, Box, Button } from '@chakra-ui/react'; -import { useQuery } from '@tanstack/react-query'; -import { useContext, useMemo } from 'react'; -import { useUiConfig } from '@/utils/hooks'; -import SourceFormFooter from '@/views/Connectors/Sources/SourcesForm/SourceFormFooter'; -import ContentContainer from '@/components/ContentContainer'; -import { useStore } from '@/stores'; -import ConnectorConnectionTest from '@/views/Connectors/ConnectorConnectionTest/ConnectorConnectionTest'; - -const CONNECT_TO_DESTINATION_KEY = 'destinationConfig'; - -const DestinationConnectionTest = (): JSX.Element | null => { - const { state, stepInfo, handleMoveForward } = useContext(SteppedFormContext); - const { forms } = state; - const { maxContentWidth } = useUiConfig(); - - let selectedDestination = forms.find(({ stepKey }) => stepKey === 'destination')?.data - ?.destination as string; - - const destinationConfigForm = forms.find(({ stepKey }) => stepKey === CONNECT_TO_DESTINATION_KEY); - - const { data } = destinationConfigForm ?? {}; - const destinationConfig = data?.[CONNECT_TO_DESTINATION_KEY]; - const processedDestinationConfig = useMemo( - () => processConnectorConfigData(destinationConfig, selectedDestination, 'destination'), - [forms], - ); - - const activeWorkspaceId = useStore((state) => state.workspaceId); - - if (+activeWorkspaceId === 18 && selectedDestination.toLowerCase() === 'postgresql') { - selectedDestination = 'AIS Datastore'; - } - - const { - data: connectionResponse, - refetch: retryDestinationConnection, - isFetching, - } = useQuery({ - queryKey: ['connector_definition', 'test-connection', 'destination', activeWorkspaceId], - queryFn: () => getConnectionStatus(processedDestinationConfig as TestConnectionPayload), - enabled: !!processedDestinationConfig && activeWorkspaceId > 0, - refetchOnMount: true, - refetchOnWindowFocus: false, - }); - - const isAnyFailed = connectionResponse?.connection_status?.status !== 'succeeded'; - - const handleOnContinueClick = () => { - handleMoveForward(stepInfo?.formKey as string, processedDestinationConfig); - }; - - return ( - - - - - - - {isAnyFailed && connectionResponse ? ( - - ) : null} - - {!isFetching ? ( - - - - {isAnyFailed - ? 'Could not open a connection to remote host' - : 'Connected successfully!'} - - - {isAnyFailed - ? connectionResponse?.connection_status?.message - : `All tests passed. Continue to finish setting up your ${selectedDestination} Destination`} - - - - ) : null} - - - - - ); -}; - -export default DestinationConnectionTest; diff --git a/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConnectionTest/index.ts b/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationConnectionTest/index.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationFinalizeForm/index.ts b/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationFinalizeForm/index.ts deleted file mode 100644 index e5bf8905..00000000 --- a/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationFinalizeForm/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from './DestinationFinalizeForm'; diff --git a/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationsForm.tsx b/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationsForm.tsx index 29e6041f..cce0d22e 100644 --- a/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationsForm.tsx +++ b/ui/src/views/Connectors/Destinations/DestinationsForm/DestinationsForm.tsx @@ -2,9 +2,9 @@ import SteppedForm from '@/components/SteppedForm'; import { Box, Drawer, DrawerBody, DrawerContent, DrawerOverlay } from '@chakra-ui/react'; import { useNavigate } from 'react-router-dom'; import SelectDestinations from './SelectDestinations'; -import DestinationConfigForm from './DestinationConfigForm'; -import DestinationFinalizeForm from './DestinationFinalizeForm'; -import DestinationConnectionTest from './DestinationConnectionTest/DestinationConnectionTest'; +import ConnectorConfigForm from '@/views/Connectors/ConnectorConfigForm'; +import ConnectorConnectionTest from '@/views/Connectors/ConnectorConnectionTest'; +import ConnectorFinaliseForm from '@/views/Connectors/ConnectorFinaliseForm'; const DestinationsForm = (): JSX.Element => { const navigate = useNavigate(); @@ -19,19 +19,19 @@ const DestinationsForm = (): JSX.Element => { { formKey: 'destinationConfig', name: 'Connect your Destination', - component: , + component: , isRequireContinueCta: false, }, { formKey: 'testDestination', name: 'Test your Destination', - component: , + component: , isRequireContinueCta: false, }, { formKey: 'finalizeDestination', name: 'Finalize your Destination', - component: , + component: , isRequireContinueCta: false, }, ]; diff --git a/ui/src/views/Connectors/Destinations/EditDestinations/EditDestinations.tsx b/ui/src/views/Connectors/Destinations/EditDestinations/EditDestinations.tsx index 73840e20..657ff9ae 100644 --- a/ui/src/views/Connectors/Destinations/EditDestinations/EditDestinations.tsx +++ b/ui/src/views/Connectors/Destinations/EditDestinations/EditDestinations.tsx @@ -13,7 +13,6 @@ import ContentContainer from '@/components/ContentContainer'; import { useEffect, useState } from 'react'; import { CreateConnectorPayload, TestConnectionPayload } from '../../types'; import { RJSFSchema } from '@rjsf/utils'; -import SourceFormFooter from '../../Sources/SourcesForm/SourceFormFooter'; import Loader from '@/components/Loader'; import { Step } from '@/components/Breadcrumbs/types'; import EntityItem from '@/components/EntityItem'; @@ -24,6 +23,7 @@ import useCustomToast from '@/hooks/useCustomToast'; import { generateUiSchema } from '@/utils/generateUiSchema'; import JSONSchemaForm from '../../../../components/JSONSchemaForm'; import { useStore } from '@/stores'; +import FormFooter from '@/components/FormFooter'; const EditDestination = (): JSX.Element => { const activeWorkspaceId = useStore((state) => state.workspaceId); @@ -214,7 +214,7 @@ const EditDestination = (): JSX.Element => { onSubmit={(formData: FormData) => handleOnTestClick(formData)} onChange={(formData: FormData) => setFormData(formData)} > - { const activeWorkspaceId = useStore((state) => state.workspaceId); @@ -210,7 +210,7 @@ const EditSource = (): JSX.Element => { onSubmit={(formData: FormData) => handleOnTestClick(formData)} onChange={(formData: FormData) => setFormData(formData)} > - { - const activeWorkspaceId = useStore((state) => state.workspaceId); - - const { state, stepInfo, handleMoveForward } = useContext(SteppedFormContext); - const { forms } = state; - - const selectedDataSource = forms.find(({ stepKey }) => stepKey === 'datasource')?.data - ?.datasource as string; - - const sourceConfigForm = forms.find(({ stepKey }) => stepKey === CONNECT_TO_SOURCES_KEY); - const { data } = sourceConfigForm ?? {}; - const sourceConfig = data?.[CONNECT_TO_SOURCES_KEY]; - const processedSourceConfig = useMemo( - () => processConnectorConfigData(sourceConfig, selectedDataSource, 'source'), - [forms], - ); - - const { - data: connectionResponse, - refetch: retrySourceConnection, - isFetching, - } = useQuery({ - queryKey: ['connector_definition', 'test-connection', 'source', activeWorkspaceId], - queryFn: () => getConnectionStatus(processedSourceConfig as TestConnectionPayload), - enabled: !!processedSourceConfig && activeWorkspaceId > 0, - refetchOnMount: true, - refetchOnWindowFocus: false, - }); - - const isAnyFailed = connectionResponse?.connection_status.status !== 'succeeded'; - - const handleOnContinueClick = () => { - handleMoveForward(stepInfo?.formKey as string, processedSourceConfig); - }; - - return ( - - - - - - - {isAnyFailed && connectionResponse ? ( - - ) : null} - - {!isFetching ? ( - - - - {isAnyFailed - ? 'Could not open a connection to remote host' - : 'Connected successfully!'} - - - {isAnyFailed - ? connectionResponse?.connection_status.message - : `All tests passed. Continue to finish setting up your ${selectedDataSource} Source`} - - - - ) : null} - - - - - ); -}; - -export default SourceConnectionTest; diff --git a/ui/src/views/Connectors/Sources/SourcesForm/SourceConnectionTest/index.ts b/ui/src/views/Connectors/Sources/SourcesForm/SourceConnectionTest/index.ts deleted file mode 100644 index c217bc5e..00000000 --- a/ui/src/views/Connectors/Sources/SourcesForm/SourceConnectionTest/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from './SourceConnectionTest'; diff --git a/ui/src/views/Connectors/Sources/SourcesForm/SourceFinalizeForm/SourceFinalizeForm.tsx b/ui/src/views/Connectors/Sources/SourcesForm/SourceFinalizeForm/SourceFinalizeForm.tsx index 208cc0fc..d3a0044d 100644 --- a/ui/src/views/Connectors/Sources/SourcesForm/SourceFinalizeForm/SourceFinalizeForm.tsx +++ b/ui/src/views/Connectors/Sources/SourcesForm/SourceFinalizeForm/SourceFinalizeForm.tsx @@ -1,5 +1,5 @@ import { Box, Input, Text, Textarea } from '@chakra-ui/react'; -import SourceFormFooter from '../SourceFormFooter'; +import FormFooter from '@/components/FormFooter'; import { useFormik } from 'formik'; import { useContext, useState } from 'react'; import { SteppedFormContext } from '@/components/SteppedForm/SteppedForm'; @@ -123,7 +123,7 @@ const SourceFinalizeForm = (): JSX.Element | null => { /> - { const navigate = useNavigate(); @@ -19,19 +19,19 @@ const SourcesForm = (): JSX.Element => { { formKey: 'connectToSources', name: 'Connect to your Source', - component: , + component: , isRequireContinueCta: false, }, { formKey: 'testSource', name: 'Test your Source', - component: , + component: , isRequireContinueCta: false, }, { formKey: 'finalizeSource', name: 'Finalize your Source', - component: , + component: , isRequireContinueCta: false, }, ]; diff --git a/ui/src/views/Connectors/constant.ts b/ui/src/views/Connectors/constant.ts index 3039f375..70d6fd03 100644 --- a/ui/src/views/Connectors/constant.ts +++ b/ui/src/views/Connectors/constant.ts @@ -1,19 +1,4 @@ -import { ConnectionStatus, ConnectorTypes, SourceListColumnType } from './types'; - -export const CONNECTORS: Record> = { - source: { - name: 'Source', - key: 'source', - }, - destination: { - name: 'Destination', - key: 'destination', - }, - model: { - name: 'Model', - key: 'model', - }, -}; +import { ConnectionStatus, SourceListColumnType } from './types'; export const CONNECTION_STATUS: ConnectionStatus[] = [ { diff --git a/ui/src/views/Connectors/types.ts b/ui/src/views/Connectors/types.ts index 23360e11..c0498c97 100644 --- a/ui/src/views/Connectors/types.ts +++ b/ui/src/views/Connectors/types.ts @@ -1,4 +1,4 @@ -export type ConnectorTypes = 'source' | 'destination' | 'model'; +export type ConnectorTypes = 'source' | 'destination'; export type DatasourceType = { icon: string; @@ -59,7 +59,7 @@ export type CreateConnectorPayload = { connector: { configuration: unknown; name: string; - connector_type: 'source' | 'destination'; + connector_type: ConnectorTypes; connector_name: string; description: string; }; diff --git a/ui/src/views/Models/ModelsForm/DefineModel/DefineSQL/DefineSQL.tsx b/ui/src/views/Models/ModelsForm/DefineModel/DefineSQL/DefineSQL.tsx index a3b25b8e..94f67554 100644 --- a/ui/src/views/Models/ModelsForm/DefineModel/DefineSQL/DefineSQL.tsx +++ b/ui/src/views/Models/ModelsForm/DefineModel/DefineSQL/DefineSQL.tsx @@ -14,7 +14,7 @@ import { useNavigate } from 'react-router-dom'; import { DefineSQLProps } from './types'; import { UpdateModelPayload } from '@/views/Models/ViewModel/types'; import ContentContainer from '@/components/ContentContainer'; -import SourceFormFooter from '@/views/Connectors/Sources/SourcesForm/SourceFormFooter'; +import FormFooter from '@/components/FormFooter'; import { CustomToastStatus } from '@/components/Toast/index'; import useCustomToast from '@/hooks/useCustomToast'; import { format } from 'sql-formatter'; @@ -276,7 +276,7 @@ const DefineSQL = ({ {isUpdateButtonVisible ? ( - ) : ( - {isUpdateButtonVisible ? ( - ) : ( - { - { ); })} - + ); diff --git a/ui/src/views/Settings/Workspace.tsx b/ui/src/views/Settings/Workspace.tsx index fde9ad03..3a9ad6bb 100644 --- a/ui/src/views/Settings/Workspace.tsx +++ b/ui/src/views/Settings/Workspace.tsx @@ -2,7 +2,7 @@ import { Box, Text, Textarea, Input, Divider } from '@chakra-ui/react'; import { useStore } from '@/stores'; import { useQuery } from '@tanstack/react-query'; import { getWorkspaces, CreateWorkspaceResponse, updateWorkspace } from '@/services/settings'; -import SourceFormFooter from '../Connectors/Sources/SourcesForm/SourceFormFooter'; +import FormFooter from '@/components/FormFooter'; import { FormikProps, useFormik } from 'formik'; import useCustomToast from '@/hooks/useCustomToast'; import { CustomToastStatus } from '@/components/Toast/index'; @@ -225,7 +225,7 @@ const Workspace = () => { /> - Date: Mon, 12 Aug 2024 14:48:59 -0400 Subject: [PATCH 25/74] chore(CE): add request response log for Zendesk (#290) --- integrations/Gemfile.lock | 2 +- .../integrations/destination/zendesk/client.rb | 17 +++++++---------- .../lib/multiwoven/integrations/rollout.rb | 2 +- .../destination/zendesk/client_spec.rb | 12 ++++++++++++ 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index f64a5eba..31bde6e7 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.7.5) + multiwoven-integrations (0.7.6) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/destination/zendesk/client.rb b/integrations/lib/multiwoven/integrations/destination/zendesk/client.rb index c708b56f..13818e4c 100644 --- a/integrations/lib/multiwoven/integrations/destination/zendesk/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/zendesk/client.rb @@ -64,21 +64,24 @@ def authenticate_client end def process_records(records, stream) + log_message_array = [] write_success = 0 write_failure = 0 records.each do |record| zendesk_data = prepare_record_data(record, stream.name) plural_stream_name = pluralize_stream_name(stream.name.downcase) + args = [plural_stream_name, @action, zendesk_data] if @action == "create" - @client.send(plural_stream_name).create!(zendesk_data) + response = @client.send(plural_stream_name).create!(zendesk_data) else existing_record = @client.send(plural_stream_name).find(id: record[:id]) - existing_record.update!(zendesk_data) + response = existing_record.update!(zendesk_data) end write_success += 1 + log_message_array << log_request_response("info", args, response) rescue StandardError => e handle_exception(e, { context: "ZENDESK:WRITE:EXCEPTION", @@ -87,9 +90,9 @@ def process_records(records, stream) sync_run_id: @sync_config.sync_run_id }) write_failure += 1 + log_message_array << log_request_response("error", args, e.message) end - - tracking_message(write_success, write_failure) + tracking_message(write_success, write_failure, log_message_array) end def pluralize_stream_name(name) @@ -122,12 +125,6 @@ def prepare_record_data(record, type) def load_catalog read_json(CATALOG_SPEC_PATH) end - - def tracking_message(success, failure) - Multiwoven::Integrations::Protocol::TrackingMessage.new( - success: success, failed: failure - ).to_multiwoven_message - end end end end diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index d6d174fb..36980b12 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.7.5" + VERSION = "0.7.6" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/spec/multiwoven/integrations/destination/zendesk/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/zendesk/client_spec.rb index 4bf63aaa..128b787b 100644 --- a/integrations/spec/multiwoven/integrations/destination/zendesk/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/destination/zendesk/client_spec.rb @@ -160,6 +160,12 @@ result = client.write(sync_config, records) expect(result.tracking.success).to eq(records.size) expect(result.tracking.failed).to eq(0) + log_message = result.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end @@ -172,6 +178,12 @@ result = client.write(sync_config, records) expect(result.tracking.success).to eq(0) expect(result.tracking.failed).to eq(records.size) + log_message = result.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("error") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end end From 5d301e77ca67c34efdd5b207b35108d4d7ef8ee9 Mon Sep 17 00:00:00 2001 From: RafaelOAiSquared <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Mon, 12 Aug 2024 23:57:18 -0400 Subject: [PATCH 26/74] Multiwoven release v0.20.0 (#294) Co-authored-by: github-actions --- release-notes.md | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/release-notes.md b/release-notes.md index 6cece38a..77ad4b32 100644 --- a/release-notes.md +++ b/release-notes.md @@ -2,30 +2,27 @@ All notable changes to this project will be documented in this file. -## [0.19.0] - 2024-08-05 +## [0.20.0] - 2024-08-12 ### 🚀 Features -- *(CE)* Add sync run type column to sync run records table -- *(CE)* Test sync extractor changes (#270) -- *(CE)* Add oracle db destination connector (#277) -- *(CE)* Add oracle db source connector (#274) - -### 🐛 Bug Fixes - -- *(CE)* Lazy init temporal after puma worker boot (#239) -- *(CE)* Logger issue in sync (#279) +- *(CE)* Connector placeholder image ### 🚜 Refactor -- *(CE)* Wrapped main layout with protector +- *(CE)* Remove account verify route +- *(CE)* Connector creation process ### ⚙️ Miscellaneous Tasks -- *(CE)* Verify user mail in signup using devise (#265) -- *(CE)* Update server gem 0.5.2 (#275) -- *(CE)* Update gem 0.6.0 (#278) -- *(CE)* Change name to databricks datawarehouse (#273) -- *(CE)* Update integration gem 0.7.1 (#272) +- *(CE)* Update README (#358) +- *(CE)* Add request response log for Klaviyo +- *(CE)* Add request response log for HTTP +- *(CE)* Add request response log for Airtable +- *(CE)* Add request response log for Slack +- *(CE)* Add request response log for MariaDB (#286) +- *(CE)* Add request response log for Google Sheets (#287) +- *(CE)* Add request response log for Iterable (#289) +- *(CE)* Add request response log for Zendesk (#290) From 13c17f9c81f9c3bac77f74424a756069a4691bf0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 17:02:25 +0530 Subject: [PATCH 27/74] refactor(CE): added disable to fields Co-authored-by: Tushar Selvakumar <54372016+macintushar@users.noreply.github.com> --- ui/src/components/Fields/Fields.tsx | 111 +++++++++++++++ .../AuthViews/SignUpAuthView.tsx | 127 ++++++++++++++++++ ui/src/views/Authentication/types.ts | 53 ++++++++ 3 files changed, 291 insertions(+) create mode 100644 ui/src/components/Fields/Fields.tsx create mode 100644 ui/src/views/Authentication/AuthViews/SignUpAuthView.tsx create mode 100644 ui/src/views/Authentication/types.ts diff --git a/ui/src/components/Fields/Fields.tsx b/ui/src/components/Fields/Fields.tsx new file mode 100644 index 00000000..42eafb7b --- /dev/null +++ b/ui/src/components/Fields/Fields.tsx @@ -0,0 +1,111 @@ +import { FormControl, Input, InputGroup, InputRightElement, Text, Tooltip } from '@chakra-ui/react'; +import HiddenInput from '../HiddenInput'; +import { ErrorMessage, FieldInputProps, FormikErrors, FormikTouched } from 'formik'; +import { FiInfo } from 'react-icons/fi'; + +export type FormFieldProps = { + name: string; + type: string; + placeholder?: string; + tooltipText?: string; + hasTooltip?: boolean; + id?: string; + getFieldProps: ( + nameOrOptions: + | string + | { + name: string; + value?: any; + onChange?: (e: any) => void; + onBlur?: (e: any) => void; + }, + ) => FieldInputProps; + touched: FormikTouched; + errors: FormikErrors; + helperText?: string; + isDisabled?: boolean; +}; + +export const FormField = ({ + name, + type, + getFieldProps, + touched, + errors, + placeholder, + tooltipText, + hasTooltip, + isDisabled, +}: FormFieldProps) => ( + + + {hasTooltip ? ( + + + + + + + + ) : ( + <> + )} + + + + + + +); + +export const PasswordField = ({ + name, + type, + getFieldProps, + touched, + errors, + placeholder, + helperText, + id, +}: FormFieldProps) => ( + + + + {helperText} + + + + + +); diff --git a/ui/src/views/Authentication/AuthViews/SignUpAuthView.tsx b/ui/src/views/Authentication/AuthViews/SignUpAuthView.tsx new file mode 100644 index 00000000..33984c68 --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/SignUpAuthView.tsx @@ -0,0 +1,127 @@ +import { Form, Formik } from 'formik'; +import { Button, HStack, Stack, Text } from '@chakra-ui/react'; +import { FormField, PasswordField } from '@/components/Fields'; +import { Link } from 'react-router-dom'; +import { SignUpAuthViewProps } from '../types'; +import { SignUpSchema } from '@/constants/schemas'; + +export const SignUpAuthView = ({ + handleSubmit, + submitting, + initialValues, + privacyPolicyUrl, + termsOfServiceUrl, + isCompanyNameDisabled, + isEmailDisabled, +}: SignUpAuthViewProps) => ( + <> + + {({ getFieldProps, touched, errors }) => ( + + + + + + + + + + + By creating an account, I agree to the{' '} + + + + Terms + + + + and + + + + Privacy Policy + + + + + + + + + Do you have an account?{' '} + + + + Sign In + + + + + + + )} + + +); diff --git a/ui/src/views/Authentication/types.ts b/ui/src/views/Authentication/types.ts new file mode 100644 index 00000000..3d2185d9 --- /dev/null +++ b/ui/src/views/Authentication/types.ts @@ -0,0 +1,53 @@ +import { ReactNode } from 'react'; + +export type AuthCardProps = { + children: ReactNode; + brandName: string; + logoUrl: string; +}; + +export type SignInAuthViewProps = { + brandName: string; + logoUrl: string; + handleSubmit: (values: any) => void; + submitting: boolean; +}; + +type InitialValues = { + company_name: string; + name: string; + email: string; + password: string; + password_confirmation: string; +}; + +export type SignUpAuthViewProps = { + brandName: string; + logoUrl: string; + handleSubmit: (values: any) => void; + submitting: boolean; + initialValues?: InitialValues; + privacyPolicyUrl: string; + termsOfServiceUrl: string; + isCompanyNameDisabled?: boolean; + isEmailDisabled?: boolean; +}; + +export type ResetPasswordFormPayload = { + password: string; + password_confirmation: string; +}; + +export type ForgotPasswordFormPayload = { + email: string; +}; + +type ChangePasswordProps = { + brandName: string; + logoUrl: string; + handleSubmit: (values: T) => void; + submitting: boolean; +}; + +export type ForgotPasswordAuthViewProps = ChangePasswordProps; +export type ResetPasswordAuthViewProps = ChangePasswordProps; From 0c35af7ea9246ffbbd1ccf6389f1b1ee52f27890 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Mon, 19 Aug 2024 01:49:54 -0400 Subject: [PATCH 28/74] fix(CE): memory bloat issue in sync (#300) Co-authored-by: afthab vp --- server/lib/reverse_etl/extractors/base.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/lib/reverse_etl/extractors/base.rb b/server/lib/reverse_etl/extractors/base.rb index 88743d00..04a018dd 100644 --- a/server/lib/reverse_etl/extractors/base.rb +++ b/server/lib/reverse_etl/extractors/base.rb @@ -84,7 +84,8 @@ def find_or_initialize_sync_record(sync_run, primary_key) # there might be a risk of losing either the update or the create due to these concurrent operations. # we can use ActiveRecord::Base.transaction to prevent such scenarios SyncRecord.find_by(sync_id: sync_run.sync_id, primary_key:) || - sync_run.sync_records.new(sync_id: sync_run.sync_id, primary_key:, created_at: DateTime.current) + SyncRecord.new(sync_id: sync_run.sync_id, sync_run_id: sync_run.id, + primary_key:, created_at: DateTime.current) end def new_record?(sync_record, fingerprint) From 38fbcb94e51b9c3b7a1206840561e3d85325f833 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Mon, 19 Aug 2024 01:56:04 -0400 Subject: [PATCH 29/74] chore(CE): update the query_source response (#305) Co-authored-by: afthab vp --- .../app/controllers/api/v1/connectors_controller.rb | 2 +- .../requests/api/v1/connectors_controller_spec.rb | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/app/controllers/api/v1/connectors_controller.rb b/server/app/controllers/api/v1/connectors_controller.rb index d83c93e3..b26fe46b 100644 --- a/server/app/controllers/api/v1/connectors_controller.rb +++ b/server/app/controllers/api/v1/connectors_controller.rb @@ -95,7 +95,7 @@ def query_source if result.success? @records = result.records.map(&:record).map(&:data) - render json: @records, status: :ok + render json: { data: @records }, status: :ok else render_error( message: result["error"], diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb index 0ed99941..cf970563 100644 --- a/server/spec/requests/api/v1/connectors_controller_spec.rb +++ b/server/spec/requests/api/v1/connectors_controller_spec.rb @@ -438,8 +438,8 @@ post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:ok) - response_hash = JSON.parse(response.body) - expect(response_hash).to eq([record1.record.data, record2.record.data]) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) end it "returns success status for a valid query for member role" do @@ -449,8 +449,8 @@ post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:ok) - response_hash = JSON.parse(response.body) - expect(response_hash).to eq([record1.record.data, record2.record.data]) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) end it "returns success status for a valid query for viewer role" do @@ -460,8 +460,8 @@ post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:ok) - response_hash = JSON.parse(response.body) - expect(response_hash).to eq([record1.record.data, record2.record.data]) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) end it "returns failure status for a invalid query" do From 46f1840948ecf889d097065dccba5c0025af381b Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Mon, 19 Aug 2024 01:58:06 -0400 Subject: [PATCH 30/74] feat(CE): destination/microsoft excel (#314) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- integrations/Gemfile.lock | 2 +- integrations/lib/multiwoven/integrations.rb | 1 + .../integrations/core/base_connector.rb | 3 +- .../multiwoven/integrations/core/constants.rb | 12 + .../core/destination_connector.rb | 8 + .../integrations/core/http_client.rb | 3 +- .../destination/airtable/client.rb | 8 - .../facebook_custom_audience/client.rb | 8 - .../destination/microsoft_excel/client.rb | 194 +++++++++++++++ .../microsoft_excel/config/catalog.json | 7 + .../microsoft_excel/config/meta.json | 15 ++ .../microsoft_excel/config/spec.json | 19 ++ .../destination/microsoft_excel/icon.svg | 18 ++ .../lib/multiwoven/integrations/rollout.rb | 3 +- .../integrations/core/http_client_spec.rb | 7 + .../microsoft_excel/client_spec.rb | 224 ++++++++++++++++++ 16 files changed, 512 insertions(+), 20 deletions(-) create mode 100644 integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb create mode 100644 integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json create mode 100644 integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/meta.json create mode 100644 integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/spec.json create mode 100644 integrations/lib/multiwoven/integrations/destination/microsoft_excel/icon.svg create mode 100644 integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 71ed2acb..ced201cb 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.7.9) + multiwoven-integrations (0.8.0) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb index 4f80fab4..fbcc5537 100644 --- a/integrations/lib/multiwoven/integrations.rb +++ b/integrations/lib/multiwoven/integrations.rb @@ -81,6 +81,7 @@ require_relative "integrations/destination/maria_db/client" require_relative "integrations/destination/databricks_lakehouse/client" require_relative "integrations/destination/oracle_db/client" +require_relative "integrations/destination/microsoft_excel/client" module Multiwoven module Integrations diff --git a/integrations/lib/multiwoven/integrations/core/base_connector.rb b/integrations/lib/multiwoven/integrations/core/base_connector.rb index b594f517..672e474c 100644 --- a/integrations/lib/multiwoven/integrations/core/base_connector.rb +++ b/integrations/lib/multiwoven/integrations/core/base_connector.rb @@ -63,7 +63,8 @@ def success_status end def failure_status(error) - ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message + message = error&.message || "failed" + ConnectionStatus.new(status: ConnectionStatusType["failed"], message: message).to_multiwoven_message end end end diff --git a/integrations/lib/multiwoven/integrations/core/constants.rb b/integrations/lib/multiwoven/integrations/core/constants.rb index 45b106f0..9ce33d04 100644 --- a/integrations/lib/multiwoven/integrations/core/constants.rb +++ b/integrations/lib/multiwoven/integrations/core/constants.rb @@ -36,6 +36,17 @@ module Constants AIRTABLE_BASES_ENDPOINT = "https://api.airtable.com/v0/meta/bases" AIRTABLE_GET_BASE_SCHEMA_ENDPOINT = "https://api.airtable.com/v0/meta/bases/{baseId}/tables" + MS_EXCEL_AUTH_ENDPOINT = "https://graph.microsoft.com/v1.0/me" + MS_EXCEL_TABLE_ROW_WRITE_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/"\ + "workbook/worksheets/%s/tables/%s/rows" + MS_EXCEL_TABLE_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/workbook/"\ + "worksheets/sheet/tables?$select=name" + MS_EXCEL_FILES_API = "https://graph.microsoft.com/v1.0/drives/%s/root/children" + MS_EXCEL_WORKSHEETS_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/"\ + "workbook/worksheets" + MS_EXCEL_SHEET_RANGE_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/"\ + "workbook/worksheets/%s/range(address='A1:Z1')/usedRange?$select=values" + AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"] AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"] @@ -44,6 +55,7 @@ module Constants HTTP_POST = "POST" HTTP_PUT = "PUT" HTTP_DELETE = "DELETE" + HTTP_PATCH = "PATCH" # google sheets GOOGLE_SHEETS_SCOPE = "https://www.googleapis.com/auth/drive" diff --git a/integrations/lib/multiwoven/integrations/core/destination_connector.rb b/integrations/lib/multiwoven/integrations/core/destination_connector.rb index f0595a31..da327412 100644 --- a/integrations/lib/multiwoven/integrations/core/destination_connector.rb +++ b/integrations/lib/multiwoven/integrations/core/destination_connector.rb @@ -15,6 +15,14 @@ def tracking_message(success, failure, log_message_array) success: success, failed: failure, logs: log_message_array ).to_multiwoven_message end + + def auth_headers(access_token) + { + "Accept" => "application/json", + "Authorization" => "Bearer #{access_token}", + "Content-Type" => "application/json" + } + end end end end diff --git a/integrations/lib/multiwoven/integrations/core/http_client.rb b/integrations/lib/multiwoven/integrations/core/http_client.rb index c0aa40c9..b119c256 100644 --- a/integrations/lib/multiwoven/integrations/core/http_client.rb +++ b/integrations/lib/multiwoven/integrations/core/http_client.rb @@ -19,13 +19,14 @@ def build_request(method, uri, payload, headers) when Constants::HTTP_GET then Net::HTTP::Get when Constants::HTTP_POST then Net::HTTP::Post when Constants::HTTP_PUT then Net::HTTP::Put + when Constants::HTTP_PATCH then Net::HTTP::Patch when Constants::HTTP_DELETE then Net::HTTP::Delete else raise ArgumentError, "Unsupported HTTP method: #{method}" end request = request_class.new(uri) headers.each { |key, value| request[key] = value } - request.body = payload.to_json if payload && %w[POST PUT].include?(method.upcase) + request.body = payload.to_json if payload && %w[POST PUT PATCH].include?(method.upcase) request end end diff --git a/integrations/lib/multiwoven/integrations/destination/airtable/client.rb b/integrations/lib/multiwoven/integrations/destination/airtable/client.rb index 8d89ff88..d2242fbe 100644 --- a/integrations/lib/multiwoven/integrations/destination/airtable/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/airtable/client.rb @@ -109,14 +109,6 @@ def create_payload(records) } end - def auth_headers(access_token) - { - "Accept" => "application/json", - "Authorization" => "Bearer #{access_token}", - "Content-Type" => "application/json" - } - end - def base_id_exists?(bases, base_id) return if extract_bases(bases).any? { |base| base["id"] == base_id } diff --git a/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb b/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb index 685cb54f..e2a69e1c 100644 --- a/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb @@ -110,14 +110,6 @@ def extract_schema_and_data(records, json_schema) [schema, data] end - def auth_headers(access_token) - { - "Accept" => "application/json", - "Authorization" => "Bearer #{access_token}", - "Content-Type" => "application/json" - } - end - def ad_account_exists?(response, ad_account_id) return if extract_data(response).any? { |ad_account| ad_account["id"] == "act_#{ad_account_id}" } diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb new file mode 100644 index 00000000..7e5805c2 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb @@ -0,0 +1,194 @@ +# frozen_string_literal: true + +module Multiwoven::Integrations::Destination + module MicrosoftExcel + include Multiwoven::Integrations::Core + class Client < DestinationConnector + prepend Multiwoven::Integrations::Core::RateLimiter + def check_connection(connection_config) + connection_config = connection_config.with_indifferent_access + drive_id = create_connection(connection_config) + if drive_id + success_status + else + failure_status(nil) + end + rescue StandardError => e + handle_exception(e, { + context: "MICROSOFT:EXCEL:CHECK_CONNECTION:EXCEPTION", + type: "error" + }) + failure_status(e) + end + + def discover(connection_config) + catalog_json = read_json(CATALOG_SPEC_PATH) + connection_config = connection_config.with_indifferent_access + token = connection_config[:token] + drive_id = create_connection(connection_config) + records = get_file(token, drive_id) + records.each do |record| + file_id = record[:id] + record[:worksheets] = get_file_data(token, drive_id, file_id) + end + catalog = Catalog.new(streams: create_streams(records, catalog_json)) + catalog.to_multiwoven_message + rescue StandardError => e + handle_exception(e, { + context: "MICROSOFT:EXCEL:DISCOVER:EXCEPTION", + type: "error" + }) + end + + def write(sync_config, records, _action = "destination_insert") + connection_config = sync_config.destination.connection_specification.with_indifferent_access + token = connection_config[:token] + file_name = sync_config.stream.name.split(", ").first + sheet_name = sync_config.stream.name.split(", ").last + drive_id = create_connection(connection_config) + excel_files = get_file(token, drive_id) + worksheet = excel_files.find { |file| file[:name] == file_name } + item_id = worksheet[:id] + + table = get_table(token, drive_id, item_id) + write_url = format(MS_EXCEL_TABLE_ROW_WRITE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name, + table_name: table["name"]) + payload = { values: records.map(&:values) } + process_write_request(write_url, payload, token, sync_config) + end + + private + + def create_connection(connection_config) + token = connection_config[:token] + response = Multiwoven::Integrations::Core::HttpClient.request( + MS_EXCEL_AUTH_ENDPOINT, + HTTP_GET, + headers: auth_headers(token) + ) + JSON.parse(response.body)["id"] + end + + def get_table(token, drive_id, item_id) + table_url = format(MS_EXCEL_TABLE_API, drive_id: drive_id, item_id: item_id) + response = Multiwoven::Integrations::Core::HttpClient.request( + table_url, + HTTP_GET, + headers: auth_headers(token) + ) + JSON.parse(response.body)["value"].first + end + + def get_file(token, drive_id) + url = format(MS_EXCEL_FILES_API, drive_id: drive_id) + response = Multiwoven::Integrations::Core::HttpClient.request( + url, + HTTP_GET, + headers: auth_headers(token) + ) + files = JSON.parse(response.body)["value"] + excel_files = files.select { |file| file["name"].match(/\.(xlsx|xls|xlsm)$/) } + excel_files.map { |file| { name: file["name"], id: file["id"] } } + end + + def get_all_sheets(token, drive_id, item_id) + base_url = format(MS_EXCEL_WORKSHEETS_API, drive_id: drive_id, item_id: item_id) + worksheet_response = Multiwoven::Integrations::Core::HttpClient.request( + base_url, + HTTP_GET, + headers: auth_headers(token) + ) + JSON.parse(worksheet_response.body)["value"] + end + + def get_file_data(token, drive_id, item_id) + result = [] + worksheets_data = get_all_sheets(token, drive_id, item_id) + worksheets_data.each do |sheet| + sheet_name = sheet["name"] + sheet_url = format(MS_EXCEL_SHEET_RANGE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name) + + sheet_response = Multiwoven::Integrations::Core::HttpClient.request( + sheet_url, + HTTP_GET, + headers: auth_headers(token) + ) + sheets_data = JSON.parse(sheet_response.body) + result << { + sheet_name: sheet_name, + column_names: sheets_data["values"].first + } + end + result + end + + def create_streams(records, catalog_json) + group_by_table(records).flat_map do |_, record| + record.map do |_, r| + Multiwoven::Integrations::Protocol::Stream.new( + name: r[:workbook], + action: StreamAction["fetch"], + json_schema: convert_to_json_schema(r[:columns]), + request_rate_limit: catalog_json["request_rate_limit"] || 60, + request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute", + request_rate_concurrency: catalog_json["request_rate_concurrency"] || 1 + ) + end + end + end + + def group_by_table(records) + result = {} + + records.each_with_index do |entries, entries_index| + entries[:worksheets].each_with_index do |sheet, entry_index| + workbook_sheet = "#{entries[:name]}, #{sheet[:sheet_name]}" + columns = sheet[:column_names].map do |column_name| + column_name = "empty column" if column_name.empty? + { + column_name: column_name, + data_type: "String", + is_nullable: true + } + end + result[entries_index] ||= {} + result[entries_index][entry_index] = { workbook: workbook_sheet, columns: columns } + end + end + result + end + + def process_write_request(write_url, payload, token, sync_config) + write_success = 0 + write_failure = 0 + log_message_array = [] + + begin + response = Multiwoven::Integrations::Core::HttpClient.request( + write_url, + HTTP_POST, + payload: payload, + headers: auth_headers(token) + ) + if success?(response) + write_success += 1 + else + write_failure += 1 + end + log_message_array << log_request_response("info", [HTTP_POST, write_url, payload], response) + rescue StandardError => e + handle_exception(e, { + context: "MICROSOFT:EXCEL:RECORD:WRITE:EXCEPTION", + type: "error", + sync_id: sync_config.sync_id, + sync_run_id: sync_config.sync_run_id + }) + write_failure += 1 + log_message_array << log_request_response("error", [HTTP_POST, write_url, payload], e.message) + end + + tracking_message(write_success, write_failure, log_message_array) + end + end + end +end diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json new file mode 100644 index 00000000..9a178849 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json @@ -0,0 +1,7 @@ +{ + "request_rate_limit": 6000, + "request_rate_limit_unit": "minute", + "request_rate_concurrency": 10, + "streams": [] +} + diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/meta.json b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/meta.json new file mode 100644 index 00000000..49455eac --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/meta.json @@ -0,0 +1,15 @@ +{ + "data": { + "name": "MicrosoftExcel", + "title": "Microsoft Excel", + "connector_type": "destination", + "category": "Database", + "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/microsoft_excel", + "github_issue_label": "destination-microsoft-excel", + "icon": "icon.svg", + "license": "MIT", + "release_stage": "alpha", + "support_level": "community", + "tags": ["language:ruby", "multiwoven"] + } +} diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/spec.json b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/spec.json new file mode 100644 index 00000000..d3fe4a99 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/spec.json @@ -0,0 +1,19 @@ +{ + "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/microsoft_excel", + "stream_type": "dynamic", + "connector_query_type": "raw_sql", + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Microsoft Excel", + "type": "object", + "required": ["token"], + "properties": { + "token": { + "description": "Token from Microsoft Graph.", + "type": "string", + "title": "Token", + "order": 0 + } + } + } +} \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/icon.svg b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/icon.svg new file mode 100644 index 00000000..3ec1e490 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/icon.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 62c6d503..769c6986 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.7.9" + VERSION = "0.8.0" ENABLED_SOURCES = %w[ Snowflake @@ -36,6 +36,7 @@ module Integrations MariaDB DatabricksLakehouse Oracle + MicrosoftExcel ].freeze end end diff --git a/integrations/spec/multiwoven/integrations/core/http_client_spec.rb b/integrations/spec/multiwoven/integrations/core/http_client_spec.rb index e90e7847..b2d2d328 100644 --- a/integrations/spec/multiwoven/integrations/core/http_client_spec.rb +++ b/integrations/spec/multiwoven/integrations/core/http_client_spec.rb @@ -39,6 +39,13 @@ module Integrations::Core end end + context "when making a PATCH request" do + it "creates a PATCH request" do + described_class.request(url, "PATCH", headers: headers) + expect(a_request(:patch, url).with(headers: headers)).to have_been_made.once + end + end + context "with an unsupported HTTP method" do it "raises an ArgumentError" do expect { described_class.request(url, "INVALID", headers: headers) }.to raise_error(ArgumentError) diff --git a/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb new file mode 100644 index 00000000..bf856fbd --- /dev/null +++ b/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb @@ -0,0 +1,224 @@ +# frozen_string_literal: true + +RSpec.describe Multiwoven::Integrations::Destination::MicrosoftExcel::Client do + include WebMock::API + + before(:each) do + WebMock.disable_net_connect!(allow_localhost: true) + end + + let(:client) { described_class.new } + let(:connection_config) do + { + token: "test" + } + end + let(:sync_config_json) do + { + source: { + name: "Sample Source Connector", + type: "source", + connection_specification: { + private_api_key: "test_api_key" + } + }, + destination: { + name: "Databricks", + type: "destination", + connection_specification: connection_config + }, + model: { + name: "ExampleModel", + query: "SELECT col1, col2, col3 FROM test_table_1", + query_type: "raw_sql", + primary_key: "col1" + }, + sync_mode: "incremental", + destination_sync_mode: "insert", + stream: { + name: "test_table.xlsx", + action: "create", + json_schema: {}, + supported_sync_modes: %w[incremental], + request_rate_limit: 4, + rate_limit_unit_seconds: 1 + } + } + end + + let(:response_body) { { "id" => "DRIVE1" }.to_json } + let(:successful_update_response_body) { { "values" => [["400", "4.4", "Fourth"]] }.to_json } + let(:failed_update_response_body) { { "values" => [["400", "4.4", "Fourth"]] }.to_json } + + describe "#check_connection" do + context "when the connection is successful" do + it "returns a succeeded connection status" do + stub_request(:get, "https://graph.microsoft.com/v1.0/me") + .to_return(status: 200, body: response_body, headers: {}) + + allow(client).to receive(:create_connection).and_return("DRIVE1") + + message = client.check_connection(sync_config_json[:destination][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("succeeded") + expect(result.message).to be_nil + end + end + + context "when the connection fails" do + it "returns a failed connection status with an error message" do + allow(client).to receive(:create_connection).and_raise(StandardError, "Connection failed") + message = client.check_connection(sync_config_json[:destination][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("failed") + expect(result.message).to include("Connection failed") + end + end + end + + describe "#discover" do + it "discovers schema successfully" do + stub_request(:get, "https://graph.microsoft.com/v1.0/me") + .to_return(status: 200, body: response_body, headers: {}) + stub_request(:get, "https://graph.microsoft.com/v1.0/drives/DRIVE1/root/children") + .to_return( + status: 200, + body: { + "value" => [ + { "id" => "file1_id", "name" => "test_file.xlsx" } + ] + }.to_json, + headers: {} + ) + allow(client).to receive(:get_all_sheets).and_return([ + { "name" => "Sheet1" } + ]) + stub_request(:get, "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/Sheet1/"\ + "range(address='A1:Z1')/usedRange?$select=values") + .to_return( + status: 200, + body: { + "values" => [%w[col1 col2 col3]] + }.to_json, + headers: {} + ) + + message = client.discover(connection_config) + catalog = message.catalog + expect(catalog).to be_a(Multiwoven::Integrations::Protocol::Catalog) + expect(catalog.streams.first.request_rate_limit).to eql(6000) + expect(catalog.streams.first.request_rate_limit_unit).to eql("minute") + expect(catalog.streams.first.request_rate_concurrency).to eql(10) + expect(catalog.streams.count).to eql(1) + expect(catalog.streams[0].supported_sync_modes).to eql(%w[incremental]) + end + end + + describe "#write" do + context "when the write operation is successful" do + it "increments the success count" do + stub_request(:get, "https://graph.microsoft.com/v1.0/me") + .to_return(status: 200, body: response_body, headers: {}) + + stub_request(:get, "https://graph.microsoft.com/v1.0/drives/DRIVE1/root/children") + .to_return( + status: 200, + body: { + "value" => [ + { "id" => "file1_id", "name" => "test_table.xlsx" } + ] + }.to_json, + headers: {} + ) + + stub_request(:get, "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/sheet/"\ + "tables?$select=name") + .to_return( + status: 200, + body: { + "value" => [ + { "name" => "Table1" } + ] + }.to_json, + headers: {} + ) + + stub_request(:post, "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/"\ + "test_table.xlsx/tables/Table1/rows") + .to_return(status: 201, body: successful_update_response_body, headers: {}) + + sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json) + records = [{ "Col1" => 400, "Col2" => 4.4, "Col3" => "Fourth" }] + + message = client.write(sync_config, records) + tracker = message.tracking + + expect(tracker.success).to eq(records.count) + expect(tracker.failed).to eq(0) + log_message = message.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") + end + end + + context "when the write operation fails" do + it "increments the failure count" do + stub_request(:get, "https://graph.microsoft.com/v1.0/me") + .to_return(status: 200, body: response_body, headers: {}) + + stub_request(:get, "https://graph.microsoft.com/v1.0/drives/DRIVE1/root/children") + .to_return( + status: 200, + body: { + "value" => [ + { "id" => "file1_id", "name" => "test_table.xlsx" } + ] + }.to_json, + headers: {} + ) + + stub_request(:get, "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/sheet/"\ + "tables?$select=name") + .to_return( + status: 200, + body: { + "value" => [ + { "name" => "Table1" } + ] + }.to_json, + headers: {} + ) + + stub_request(:post, + "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/"\ + "test_table.xlsx/tables/Table1/rows") + .to_return(status: 400, body: failed_update_response_body, headers: {}) + + sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json) + records = [{ "Col1" => 400, "Col2" => 4.4, "Col3" => "Fourth" }] + + message = client.write(sync_config, records) + tracker = message.tracking + expect(tracker.failed).to eq(records.count) + expect(tracker.success).to eq(0) + log_message = message.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") + end + end + end + + describe "#meta_data" do + # change this to rollout validation for all connector rolling out + it "client class_name and meta name is same" do + meta_name = client.class.to_s.split("::")[-2] + expect(client.send(:meta_data)[:data][:name]).to eq(meta_name) + end + end +end From c01818cdafadba9eff50983868128aa827253304 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Mon, 19 Aug 2024 04:13:14 -0400 Subject: [PATCH 31/74] chore(CE): fix oci8 version (#313) Co-authored-by: afthab vp --- integrations/Gemfile | 2 +- integrations/Gemfile.lock | 4 ++-- integrations/lib/multiwoven/integrations/rollout.rb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integrations/Gemfile b/integrations/Gemfile index 1b82fbae..757d6502 100644 --- a/integrations/Gemfile +++ b/integrations/Gemfile @@ -67,7 +67,7 @@ gem "mysql2" gem "aws-sdk-sts" -gem "ruby-oci8" +gem "ruby-oci8", "~> 2.2.12" group :development, :test do gem "simplecov", require: false diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index ced201cb..eaf02de0 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.8.0) + multiwoven-integrations (0.8.1) activesupport async-websocket aws-sdk-athena @@ -360,7 +360,7 @@ DEPENDENCIES rspec (~> 3.0) rubocop (~> 1.21) ruby-limiter - ruby-oci8 + ruby-oci8 (~> 2.2.12) ruby-odbc! rubyzip sequel diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 769c6986..7c91233c 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.8.0" + VERSION = "0.8.1" ENABLED_SOURCES = %w[ Snowflake From 5539104bbc39fce2a991210fe6f46d373e030404 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Mon, 19 Aug 2024 04:16:17 -0400 Subject: [PATCH 32/74] chore(CE): update user read permission (#320) Co-authored-by: afthab vp --- ...40813084446_update_user_read_permission.rb | 40 +++++++++++++++++++ server/db/data_schema.rb | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 server/db/data/20240813084446_update_user_read_permission.rb diff --git a/server/db/data/20240813084446_update_user_read_permission.rb b/server/db/data/20240813084446_update_user_read_permission.rb new file mode 100644 index 00000000..245fa719 --- /dev/null +++ b/server/db/data/20240813084446_update_user_read_permission.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class UpdateUserReadPermission < ActiveRecord::Migration[7.1] + def change + member_role = Role.find_by(role_name: "Member") + viewer_role = Role.find_by(role_name: "Viewer") + + member_role&.update!( + policies: { + permissions: { + connector_definition: { create: true, read: true, update: true, delete: true }, + connector: { create: true, read: true, update: true, delete: true }, + model: { create: true, read: true, update: true, delete: true }, + report: { create: true, read: true, update: true, delete: true }, + sync_record: { create: true, read: true, update: true, delete: true }, + sync_run: { create: true, read: true, update: true, delete: true }, + sync: { create: true, read: true, update: true, delete: true }, + user: { create: false, read: true, update: false, delete: false }, + workspace: { create: false, read: true, update: false, delete: false } + } + } + ) + + viewer_role&.update!( + policies: { + permissions: { + connector_definition: { create: false, read: true, update: false, delete: false }, + connector: { create: false, read: true, update: false, delete: false }, + model: { create: false, read: true, update: false, delete: false }, + report: { create: false, read: true, update: false, delete: false }, + sync_record: { create: false, read: true, update: false, delete: false }, + sync_run: { create: false, read: true, update: false, delete: false }, + sync: { create: false, read: true, update: false, delete: false }, + user: { create: false, read: true, update: false, delete: false }, + workspace: { create: false, read: true, update: false, delete: false } + } + } + ) + end +end diff --git a/server/db/data_schema.rb b/server/db/data_schema.rb index f783f44d..9c182f0d 100644 --- a/server/db/data_schema.rb +++ b/server/db/data_schema.rb @@ -1,3 +1,3 @@ # frozen_string_literal: true -DataMigrate::Data.define(version: 20_240_702_095_541) +DataMigrate::Data.define(version: 20_240_813_084_446) From 85c6ce353762811e79dcf4eb79099fe31e3762c1 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Mon, 19 Aug 2024 04:46:27 -0400 Subject: [PATCH 33/74] chore(CE): update server gem (#312) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- server/Gemfile | 2 +- server/Gemfile.lock | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/server/Gemfile b/server/Gemfile index 5dc78e2c..cd52ce17 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -13,7 +13,7 @@ gem "interactor", "~> 3.0" gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git" -gem "multiwoven-integrations", "~> 0.7.1" +gem "multiwoven-integrations", "~> 0.8.1" gem "temporal-ruby", github: "coinbase/temporal-ruby" diff --git a/server/Gemfile.lock b/server/Gemfile.lock index fd46ce10..eb77ef58 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -108,8 +108,8 @@ GEM appsignal (3.7.5) rack ast (2.4.2) - async (2.14.2) - console (~> 1.25, >= 1.25.2) + async (2.15.3) + console (~> 1.26) fiber-annotation io-event (~> 1.6, >= 1.6.5) async-http (0.69.0) @@ -121,7 +121,7 @@ GEM protocol-http1 (~> 0.19) protocol-http2 (~> 0.18) traces (>= 0.10) - async-pool (0.7.0) + async-pool (0.8.0) async (>= 1.25) async-websocket (0.28.0) async-http (~> 0.54) @@ -1769,7 +1769,7 @@ GEM fiber-annotation (0.2.0) fiber-local (1.1.0) fiber-storage - fiber-storage (0.1.2) + fiber-storage (1.0.0) fugit (1.10.1) et-orbi (~> 1, >= 1.2.7) raabro (~> 1.4) @@ -1781,7 +1781,7 @@ GEM gli (2.21.5) globalid (1.2.1) activesupport (>= 6.1) - google-apis-bigquery_v2 (0.73.0) + google-apis-bigquery_v2 (0.75.0) google-apis-core (>= 0.15.0, < 2.a) google-apis-core (0.15.1) addressable (~> 2.5, >= 2.5.1) @@ -1791,7 +1791,7 @@ GEM mutex_m representable (~> 3.0) retriable (>= 2.0, < 4.a) - google-apis-sheets_v4 (0.33.0) + google-apis-sheets_v4 (0.34.0) google-apis-core (>= 0.15.0, < 2.a) google-cloud-bigquery (1.50.0) bigdecimal (~> 3.0) @@ -1801,7 +1801,7 @@ GEM google-cloud-core (~> 1.6) googleauth (~> 1.9) mini_mime (~> 1.0) - google-cloud-core (1.7.0) + google-cloud-core (1.7.1) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) google-cloud-env (2.1.1) @@ -1898,7 +1898,7 @@ GEM msgpack (1.7.2) multi_json (1.15.0) multipart-post (2.4.1) - multiwoven-integrations (0.7.1) + multiwoven-integrations (0.8.1) activesupport async-websocket aws-sdk-athena @@ -2096,11 +2096,11 @@ GEM rubocop (>= 1.33.0, < 2.0) rubocop-ast (>= 1.30.0, < 2.0) ruby-limiter (2.3.0) - ruby-oci8 (2.2.12) + ruby-oci8 (2.2.14) ruby-progressbar (1.13.0) ruby2_keywords (0.0.5) rubyzip (2.3.2) - sequel (5.83.0) + sequel (5.83.1) bigdecimal shoulda-matchers (5.3.0) activesupport (>= 5.2.0) @@ -2123,9 +2123,9 @@ GEM faraday-multipart gli hashie - sorbet-runtime (0.5.11504) + sorbet-runtime (0.5.11518) stringio (3.1.0) - stripe (12.4.0) + stripe (12.5.0) strong_migrations (1.8.0) activerecord (>= 5.2) thor (1.3.0) @@ -2197,7 +2197,7 @@ DEPENDENCIES jwt kaminari liquid - multiwoven-integrations (~> 0.7.1) + multiwoven-integrations (~> 0.8.1) mysql2 newrelic_rpm parallel From 28cc8dc68359a985cf9c7ee10824aed6a190fb24 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Mon, 19 Aug 2024 04:58:14 -0400 Subject: [PATCH 34/74] chore(CE): update connector name (#317) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- integrations/Gemfile.lock | 2 +- .../integrations/destination/airtable/config/meta.json | 2 +- .../destination/databricks_lakehouse/config/meta.json | 2 +- .../destination/facebook_custom_audience/config/meta.json | 2 +- .../multiwoven/integrations/destination/http/config/meta.json | 2 +- .../integrations/destination/oracle_db/config/meta.json | 2 +- integrations/lib/multiwoven/integrations/rollout.rb | 2 +- .../multiwoven/integrations/source/databricks/config/meta.json | 2 +- .../multiwoven/integrations/source/oracle_db/config/meta.json | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index eaf02de0..ef94c94e 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.8.1) + multiwoven-integrations (0.8.2) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/destination/airtable/config/meta.json b/integrations/lib/multiwoven/integrations/destination/airtable/config/meta.json index c8e11bfc..6676114c 100644 --- a/integrations/lib/multiwoven/integrations/destination/airtable/config/meta.json +++ b/integrations/lib/multiwoven/integrations/destination/airtable/config/meta.json @@ -1,7 +1,7 @@ { "data": { "name": "Airtable", - "title": "airtable", + "title": "Airtable", "connector_type": "destination", "category": "Productivity Tools", "documentation_url": "https://docs.multiwoven.com/destinations/productivity-tools/airtable", diff --git a/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json b/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json index 34e949ea..4d9fdb72 100644 --- a/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json +++ b/integrations/lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json @@ -1,7 +1,7 @@ { "data": { "name": "DatabricksLakehouse", - "title": "Databricks Datawarehouse", + "title": "Databricks Data Warehouse", "connector_type": "destination", "category": "Database", "documentation_url": "https://docs.multiwoven.com/destinations/databricks_lakehouse", diff --git a/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json b/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json index 52304399..a38ea456 100644 --- a/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json +++ b/integrations/lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json @@ -3,7 +3,7 @@ "name": "FacebookCustomAudience", "title": "Facebook Custom Audiences", "connector_type": "destination", - "category": "Adtech", + "category": "Ad-Tech", "documentation_url": "https://docs.mutliwoven.com", "github_issue_label": "destination-facebook", "icon": "icon.svg", diff --git a/integrations/lib/multiwoven/integrations/destination/http/config/meta.json b/integrations/lib/multiwoven/integrations/destination/http/config/meta.json index 9e83ea14..da67d965 100644 --- a/integrations/lib/multiwoven/integrations/destination/http/config/meta.json +++ b/integrations/lib/multiwoven/integrations/destination/http/config/meta.json @@ -3,7 +3,7 @@ "name": "Http", "title": "http", "connector_type": "destination", - "category": "Http", + "category": "HTTP", "documentation_url": "https://docs.multiwoven.com/destinations/http", "github_issue_label": "destination-http", "icon": "icon.svg", diff --git a/integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json index 5662698c..023cb80b 100644 --- a/integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json +++ b/integrations/lib/multiwoven/integrations/destination/oracle_db/config/meta.json @@ -1,7 +1,7 @@ { "data": { "name": "Oracle", - "title": "Oracle", + "title": "Oracle DB", "connector_type": "destination", "category": "Database", "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle", diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 7c91233c..9fc5944e 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.8.1" + VERSION = "0.8.2" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/lib/multiwoven/integrations/source/databricks/config/meta.json b/integrations/lib/multiwoven/integrations/source/databricks/config/meta.json index d484108e..a3efc671 100644 --- a/integrations/lib/multiwoven/integrations/source/databricks/config/meta.json +++ b/integrations/lib/multiwoven/integrations/source/databricks/config/meta.json @@ -1,7 +1,7 @@ { "data": { "name": "Databricks", - "title": "Databricks", + "title": "Databricks Data Warehouse", "connector_type": "source", "category": "Data Warehouse", "documentation_url": "https://docs.mutliwoven.com", diff --git a/integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json b/integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json index 833d36f0..71605dc7 100644 --- a/integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json +++ b/integrations/lib/multiwoven/integrations/source/oracle_db/config/meta.json @@ -1,7 +1,7 @@ { "data": { "name": "Oracle", - "title": "Oracle", + "title": "Oracle DB", "connector_type": "source", "category": "Database", "documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle", From 467b8bf3bd671ef9c0f8681f0f0c97e1be456608 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Mon, 19 Aug 2024 06:18:16 -0400 Subject: [PATCH 35/74] fix(CE): fix discover and table url (#316) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- integrations/Gemfile.lock | 2 +- .../lib/multiwoven/integrations/core/constants.rb | 3 +-- .../destination/microsoft_excel/client.rb | 14 +++++++++----- .../microsoft_excel/config/catalog.json | 2 +- .../lib/multiwoven/integrations/rollout.rb | 2 +- .../destination/microsoft_excel/client_spec.rb | 8 ++++---- 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index ef94c94e..79d31293 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.8.2) + multiwoven-integrations (0.8.3) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/core/constants.rb b/integrations/lib/multiwoven/integrations/core/constants.rb index 9ce33d04..7c8e5248 100644 --- a/integrations/lib/multiwoven/integrations/core/constants.rb +++ b/integrations/lib/multiwoven/integrations/core/constants.rb @@ -40,13 +40,12 @@ module Constants MS_EXCEL_TABLE_ROW_WRITE_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/"\ "workbook/worksheets/%s/tables/%s/rows" MS_EXCEL_TABLE_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/workbook/"\ - "worksheets/sheet/tables?$select=name" + "worksheets/%s/tables?$select=name" MS_EXCEL_FILES_API = "https://graph.microsoft.com/v1.0/drives/%s/root/children" MS_EXCEL_WORKSHEETS_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/"\ "workbook/worksheets" MS_EXCEL_SHEET_RANGE_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/"\ "workbook/worksheets/%s/range(address='A1:Z1')/usedRange?$select=values" - AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"] AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"] diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb index 7e5805c2..cc120a68 100644 --- a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/client.rb @@ -49,8 +49,7 @@ def write(sync_config, records, _action = "destination_insert") excel_files = get_file(token, drive_id) worksheet = excel_files.find { |file| file[:name] == file_name } item_id = worksheet[:id] - - table = get_table(token, drive_id, item_id) + table = get_table(token, drive_id, item_id, sheet_name) write_url = format(MS_EXCEL_TABLE_ROW_WRITE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name, table_name: table["name"]) payload = { values: records.map(&:values) } @@ -69,8 +68,8 @@ def create_connection(connection_config) JSON.parse(response.body)["id"] end - def get_table(token, drive_id, item_id) - table_url = format(MS_EXCEL_TABLE_API, drive_id: drive_id, item_id: item_id) + def get_table(token, drive_id, item_id, sheet_name) + table_url = format(MS_EXCEL_TABLE_API, drive_id: drive_id, item_id: item_id, sheet_name: sheet_name) response = Multiwoven::Integrations::Core::HttpClient.request( table_url, HTTP_GET, @@ -114,9 +113,14 @@ def get_file_data(token, drive_id, item_id) headers: auth_headers(token) ) sheets_data = JSON.parse(sheet_response.body) + column_names = if sheets_data.key?("error") + ["Column A"] + else + sheets_data["values"].first + end result << { sheet_name: sheet_name, - column_names: sheets_data["values"].first + column_names: column_names } end result diff --git a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json index 9a178849..ace9ffee 100644 --- a/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json +++ b/integrations/lib/multiwoven/integrations/destination/microsoft_excel/config/catalog.json @@ -1,7 +1,7 @@ { "request_rate_limit": 6000, "request_rate_limit_unit": "minute", - "request_rate_concurrency": 10, + "request_rate_concurrency": 1, "streams": [] } diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 9fc5944e..25b5219f 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.8.2" + VERSION = "0.8.3" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb index bf856fbd..46a5280d 100644 --- a/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/destination/microsoft_excel/client_spec.rb @@ -36,7 +36,7 @@ sync_mode: "incremental", destination_sync_mode: "insert", stream: { - name: "test_table.xlsx", + name: "test_table.xlsx, sheet", action: "create", json_schema: {}, supported_sync_modes: %w[incremental], @@ -108,7 +108,7 @@ expect(catalog).to be_a(Multiwoven::Integrations::Protocol::Catalog) expect(catalog.streams.first.request_rate_limit).to eql(6000) expect(catalog.streams.first.request_rate_limit_unit).to eql("minute") - expect(catalog.streams.first.request_rate_concurrency).to eql(10) + expect(catalog.streams.first.request_rate_concurrency).to eql(1) expect(catalog.streams.count).to eql(1) expect(catalog.streams[0].supported_sync_modes).to eql(%w[incremental]) end @@ -144,7 +144,7 @@ ) stub_request(:post, "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/"\ - "test_table.xlsx/tables/Table1/rows") + "sheet/tables/Table1/rows") .to_return(status: 201, body: successful_update_response_body, headers: {}) sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json) @@ -194,7 +194,7 @@ stub_request(:post, "https://graph.microsoft.com/v1.0/drives/DRIVE1/items/file1_id/workbook/worksheets/"\ - "test_table.xlsx/tables/Table1/rows") + "sheet/tables/Table1/rows") .to_return(status: 400, body: failed_update_response_body, headers: {}) sync_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json) From 51eae2171460ecb5c803acacecd67fa43ba309e7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 20 Aug 2024 10:09:41 +0530 Subject: [PATCH 36/74] feat(CE): Enable/disable sync (#321) Co-authored-by: Pablo Rivera Bengoechea --- server/app/contracts/sync_contracts.rb | 7 ++++ .../api/v1/schedule_syncs_controller.rb | 8 +++++ .../controllers/api/v1/syncs_controller.rb | 13 ++++++- server/app/models/sync.rb | 22 ++++++------ server/app/policies/sync_policy.rb | 4 +++ server/config/routes.rb | 3 ++ server/spec/contracts/sync_contracts_spec.rb | 18 ++++++++++ server/spec/models/sync_spec.rb | 18 ++++++++++ .../api/v1/schedule_syncs_controller_spec.rb | 13 +++++++ .../requests/api/v1/syncs_controller_spec.rb | 34 +++++++++++++++++++ 10 files changed, 127 insertions(+), 13 deletions(-) diff --git a/server/app/contracts/sync_contracts.rb b/server/app/contracts/sync_contracts.rb index f7eb414b..d267fecc 100644 --- a/server/app/contracts/sync_contracts.rb +++ b/server/app/contracts/sync_contracts.rb @@ -146,6 +146,13 @@ class Update < Dry::Validation::Contract end end + class Enable < Dry::Validation::Contract + params do + required(:id).filled(:integer) + required(:enable).filled(:bool) + end + end + class Destroy < Dry::Validation::Contract params do required(:id).filled(:integer) diff --git a/server/app/controllers/api/v1/schedule_syncs_controller.rb b/server/app/controllers/api/v1/schedule_syncs_controller.rb index 80fec1c2..7944f3a2 100644 --- a/server/app/controllers/api/v1/schedule_syncs_controller.rb +++ b/server/app/controllers/api/v1/schedule_syncs_controller.rb @@ -5,6 +5,7 @@ module V1 class ScheduleSyncsController < ApplicationController include Syncs before_action :set_sync + before_action :validate_sync_status before_action :validate_sync_schedule_type def create @@ -36,6 +37,13 @@ def set_sync render_error(message: "Sync not found", status: :not_found) unless @sync end + def validate_sync_status + return unless @sync.disabled? + + render_error(message: "Sync is disabled", + status: :failed_dependency) + end + def validate_sync_schedule_type return if @sync.schedule_type == "manual" diff --git a/server/app/controllers/api/v1/syncs_controller.rb b/server/app/controllers/api/v1/syncs_controller.rb index 902c3861..9667f162 100644 --- a/server/app/controllers/api/v1/syncs_controller.rb +++ b/server/app/controllers/api/v1/syncs_controller.rb @@ -4,7 +4,7 @@ module Api module V1 class SyncsController < ApplicationController include Syncs - before_action :set_sync, only: %i[show update destroy] + before_action :set_sync, only: %i[show update enable destroy] before_action :modify_sync_params, only: %i[create update] after_action :event_logger @@ -84,6 +84,17 @@ def configurations end end + def enable + authorize current_workspace, policy_class: SyncPolicy + params[:enable] ? @sync.enable : @sync.disable + if @sync.save + render json: @sync, status: :ok + else + render_error(message: "Sync update failed", status: :unprocessable_entity, + details: format_errors(result.sync)) + end + end + private def set_sync diff --git a/server/app/models/sync.rb b/server/app/models/sync.rb index a72cb482..f9af1d59 100644 --- a/server/app/models/sync.rb +++ b/server/app/models/sync.rb @@ -119,14 +119,18 @@ def schedule_cron_expression def schedule_sync? (new_record? || saved_change_to_sync_interval? || saved_change_to_sync_interval_unit || - saved_change_to_cron_expression?) && !manual? + saved_change_to_cron_expression? || saved_change_to_status?) && !manual? end def schedule_sync - Temporal.start_workflow( - Workflows::ScheduleSyncWorkflow, - id - ) + if saved_change_to_status? && status == "disabled" + Temporal.start_workflow(Workflows::TerminateWorkflow, id, options: { workflow_id: "terminate-#{id}" }) + elsif new_record? || (saved_change_to_status? && status == "pending") + Temporal.start_workflow( + Workflows::ScheduleSyncWorkflow, + id + ) + end rescue StandardError => e Utils::ExceptionReporter.report(e, { sync_id: id @@ -137,13 +141,7 @@ def schedule_sync def perform_post_discard_sync sync_runs.discard_all terminate_workflow_id = "terminate-#{workflow_id}" - Temporal.start_workflow( - Workflows::TerminateWorkflow, - workflow_id, - options: { - workflow_id: terminate_workflow_id - } - ) + Temporal.start_workflow(Workflows::TerminateWorkflow, workflow_id, options: { workflow_id: terminate_workflow_id }) rescue StandardError => e Utils::ExceptionReporter.report(e, { sync_id: id diff --git a/server/app/policies/sync_policy.rb b/server/app/policies/sync_policy.rb index 5422ff23..5370b5d5 100644 --- a/server/app/policies/sync_policy.rb +++ b/server/app/policies/sync_policy.rb @@ -24,4 +24,8 @@ def destroy? def configurations? permitted?(:read, :sync) end + + def enable? + permitted?(:update, :sync) + end end diff --git a/server/config/routes.rb b/server/config/routes.rb index bf6ecbd6..b1b0aeaa 100644 --- a/server/config/routes.rb +++ b/server/config/routes.rb @@ -30,6 +30,9 @@ collection do get :configurations end + member do + patch :enable + end resources :sync_runs, only: %i[index show] do resources :sync_records, only: [:index] end diff --git a/server/spec/contracts/sync_contracts_spec.rb b/server/spec/contracts/sync_contracts_spec.rb index 34570c83..9d9560b7 100644 --- a/server/spec/contracts/sync_contracts_spec.rb +++ b/server/spec/contracts/sync_contracts_spec.rb @@ -200,6 +200,24 @@ end end + describe SyncContracts::Enable do + subject(:contract) { described_class.new } + + context "with valid parameters " do + let(:valid_inputs) { { id: 1, enable: true } } + let(:invalid_inputs) { { id: 1, enable: "disabled" } } + + it "passes validation" do + expect(contract.call(valid_inputs)).to be_success + end + + it "fails validation" do + result = contract.call(invalid_inputs) + expect(result.errors[:enable]).to include("must be boolean") + end + end + end + describe SyncContracts::Destroy do subject(:contract) { described_class.new } diff --git a/server/spec/models/sync_spec.rb b/server/spec/models/sync_spec.rb index c0172458..58c99f1b 100644 --- a/server/spec/models/sync_spec.rb +++ b/server/spec/models/sync_spec.rb @@ -156,6 +156,24 @@ ) end + it "terminate a sync workflow if sync is disabled and schedule sync workflow if sync is enabled" do + sync.disable + sync.save(validate: false) + expect(Temporal).to have_received(:start_workflow).with( + Workflows::TerminateWorkflow, + sync.id, + options: { + workflow_id: "terminate-#{sync.id}" + } + ) + sync.enable + sync.save(validate: false) + expect(Temporal).to have_received(:start_workflow).with( + Workflows::ScheduleSyncWorkflow, + sync.id + ) + end + it "does not schedule a sync workflow if sync interval does not change" do sync.primary_key = "primary_key" sync.save diff --git a/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb b/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb index a9131a4b..f4ef0689 100644 --- a/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb +++ b/server/spec/requests/api/v1/schedule_syncs_controller_spec.rb @@ -101,6 +101,19 @@ expect(result["errors"][0]["detail"]).to eq(error_message) end end + + context "when sync id is correct but it is disabled" do + it "returns failure" do + sync.update(status: "disabled") + error_message = "Sync is disabled" + post "/api/v1/schedule_syncs", params: request_body.to_json, headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + + result = JSON.parse(response.body) + expect(result["errors"][0]["status"]).to eq(424) + expect(result["errors"][0]["detail"]).to eq(error_message) + end + end end describe "DELETE /api/v1/syncs/id" do diff --git a/server/spec/requests/api/v1/syncs_controller_spec.rb b/server/spec/requests/api/v1/syncs_controller_spec.rb index 41d813c9..a7699da6 100644 --- a/server/spec/requests/api/v1/syncs_controller_spec.rb +++ b/server/spec/requests/api/v1/syncs_controller_spec.rb @@ -444,6 +444,40 @@ end end + describe "PATCH /api/v1/syncs/enable - Enable/Disable sync" do + let(:request_body) do + { enable: true } + end + + context "when it is an unauthenticated user for update sync" do + it "returns unauthorized" do + patch "/api/v1/syncs/#{syncs.first.id}/enable" + expect(response).to have_http_status(:unauthorized) + end + end + + context "when it is an authenticated user and update sync" do + it "disables and enables a sync and returns success" do + request_body[:enable] = false + patch "/api/v1/syncs/#{syncs.first.id}/enable", params: request_body.to_json, headers: + { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:ok) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash.dig(:data, :id)).to be_present + expect(response_hash.dig(:data, :id)).to eq(syncs.first.id.to_s) + expect(response_hash.dig(:data, :attributes, :status)).to eq("disabled") + request_body[:enable] = true + patch "/api/v1/syncs/#{syncs.first.id}/enable", params: request_body.to_json, headers: + { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:ok) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash.dig(:data, :id)).to be_present + expect(response_hash.dig(:data, :id)).to eq(syncs.first.id.to_s) + expect(response_hash.dig(:data, :attributes, :status)).to eq("pending") + end + end + end + describe "POST /api/v1/syncs - Create sync" do let(:request_body) do { From 5eecc25074338853c01a12aa804b490e308fcc82 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 20 Aug 2024 10:10:29 +0530 Subject: [PATCH 37/74] chore(CE): add request response log for Hubspot (#323) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- integrations/Gemfile.lock | 2 +- .../integrations/destination/hubspot/client.rb | 16 +++++++--------- .../lib/multiwoven/integrations/rollout.rb | 2 +- .../destination/hubspot/client_spec.rb | 12 ++++++++++++ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 79d31293..e6d685ea 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.8.3) + multiwoven-integrations (0.8.4) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/destination/hubspot/client.rb b/integrations/lib/multiwoven/integrations/destination/hubspot/client.rb index 57a2e5e7..68ddfae8 100644 --- a/integrations/lib/multiwoven/integrations/destination/hubspot/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/hubspot/client.rb @@ -55,13 +55,15 @@ def initialize_client(config) end def process_records(records, stream) + log_message_array = [] write_success = 0 write_failure = 0 properties = stream.json_schema.with_indifferent_access[:properties] records.each do |record_object| record = extract_data(record_object, properties) - send_data_to_hubspot(stream.name, record) + request, response = *send_data_to_hubspot(stream.name, record) write_success += 1 + log_message_array << log_request_response("info", request, response) rescue StandardError => e handle_exception(e, { context: "HUBSPOT:CRM:WRITE:EXCEPTION", @@ -70,15 +72,17 @@ def process_records(records, stream) sync_run_id: @sync_config.sync_run_id }) write_failure += 1 + log_message_array << log_request_response("error", request, e.message) end - tracking_message(write_success, write_failure) + tracking_message(write_success, write_failure, log_message_array) end def send_data_to_hubspot(stream_name, record = {}) args = build_args(@action, stream_name, record) hubspot_stream = @client.crm.send(stream_name) hubspot_data = { simple_public_object_input_for_create: args } - hubspot_stream.basic_api.send(@action, hubspot_data) + response = hubspot_stream.basic_api.send(@action, hubspot_data) + [args, response] end def build_args(action, stream_name, record) @@ -108,12 +112,6 @@ def load_catalog read_json(CATALOG_SPEC_PATH) end - def tracking_message(success, failure) - Multiwoven::Integrations::Protocol::TrackingMessage.new( - success: success, failed: failure - ).to_multiwoven_message - end - def log_debug(message) Multiwoven::Integrations::Service.logger.debug(message) end diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 25b5219f..9518790e 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.8.3" + VERSION = "0.8.4" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/spec/multiwoven/integrations/destination/hubspot/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/hubspot/client_spec.rb index 8ce0cf06..a728a384 100644 --- a/integrations/spec/multiwoven/integrations/destination/hubspot/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/destination/hubspot/client_spec.rb @@ -119,6 +119,12 @@ expect(response.tracking.success).to eq(records.size) expect(response.tracking.failed).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end @@ -133,6 +139,12 @@ expect(response.tracking.failed).to eq(records.size) expect(response.tracking.success).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("error") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end end From 35350f319546536a1277b76d42f4321e17dc725f Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Tue, 20 Aug 2024 02:27:52 -0400 Subject: [PATCH 38/74] Multiwoven release v0.21.0 (#324) Co-authored-by: github-actions --- release-notes.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/release-notes.md b/release-notes.md index 77ad4b32..fdaa3e73 100644 --- a/release-notes.md +++ b/release-notes.md @@ -2,27 +2,27 @@ All notable changes to this project will be documented in this file. -## [0.20.0] - 2024-08-12 +## [0.21.0] - 2024-08-19 ### 🚀 Features -- *(CE)* Connector placeholder image +- *(CE)* Destination/microsoft excel (#314) + +### 🐛 Bug Fixes + +- *(CE)* Memory bloat issue in sync (#300) +- *(CE)* Fix discover and table url (#316) ### 🚜 Refactor -- *(CE)* Remove account verify route -- *(CE)* Connector creation process +- *(CE)* Added disable to fields ### ⚙️ Miscellaneous Tasks -- *(CE)* Update README (#358) -- *(CE)* Add request response log for Klaviyo -- *(CE)* Add request response log for HTTP -- *(CE)* Add request response log for Airtable -- *(CE)* Add request response log for Slack -- *(CE)* Add request response log for MariaDB (#286) -- *(CE)* Add request response log for Google Sheets (#287) -- *(CE)* Add request response log for Iterable (#289) -- *(CE)* Add request response log for Zendesk (#290) +- *(CE)* Update the query_source response (#305) +- *(CE)* Fix oci8 version (#313) +- *(CE)* Update user read permission (#320) +- *(CE)* Update server gem (#312) +- *(CE)* Update connector name (#317) From 9537945322e8661bcb8792eda77e27e97b3b60e9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 10:20:12 +0530 Subject: [PATCH 39/74] chore(CE): add request response log for Stripe (#328) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- integrations/Gemfile.lock | 2 +- .../integrations/destination/stripe/client.rb | 16 +++++++--------- .../lib/multiwoven/integrations/rollout.rb | 2 +- .../destination/stripe/client_spec.rb | 12 ++++++++++++ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index e6d685ea..7005eb63 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.8.4) + multiwoven-integrations (0.8.5) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/destination/stripe/client.rb b/integrations/lib/multiwoven/integrations/destination/stripe/client.rb index 8e72c3a8..5dc24bec 100644 --- a/integrations/lib/multiwoven/integrations/destination/stripe/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/stripe/client.rb @@ -52,14 +52,17 @@ def initialize_client(config) end def process_records(records, stream) + log_message_array = [] write_success = 0 write_failure = 0 properties = stream.json_schema[:properties] records.each do |record_object| record = extract_data(record_object, properties) - klass = @client.const_get(stream.name) - klass.send(@action, record) + args = [stream.name, "Id", record] + klass = @client.const_get(stream.name) + response = klass.send(@action, record) write_success += 1 + log_message_array << log_request_response("info", args, response) rescue StandardError => e handle_exception(e, { context: "STRIPE:CRM:WRITE:EXCEPTION", @@ -68,8 +71,9 @@ def process_records(records, stream) sync_run_id: @sync_config.sync_run_id }) write_failure += 1 + log_message_array << log_request_response("error", args, e.message) end - tracking_message(write_success, write_failure) + tracking_message(write_success, write_failure, log_message_array) end def authenticate_client @@ -80,12 +84,6 @@ def load_catalog read_json(CATALOG_SPEC_PATH) end - def tracking_message(success, failure) - Multiwoven::Integrations::Protocol::TrackingMessage.new( - success: success, failed: failure - ).to_multiwoven_message - end - def log_debug(message) Multiwoven::Integrations::Service.logger.debug(message) end diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 9518790e..b36e538c 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.8.4" + VERSION = "0.8.5" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/spec/multiwoven/integrations/destination/stripe/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/stripe/client_spec.rb index b8591523..8bf82abf 100644 --- a/integrations/spec/multiwoven/integrations/destination/stripe/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/destination/stripe/client_spec.rb @@ -98,6 +98,12 @@ expect(response.tracking.success).to eq(records.size) expect(response.tracking.failed).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end @@ -111,6 +117,12 @@ expect(response.tracking.failed).to eq(records.size) expect(response.tracking.success).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("error") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end end From 042f7ed1eeecc2dbe2cd52cc0d6efd6c12e33b44 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Wed, 21 Aug 2024 02:08:09 -0400 Subject: [PATCH 40/74] refactor(CE): changed condition to render toast (#315) --- ui/src/components/Fields/index.ts | 1 + ui/src/constants/schemas.ts | 42 +++++++++++++++++++++++++++++++ ui/src/hooks/useErrorToast.tsx | 29 ++++++++++++--------- 3 files changed, 60 insertions(+), 12 deletions(-) create mode 100644 ui/src/components/Fields/index.ts create mode 100644 ui/src/constants/schemas.ts diff --git a/ui/src/components/Fields/index.ts b/ui/src/components/Fields/index.ts new file mode 100644 index 00000000..23cf74ee --- /dev/null +++ b/ui/src/components/Fields/index.ts @@ -0,0 +1 @@ +export { FormField, PasswordField } from './Fields'; diff --git a/ui/src/constants/schemas.ts b/ui/src/constants/schemas.ts new file mode 100644 index 00000000..0b5a503f --- /dev/null +++ b/ui/src/constants/schemas.ts @@ -0,0 +1,42 @@ +import * as Yup from 'yup'; + +const email = Yup.string() + .email('Please enter a valid email address') + .required('Email is required'); + +const password = Yup.string() + .min(8, 'Password must be at least 8 characters') + .max(128, 'Password cannot be more than 128 characters') + .matches(/[A-Z]/, 'Password must contain at least one uppercase letter') + .matches(/[a-z]/, 'Password must contain at least one lowercase letter') + .matches(/\d/, 'Password must contain at least one digit') + .matches(/[@$!%*?&]/, 'Password must contain at least one special character') + .required('Password is required'); + +const password_confirmation = Yup.string() + .oneOf([Yup.ref('password'), ''], 'Passwords must match') + .required('Confirm Password is required'); + +export const SignInSchema = Yup.object().shape({ + email: email, + password: Yup.string() + .min(8, 'Password must be at least 8 characters') + .required('Password is required'), +}); + +export const SignUpSchema = Yup.object().shape({ + company_name: Yup.string().required('Company name is required'), + name: Yup.string().required('Name is required'), + email: email, + password: password, + password_confirmation: password_confirmation, +}); + +export const ForgotPasswordSchema = Yup.object().shape({ + email: email, +}); + +export const ResetPasswordSchema = Yup.object().shape({ + password: password, + password_confirmation: password_confirmation, +}); diff --git a/ui/src/hooks/useErrorToast.tsx b/ui/src/hooks/useErrorToast.tsx index 94474929..355ced0e 100644 --- a/ui/src/hooks/useErrorToast.tsx +++ b/ui/src/hooks/useErrorToast.tsx @@ -1,21 +1,26 @@ -import { useCallback, useEffect } from 'react'; +import { useCallback } from 'react'; import useCustomToast from '@/hooks/useCustomToast'; import { CustomToastStatus } from '@/components/Toast'; import { ErrorResponse } from '@/services/common'; -export const useErrorToast = (isError: boolean, data: any, isFetched: boolean, message: string) => { +export const useErrorToast = () => { const showToast = useCustomToast(); - useEffect(() => { - if (isError || (!data && isFetched)) { - showToast({ - title: `Error: ${message}`, - description: message, - status: CustomToastStatus.Error, - position: 'bottom-right', - }); - } - }, [isError, data, isFetched, showToast, message]); + const showErrorToast = useCallback( + (message: string, isError: boolean, data: any, isFetched: boolean) => { + if (isError && !data && isFetched) { + showToast({ + status: CustomToastStatus.Warning, + title: message, + position: 'bottom-right', + isClosable: true, + }); + } + }, + [showToast], + ); + + return showErrorToast; }; export const useAPIErrorsToast = () => { From b5d9728bad75b164f4d234cc49675e119ac69fd0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 13:40:16 +0530 Subject: [PATCH 41/74] chore(CE): add request response log for SalesforceCrm (#326) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- integrations/Gemfile.lock | 2 +- .../destination/salesforce_crm/client.rb | 16 +++++++--------- .../lib/multiwoven/integrations/rollout.rb | 2 +- .../destination/salesforce_crm/client_spec.rb | 12 ++++++++++++ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 7005eb63..9c0286e2 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.8.5) + multiwoven-integrations (0.8.6) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/destination/salesforce_crm/client.rb b/integrations/lib/multiwoven/integrations/destination/salesforce_crm/client.rb index c15558f6..6bdf0914 100644 --- a/integrations/lib/multiwoven/integrations/destination/salesforce_crm/client.rb +++ b/integrations/lib/multiwoven/integrations/destination/salesforce_crm/client.rb @@ -59,13 +59,15 @@ def initialize_client(config) end def process_records(records, stream) + log_message_array = [] write_success = 0 write_failure = 0 properties = stream.json_schema[:properties] records.each do |record_object| record = extract_data(record_object, properties) - process_record(stream, record) + request, response = *process_record(stream, record) write_success += 1 + log_message_array << log_request_response("info", request, response) rescue StandardError => e # TODO: add sync_id and sync_run_id to the logs handle_exception(e, { @@ -75,8 +77,9 @@ def process_records(records, stream) sync_run_id: @sync_config.sync_run_id }) write_failure += 1 + log_message_array << log_request_response("error", request, e.message) end - tracking_message(write_success, write_failure) + tracking_message(write_success, write_failure, log_message_array) end def process_record(stream, record) @@ -86,7 +89,8 @@ def process_record(stream, record) def send_data_to_salesforce(stream_name, record = {}) method_name = "#{@action}!" args = build_args(@action, stream_name, record) - @client.send(method_name, *args) + response = @client.send(method_name, *args) + [args, response] end def build_args(action, stream_name, record) @@ -116,12 +120,6 @@ def load_catalog read_json(CATALOG_SPEC_PATH) end - def tracking_message(success, failure) - Multiwoven::Integrations::Protocol::TrackingMessage.new( - success: success, failed: failure - ).to_multiwoven_message - end - def log_debug(message) Multiwoven::Integrations::Service.logger.debug(message) end diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index b36e538c..d1135b35 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.8.5" + VERSION = "0.8.6" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/spec/multiwoven/integrations/destination/salesforce_crm/client_spec.rb b/integrations/spec/multiwoven/integrations/destination/salesforce_crm/client_spec.rb index d1f0838b..61d136f2 100644 --- a/integrations/spec/multiwoven/integrations/destination/salesforce_crm/client_spec.rb +++ b/integrations/spec/multiwoven/integrations/destination/salesforce_crm/client_spec.rb @@ -103,6 +103,12 @@ expect(response.tracking.success).to eq(records.size) expect(response.tracking.failed).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("info") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end @@ -117,6 +123,12 @@ expect(response.tracking.failed).to eq(records.size) expect(response.tracking.success).to eq(0) + log_message = response.tracking.logs.first + expect(log_message).to be_a(Multiwoven::Integrations::Protocol::LogMessage) + expect(log_message.level).to eql("error") + + expect(log_message.message).to include("request") + expect(log_message.message).to include("response") end end end From 271f297b44089fb94115bae958eaac09ea9dbd14 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 14:02:39 +0530 Subject: [PATCH 42/74] feat(CE): Databricks AI Model connector (#325) Co-authored-by: afthab vp --- integrations/Gemfile.lock | 2 +- integrations/lib/multiwoven/integrations.rb | 1 + .../integrations/core/base_connector.rb | 8 + .../multiwoven/integrations/core/constants.rb | 4 + .../core/destination_connector.rb | 8 - .../integrations/protocol/protocol.rb | 6 +- .../lib/multiwoven/integrations/rollout.rb | 3 +- .../source/databrics_model/client.rb | 87 ++++++++ .../databrics_model/config/catalog.json | 6 + .../source/databrics_model/config/meta.json | 16 ++ .../source/databrics_model/config/spec.json | 47 +++++ .../source/databrics_model/icon.svg | 19 ++ .../integrations/protocol/protocol_spec.rb | 43 +++- .../source/databricks_model/client_spec.rb | 193 ++++++++++++++++++ 14 files changed, 423 insertions(+), 20 deletions(-) create mode 100644 integrations/lib/multiwoven/integrations/source/databrics_model/client.rb create mode 100644 integrations/lib/multiwoven/integrations/source/databrics_model/config/catalog.json create mode 100644 integrations/lib/multiwoven/integrations/source/databrics_model/config/meta.json create mode 100644 integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json create mode 100644 integrations/lib/multiwoven/integrations/source/databrics_model/icon.svg create mode 100644 integrations/spec/multiwoven/integrations/source/databricks_model/client_spec.rb diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 9c0286e2..fcad59f0 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.8.6) + multiwoven-integrations (0.9.0) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb index fbcc5537..02f2e262 100644 --- a/integrations/lib/multiwoven/integrations.rb +++ b/integrations/lib/multiwoven/integrations.rb @@ -62,6 +62,7 @@ require_relative "integrations/source/amazon_s3/client" require_relative "integrations/source/maria_db/client" require_relative "integrations/source/oracle_db/client" +require_relative "integrations/source/databrics_model/client" # Destination require_relative "integrations/destination/klaviyo/client" diff --git a/integrations/lib/multiwoven/integrations/core/base_connector.rb b/integrations/lib/multiwoven/integrations/core/base_connector.rb index 672e474c..68230b51 100644 --- a/integrations/lib/multiwoven/integrations/core/base_connector.rb +++ b/integrations/lib/multiwoven/integrations/core/base_connector.rb @@ -66,6 +66,14 @@ def failure_status(error) message = error&.message || "failed" ConnectionStatus.new(status: ConnectionStatusType["failed"], message: message).to_multiwoven_message end + + def auth_headers(access_token) + { + "Accept" => "application/json", + "Authorization" => "Bearer #{access_token}", + "Content-Type" => "application/json" + } + end end end end diff --git a/integrations/lib/multiwoven/integrations/core/constants.rb b/integrations/lib/multiwoven/integrations/core/constants.rb index 7c8e5248..95e369dc 100644 --- a/integrations/lib/multiwoven/integrations/core/constants.rb +++ b/integrations/lib/multiwoven/integrations/core/constants.rb @@ -46,6 +46,10 @@ module Constants "workbook/worksheets" MS_EXCEL_SHEET_RANGE_API = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/"\ "workbook/worksheets/%s/range(address='A1:Z1')/usedRange?$select=values" + + DATABRICKS_HEALTH_URL = "https://%s/api/2.0/serving-endpoints/%s" + DATABRICKS_SERVING_URL = "https://%s/serving-endpoints/%s/invocations" + AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"] AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"] diff --git a/integrations/lib/multiwoven/integrations/core/destination_connector.rb b/integrations/lib/multiwoven/integrations/core/destination_connector.rb index da327412..f0595a31 100644 --- a/integrations/lib/multiwoven/integrations/core/destination_connector.rb +++ b/integrations/lib/multiwoven/integrations/core/destination_connector.rb @@ -15,14 +15,6 @@ def tracking_message(success, failure, log_message_array) success: success, failed: failure, logs: log_message_array ).to_multiwoven_message end - - def auth_headers(access_token) - { - "Accept" => "application/json", - "Authorization" => "Bearer #{access_token}", - "Content-Type" => "application/json" - } - end end end end diff --git a/integrations/lib/multiwoven/integrations/protocol/protocol.rb b/integrations/lib/multiwoven/integrations/protocol/protocol.rb index 9b4c5efd..22fcdfbb 100644 --- a/integrations/lib/multiwoven/integrations/protocol/protocol.rb +++ b/integrations/lib/multiwoven/integrations/protocol/protocol.rb @@ -10,10 +10,10 @@ module Types SyncStatus = Types::String.enum("started", "running", "complete", "incomplete") DestinationSyncMode = Types::String.enum("insert", "upsert") ConnectorType = Types::String.enum("source", "destination") - ConnectorQueryType = Types::String.enum("raw_sql", "soql") - ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector") + ConnectorQueryType = Types::String.enum("raw_sql", "soql", "ai_ml") + ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector", "ai_ml") ConnectionStatusType = Types::String.enum("succeeded", "failed") - StreamType = Types::String.enum("static", "dynamic") + StreamType = Types::String.enum("static", "dynamic", "user_defined") StreamAction = Types::String.enum("fetch", "create", "update", "delete") MultiwovenMessageType = Types::String.enum( "record", "log", "connector_spec", diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index d1135b35..2088482b 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.8.6" + VERSION = "0.9.0" ENABLED_SOURCES = %w[ Snowflake @@ -16,6 +16,7 @@ module Integrations AmazonS3 MariaDB Oracle + DatabricksModel ].freeze ENABLED_DESTINATIONS = %w[ diff --git a/integrations/lib/multiwoven/integrations/source/databrics_model/client.rb b/integrations/lib/multiwoven/integrations/source/databrics_model/client.rb new file mode 100644 index 00000000..63db472d --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/databrics_model/client.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Multiwoven::Integrations::Source + module DatabricksModel + include Multiwoven::Integrations::Core + class Client < SourceConnector + def check_connection(connection_config) + connection_config = connection_config.with_indifferent_access + url = build_url(DATABRICKS_HEALTH_URL, connection_config) + response = Multiwoven::Integrations::Core::HttpClient.request( + url, + HTTP_GET, + headers: auth_headers(connection_config[:token]) + ) + if success?(response) + success_status + else + failure_status(nil) + end + rescue StandardError => e + ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message + end + + def discover(_connection_config = nil) + catalog_json = read_json(CATALOG_SPEC_PATH) + catalog = build_catalog(catalog_json) + catalog.to_multiwoven_message + rescue StandardError => e + handle_exception(e, { + context: "DATABRICKS MODEL:DISCOVER:EXCEPTION", + type: "error" + }) + end + + def read(sync_config) + connection_config = sync_config.source.connection_specification + connection_config = connection_config.with_indifferent_access + # The server checks the ConnectorQueryType. + # If it's "ai_ml," the server calculates the payload and passes it as a query in the sync config model protocol. + # This query is then sent to the AI/ML model. + payload = JSON.parse(sync_config.model.query) + run_model(connection_config, payload) + rescue StandardError => e + handle_exception(e, { + context: "DATABRICKS MODEL:READ:EXCEPTION", + type: "error" + }) + end + + private + + def run_model(connection_config, payload) + connection_config = connection_config.with_indifferent_access + + url = build_url(DATABRICKS_SERVING_URL, connection_config) + token = connection_config[:token] + response = send_request(url, token, payload) + process_response(response) + rescue StandardError => e + handle_exception(e, context: "DATABRICKS MODEL:RUN_MODEL:EXCEPTION", type: "error") + end + + def process_response(response) + if success?(response) + data = JSON.parse(response.body) + [RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message] + else + create_log_message("DATABRICKS MODEL:RUN_MODEL", "error", "request failed") + end + end + + def build_url(url, connection_config) + format(url, databricks_host: connection_config[:databricks_host], + endpoint_name: connection_config[:endpoint_name]) + end + + def send_request(url, token, payload) + Multiwoven::Integrations::Core::HttpClient.request( + url, + HTTP_POST, + payload: payload, + headers: auth_headers(token) + ) + end + end + end +end diff --git a/integrations/lib/multiwoven/integrations/source/databrics_model/config/catalog.json b/integrations/lib/multiwoven/integrations/source/databrics_model/config/catalog.json new file mode 100644 index 00000000..dacb788b --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/databrics_model/config/catalog.json @@ -0,0 +1,6 @@ +{ + "request_rate_limit": 600, + "request_rate_limit_unit": "minute", + "request_rate_concurrency": 10, + "streams": [] +} diff --git a/integrations/lib/multiwoven/integrations/source/databrics_model/config/meta.json b/integrations/lib/multiwoven/integrations/source/databrics_model/config/meta.json new file mode 100644 index 00000000..078e2f18 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/databrics_model/config/meta.json @@ -0,0 +1,16 @@ +{ + "data": { + "name": "DatabricksModel", + "title": "Databricks Model", + "connector_type": "source", + "category": "AI Model", + "documentation_url": "https://docs.mutliwoven.com", + "github_issue_label": "source-databricks-foundation", + "icon": "icon.svg", + "license": "MIT", + "release_stage": "alpha", + "support_level": "community", + "tags": ["language:ruby", "multiwoven"] + } + } + \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json b/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json new file mode 100644 index 00000000..ef7ca0c9 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json @@ -0,0 +1,47 @@ +{ + "documentation_url": "https://docs.multiwoven.com/integrations/sources/databricks", + "stream_type": "user_defined", + "connector_query_type": "ai_ml", + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Databricks Model", + "type": "object", + "required": ["databricks_host", "token", "model_name"], + "properties": { + "databricks_host": { + "title": "databricks_host", + "description": "databricks_host", + "type": "string", + "examples": ["app.databricks.com"], + "order": 0 + }, + "token": { + "title": "Databricks Token", + "description": "personal access token", + "type": "string", + "multiwoven_secret": true, + "order": 1 + }, + "endpoint": { + "title": "Endpoint name", + "description": "Endpoint name", + "examples": ["databricks-dbrx-instruct"], + "type": "string", + "order": 2 + }, + "request_format": { + "title": "Request Format", + "description": "Sample Request Format", + "type": "string", + "order": 3 + }, + "response_format": { + "title": "Response Format", + "description": "Sample Response Format", + "type": "string", + "order": 4 + } + } + } + } + \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/source/databrics_model/icon.svg b/integrations/lib/multiwoven/integrations/source/databrics_model/icon.svg new file mode 100644 index 00000000..553c3e41 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/databrics_model/icon.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/integrations/spec/multiwoven/integrations/protocol/protocol_spec.rb b/integrations/spec/multiwoven/integrations/protocol/protocol_spec.rb index bcb9e589..f0d2c640 100644 --- a/integrations/spec/multiwoven/integrations/protocol/protocol_spec.rb +++ b/integrations/spec/multiwoven/integrations/protocol/protocol_spec.rb @@ -61,6 +61,18 @@ module Integrations::Protocol expect(instance.supported_destination_sync_modes).to eq(["insert"]) expect(instance.connector_query_type).to eq("soql") end + + it "creates an instance from JSON connector_query_type ai_ml" do + json_data[:connector_query_type] = "ai_ml" + json_data[:stream_type] = "user_defined" + instance = ConnectorSpecification.from_json(json_data.to_json) + expect(instance).to be_a(ConnectorSpecification) + expect(instance.connection_specification).to eq(key: "value") + expect(instance.supports_normalization).to eq(true) + expect(instance.supported_destination_sync_modes).to eq(["insert"]) + expect(instance.connector_query_type).to eq("ai_ml") + expect(instance.stream_type).to eq("user_defined") + end end describe "#to_multiwoven_message" do @@ -366,25 +378,42 @@ module Integrations::Protocol model = Model.new(name: "Test", query: "SELECT * FROM table", query_type: "table_selector", primary_key: "id") expect(ModelQueryType.values).to include(model.query_type) end + + it "has a query_type 'ai_ml'" do + model = Model.new(name: "Test", query: "SELECT * FROM table", query_type: "ai_ml", primary_key: "id") + expect("ai_ml").to include(model.query_type) + end end end RSpec.describe Connector do context ".from_json" do - it "creates an instance from JSON" do - json_data = { - "name": "example_connector", - "type": "source", - "connection_specification": { "key": "value" } - }.to_json + json_data = { + "name": "example_connector", + "type": "source", + "connection_specification": { "key": "value" }, + "query_type": "raw_sql" + } - connector = Connector.from_json(json_data) + it "creates an instance from JSON" do + connector = Connector.from_json(json_data.to_json) expect(connector).to be_a(described_class) expect(connector.name).to eq("example_connector") expect(connector.type).to eq("source") expect(connector.query_type).to eq("raw_sql") expect(connector.connection_specification).to eq(key: "value") end + + it "creates an instance from JSON connector_query_type ai_ml" do + json_data[:query_type] = "ai_ml" + + connector = Connector.from_json(json_data.to_json) + expect(connector).to be_a(described_class) + expect(connector.name).to eq("example_connector") + expect(connector.type).to eq("source") + expect(connector.query_type).to eq("ai_ml") + expect(connector.connection_specification).to eq(key: "value") + end end end diff --git a/integrations/spec/multiwoven/integrations/source/databricks_model/client_spec.rb b/integrations/spec/multiwoven/integrations/source/databricks_model/client_spec.rb new file mode 100644 index 00000000..ff80de03 --- /dev/null +++ b/integrations/spec/multiwoven/integrations/source/databricks_model/client_spec.rb @@ -0,0 +1,193 @@ +# frozen_string_literal: true + +RSpec.describe Multiwoven::Integrations::Source::DatabricksModel::Client do + include WebMock::API + + before(:each) do + WebMock.disable_net_connect!(allow_localhost: true) + end + + let(:client) { described_class.new } + let(:mock_http_session) { double("Net::Http::Session") } + + let(:payload) do + { + messages: [ + { + role: "user", + content: "Hello there" + } + ] + } + end + + let(:sync_config_json) do + { + source: { + name: "databrick-model-source", + type: "source", + connection_specification: { + databricks_host: "test-host.databricks.com", + token: "test_token", + endpoint_name: "test", + request_format: "{}", + response_format: "{}" + } + }, + destination: { + name: "DestinationConnectorName", + type: "destination", + connection_specification: { + example_destination_key: "example_destination_value" + } + }, + model: { + name: "ExampleModel", + query: payload.to_json, + query_type: "ai_ml", + primary_key: "id" + }, + stream: { + name: "example_stream", + json_schema: { "field1": "type1" }, + request_method: "POST", + request_rate_limit: 4, + rate_limit_unit_seconds: 1 + }, + sync_mode: "full_refresh", + cursor_field: "timestamp", + destination_sync_mode: "upsert", + sync_id: "1" + } + end + + let(:sync_config) { Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config_json.to_json) } + + before do + allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request) + end + + let(:headers) do + { + "Accept" => "application/json", + "Authorization" => "Bearer test_token", + "Content-Type" => "application/json" + } + end + + describe "#check_connection" do + context "when the connection is successful" do + let(:response_body) { { "message" => "success" }.to_json } + before do + response = Net::HTTPSuccess.new("1.1", "200", "Unauthorized") + response.content_type = "application/json" + allow(response).to receive(:body).and_return(response_body) + allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request) + .with("https://test-host.databricks.com/api/2.0/serving-endpoints/test", + "GET", + headers: headers) + .and_return(response) + end + + it "returns a succeeded connection status" do + message = client.check_connection(sync_config_json[:source][:connection_specification]) + result = message.connection_status + expect(result).to be_a(Multiwoven::Integrations::Protocol::ConnectionStatus) + expect(result.status).to eq("succeeded") + expect(result.message).to be_nil + end + end + + context "when the connection fails" do + let(:response_body) { { "message" => "failed" }.to_json } + before do + response = Net::HTTPSuccess.new("1.1", "401", "Unauthorized") + response.content_type = "application/json" + allow(response).to receive(:body).and_return(response_body) + allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request) + .with("https://test-host.databricks.com/api/2.0/serving-endpoints/test", + "GET", + headers: headers) + .and_return(response) + end + + it "returns a failed connection status with an error message" do + message = client.check_connection(sync_config_json[:source][:connection_specification]) + result = message.connection_status + expect(result).to be_a(Multiwoven::Integrations::Protocol::ConnectionStatus) + expect(result.status).to eq("failed") + end + end + end + + describe "#discover" do + it "successfully returns the catalog message" do + message = client.discover(nil) + catalog = message.catalog + expect(catalog).to be_a(Multiwoven::Integrations::Protocol::Catalog) + expect(catalog.request_rate_limit).to eql(600) + expect(catalog.request_rate_limit_unit).to eql("minute") + expect(catalog.request_rate_concurrency).to eql(10) + end + + it "handles exceptions during discovery" do + allow(client).to receive(:read_json).and_raise(StandardError.new("test error")) + expect(client).to receive(:handle_exception).with( + an_instance_of(StandardError), + hash_including(context: "DATABRICKS MODEL:DISCOVER:EXCEPTION", type: "error") + ) + client.discover + end + end + + describe "#read" do + context "when the read is successful" do + let(:response_body) do + { + "id": "chatcmpl_090b5500-1e59-4226-878c-2e1ec5e0b3d3", + "object": "chat.completion", + "created": 1_724_067_784, + "model": "dbrx-instruct-071224", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + } + } + ] + }.to_json + end + before do + response = Net::HTTPSuccess.new("1.1", "200", "Unauthorized") + response.content_type = "application/json" + allow(response).to receive(:body).and_return(response_body) + allow(Multiwoven::Integrations::Core::HttpClient).to receive(:request) + .with("https://test-host.databricks.com/serving-endpoints/test/invocations", + "POST", + payload: JSON.parse(payload.to_json), + headers: headers) + .and_return(response) + end + it "successfully reads records" do + records = client.read(sync_config) + expect(records).to be_an(Array) + expect(records.first.record).to be_a(Multiwoven::Integrations::Protocol::RecordMessage) + expect(records.first.record.data).to eq(JSON.parse(response_body)) + end + end + context "when the read is failed" do + it "handles exceptions during reading" do + error_instance = StandardError.new("test error") + allow(client).to receive(:run_model).and_raise(error_instance) + expect(client).to receive(:handle_exception).with( + error_instance, + hash_including(context: "DATABRICKS MODEL:READ:EXCEPTION", type: "error") + ) + + client.read(sync_config) + end + end + end +end From 90742d801751f1de6ec334068d191c6b2b2974e7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:34:13 +0530 Subject: [PATCH 43/74] fix(CE): correction in databricks model connection_spec (#331) Co-authored-by: afthab vp --- integrations/Gemfile.lock | 2 +- integrations/lib/multiwoven/integrations/rollout.rb | 2 +- .../integrations/source/databrics_model/config/spec.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index fcad59f0..160d7ed9 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.9.0) + multiwoven-integrations (0.9.1) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 2088482b..a2409100 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.9.0" + VERSION = "0.9.1" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json b/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json index ef7ca0c9..9fb6ddac 100644 --- a/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json +++ b/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json @@ -6,7 +6,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Databricks Model", "type": "object", - "required": ["databricks_host", "token", "model_name"], + "required": ["databricks_host", "token", "endpoint"], "properties": { "databricks_host": { "title": "databricks_host", From 5ac0588b25387019101e807a0ab59e7e93905eaf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:34:45 +0530 Subject: [PATCH 44/74] chore(CE): upgrade version to 0.9.1 (#330) Co-authored-by: afthab vp --- server/Gemfile | 2 +- server/Gemfile.lock | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/server/Gemfile b/server/Gemfile index cd52ce17..c4e15073 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -13,7 +13,7 @@ gem "interactor", "~> 3.0" gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git" -gem "multiwoven-integrations", "~> 0.8.1" +gem "multiwoven-integrations", "~> 0.9.1" gem "temporal-ruby", github: "coinbase/temporal-ruby" diff --git a/server/Gemfile.lock b/server/Gemfile.lock index eb77ef58..383d6078 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -112,12 +112,12 @@ GEM console (~> 1.26) fiber-annotation io-event (~> 1.6, >= 1.6.5) - async-http (0.69.0) + async-http (0.70.0) async (>= 2.10.2) async-pool (~> 0.7) io-endpoint (~> 0.11) io-stream (~> 0.4) - protocol-http (~> 0.26) + protocol-http (~> 0.28) protocol-http1 (~> 0.19) protocol-http2 (~> 0.18) traces (>= 0.10) @@ -1781,7 +1781,7 @@ GEM gli (2.21.5) globalid (1.2.1) activesupport (>= 6.1) - google-apis-bigquery_v2 (0.75.0) + google-apis-bigquery_v2 (0.76.0) google-apis-core (>= 0.15.0, < 2.a) google-apis-core (0.15.1) addressable (~> 2.5, >= 2.5.1) @@ -1898,7 +1898,7 @@ GEM msgpack (1.7.2) multi_json (1.15.0) multipart-post (2.4.1) - multiwoven-integrations (0.8.1) + multiwoven-integrations (0.9.1) activesupport async-websocket aws-sdk-athena @@ -1973,7 +1973,7 @@ GEM premailer (~> 1.7, >= 1.7.9) process_executer (1.1.0) protocol-hpack (1.5.0) - protocol-http (0.28.1) + protocol-http (0.28.2) protocol-http1 (0.19.1) protocol-http (~> 0.22) protocol-http2 (0.18.0) @@ -2123,7 +2123,7 @@ GEM faraday-multipart gli hashie - sorbet-runtime (0.5.11518) + sorbet-runtime (0.5.11532) stringio (3.1.0) stripe (12.5.0) strong_migrations (1.8.0) @@ -2197,7 +2197,7 @@ DEPENDENCIES jwt kaminari liquid - multiwoven-integrations (~> 0.8.1) + multiwoven-integrations (~> 0.9.1) mysql2 newrelic_rpm parallel From ea2d5c057f8e3af510250d4d740348ec0028fc07 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:35:42 +0530 Subject: [PATCH 45/74] chore(CE): refactor error message to agreed standard (#332) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- server/app/controllers/api/v1/auth_controller.rb | 2 +- server/app/controllers/api/v1/reports_controller.rb | 2 +- server/app/controllers/application_controller.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/app/controllers/api/v1/auth_controller.rb b/server/app/controllers/api/v1/auth_controller.rb index 9a0f1197..e03d300d 100644 --- a/server/app/controllers/api/v1/auth_controller.rb +++ b/server/app/controllers/api/v1/auth_controller.rb @@ -93,7 +93,7 @@ def resend_verification attributes: { message: "Email verification link sent successfully!" } } }, status: :ok else - render json: { errors: [{ detail: result.error || result.errors }] }, status: :unprocessable_entity + render_error(message: result.error, status: :unprocessable_entity) end end end diff --git a/server/app/controllers/api/v1/reports_controller.rb b/server/app/controllers/api/v1/reports_controller.rb index 797a4ee0..334d6ef4 100644 --- a/server/app/controllers/api/v1/reports_controller.rb +++ b/server/app/controllers/api/v1/reports_controller.rb @@ -13,7 +13,7 @@ def index if result.success? render json: result.workspace_activity, status: :ok else - render json: { error: result.error }, status: :unprocessable_entity + render_error(message: result.error, status: :unprocessable_entity) end end diff --git a/server/app/controllers/application_controller.rb b/server/app/controllers/application_controller.rb index 951725bc..a94621be 100644 --- a/server/app/controllers/application_controller.rb +++ b/server/app/controllers/application_controller.rb @@ -21,7 +21,7 @@ def authenticate_user! return if user_signed_in? # If not authenticated, return a 401 unauthorized response - render json: { error: "Unauthorized" }, status: :unauthorized + render_error(message: "Unauthorized", status: :unauthorized) end def current_workspace From ccfb1ff8c930fd41dd3adca33981f7ddb468b43a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:38:12 +0530 Subject: [PATCH 46/74] feat(CE): catalog creation via API (#329) Co-authored-by: datafloyd --- server/app/contracts/catalog_contracts.rb | 12 +++ .../controllers/api/v1/catalogs_controller.rb | 44 ++++++++ .../interactors/catalogs/create_catalog.rb | 101 ++++++++++++++++++ server/app/models/connector.rb | 4 + server/config/routes.rb | 1 + .../spec/contracts/catalog_contracts_spec.rb | 99 +++++++++++++++++ .../catalogs/create_catalog_spec.rb | 54 ++++++++++ server/spec/requests/api/v1/catalogs_spec.rb | 95 ++++++++++++++++ 8 files changed, 410 insertions(+) create mode 100644 server/app/contracts/catalog_contracts.rb create mode 100644 server/app/controllers/api/v1/catalogs_controller.rb create mode 100644 server/app/interactors/catalogs/create_catalog.rb create mode 100644 server/spec/contracts/catalog_contracts_spec.rb create mode 100644 server/spec/interactors/catalogs/create_catalog_spec.rb create mode 100644 server/spec/requests/api/v1/catalogs_spec.rb diff --git a/server/app/contracts/catalog_contracts.rb b/server/app/contracts/catalog_contracts.rb new file mode 100644 index 00000000..7e0c7ad0 --- /dev/null +++ b/server/app/contracts/catalog_contracts.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module CatalogContracts + class Create < Dry::Validation::Contract + params do + required(:connector_id).filled(:integer) + required(:catalog).hash do + required(:json_schema).filled(:hash) + end + end + end +end diff --git a/server/app/controllers/api/v1/catalogs_controller.rb b/server/app/controllers/api/v1/catalogs_controller.rb new file mode 100644 index 00000000..9aa840bd --- /dev/null +++ b/server/app/controllers/api/v1/catalogs_controller.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Api + module V1 + class CatalogsController < ApplicationController + include Catalogs + before_action :set_connector, only: %i[create] + + def create + authorize current_workspace, policy_class: ConnectorPolicy + result = CreateCatalog.call( + connector: @connector, + catalog_params: catalog_params.to_h + ) + + if result.success? + @catalog = result.catalog + render json: @catalog, status: :created + else + render_error( + message: "Catalog creation failed", + status: :unprocessable_entity, + details: format_errors(result.catalog) + ) + end + end + + private + + def set_connector + @connector = current_workspace.connectors.find(params[:connector_id]) + rescue ActiveRecord::RecordNotFound + render_error( + message: "Connector not found", + status: :not_found + ) + end + + def catalog_params + params.require(:catalog).permit(json_schema: {}) + end + end + end +end diff --git a/server/app/interactors/catalogs/create_catalog.rb b/server/app/interactors/catalogs/create_catalog.rb new file mode 100644 index 00000000..436d321c --- /dev/null +++ b/server/app/interactors/catalogs/create_catalog.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Catalogs + class CreateCatalog + include Interactor + + DEFAULT_URL = "unknown" + DEFAULT_BATCH_SUPPORT = false + DEFAULT_BATCH_SIZE = 0 + DEFAULT_REQUEST_METHOD = "POST" + + def call + validate_catalog_params! + catalog = build_catalog + catalog.save + + if catalog.persisted? + context.catalog = catalog + else + context.fail!(error: "Failed to persist catalog", catalog:) + end + end + + private + + def validate_catalog_params! + context.catalog_params = context.catalog_params.with_indifferent_access + return if context.catalog_params[:json_schema].present? + + context.fail!(error: "json_schema must be present in catalog_params") + end + + def build_catalog + context.connector.build_catalog( + workspace_id: context.connector.workspace_id, + catalog: catalog_params, + catalog_hash: generate_catalog_hash + ) + end + + def catalog_params + { + streams: [build_stream_params], + request_rate_concurrency:, + request_rate_limit:, + request_rate_limit_unit: + } + end + + def build_stream_params + { + name: stream_name, + url: stream_url, + json_schema: context.catalog_params[:json_schema], + batch_support:, + batch_size:, + request_method: + } + end + + def stream_name + context.catalog_params[:name] || context.connector.name + end + + def stream_url + context.catalog_params[:url] || DEFAULT_URL + end + + def batch_support + context.catalog_params[:batch_support] || DEFAULT_BATCH_SUPPORT + end + + def batch_size + context.catalog_params[:batch_size] || DEFAULT_BATCH_SIZE + end + + def request_method + context.catalog_params[:request_method] || DEFAULT_REQUEST_METHOD + end + + def request_rate_concurrency + context.catalog_params[:request_rate_concurrency] || default_catalog[:request_rate_concurrency] + end + + def request_rate_limit + context.catalog_params[:request_rate_limit] || default_catalog[:request_rate_limit] + end + + def request_rate_limit_unit + context.catalog_params[:request_rate_limit_unit] || default_catalog[:request_rate_limit_unit] + end + + def generate_catalog_hash + Digest::SHA1.hexdigest(catalog_params.to_s) + end + + def default_catalog + @default_catalog ||= context.connector.pull_catalog + end + end +end diff --git a/server/app/models/connector.rb b/server/app/models/connector.rb index ac1d7212..9b86d5e4 100644 --- a/server/app/models/connector.rb +++ b/server/app/models/connector.rb @@ -86,4 +86,8 @@ def connector_query_type connector_spec = client.connector_spec connector_spec&.connector_query_type || "raw_sql" end + + def pull_catalog + connector_client.new.discover(configuration).catalog.to_h.with_indifferent_access + end end diff --git a/server/config/routes.rb b/server/config/routes.rb index b1b0aeaa..3fb17bdb 100644 --- a/server/config/routes.rb +++ b/server/config/routes.rb @@ -25,6 +25,7 @@ post :query_source end end + resources :catalogs, only: %i[create] resources :models resources :syncs do collection do diff --git a/server/spec/contracts/catalog_contracts_spec.rb b/server/spec/contracts/catalog_contracts_spec.rb new file mode 100644 index 00000000..2e2ed7a7 --- /dev/null +++ b/server/spec/contracts/catalog_contracts_spec.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe CatalogContracts::Create do + subject(:contract) { described_class.new } + + describe "validations" do + context "when valid parameters are provided" do + let(:valid_params) do + { + connector_id: 1, + catalog: { + json_schema: { + type: "object", + properties: { + name: { type: "string" } + }, + required: ["name"] + } + } + } + end + + it "passes validation" do + result = contract.call(valid_params) + expect(result).to be_success + end + end + + context "when connector_id is missing" do + let(:invalid_params) do + { + catalog: { + json_schema: { + type: "object", + properties: { + name: { type: "string" } + }, + required: ["name"] + } + } + } + end + + it "fails validation" do + result = contract.call(invalid_params) + expect(result).to_not be_success + expect(result.errors[:connector_id]).to include("is missing") + end + end + + context "when catalog is missing" do + let(:invalid_params) do + { + connector_id: 1 + } + end + + it "fails validation" do + result = contract.call(invalid_params) + expect(result).to_not be_success + expect(result.errors[:catalog]).to include("is missing") + end + end + + context "when json_schema is missing" do + let(:invalid_params) do + { + connector_id: 1, + catalog: {} + } + end + + it "fails validation" do + result = contract.call(invalid_params) + expect(result).to_not be_success + expect(result.errors[:catalog][:json_schema]).to include("is missing") + end + end + + context "when json_schema is not a hash" do + let(:invalid_params) do + { + connector_id: 1, + catalog: { + json_schema: "invalid_schema" + } + } + end + + it "fails validation" do + result = contract.call(invalid_params) + expect(result).to_not be_success + expect(result.errors[:catalog][:json_schema]).to include("must be a hash") + end + end + end +end diff --git a/server/spec/interactors/catalogs/create_catalog_spec.rb b/server/spec/interactors/catalogs/create_catalog_spec.rb new file mode 100644 index 00000000..a495af37 --- /dev/null +++ b/server/spec/interactors/catalogs/create_catalog_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Catalogs::CreateCatalog do + let(:workspace) { create(:workspace) } + let(:connector) { create(:connector, workspace:, connector_name: "Klaviyo", connector_type: "destination") } + + context "with valid params" do + let(:catalog_params) do + { + "name" => "Test Catalog", + "url" => "http://example.com", + "json_schema" => { "type" => "object" }, + "batch_support" => true, + "batch_size" => 10, + "request_method" => "POST" + } + end + + it "creates catalog" do + result = described_class.call( + connector:, + catalog_params: + ) + expect(result.success?).to eq(true) + expect(result.catalog.catalog["streams"].first).to eql(catalog_params) + expect(result.catalog.workspace_id).to eql(connector.workspace_id) + expect(result.catalog.connector_id).to eql(connector.id) + + default_catalog = connector.pull_catalog + expect(result.catalog.catalog["request_rate_limit"]).to eql(default_catalog[:request_rate_limit]) + expect(result.catalog.catalog["request_rate_limit_unit"]).to eql(default_catalog[:request_rate_limit_unit]) + expect(result.catalog.catalog["request_rate_concurrency"]).to eql(default_catalog[:request_rate_concurrency]) + end + end + + context "with invalid params" do + let(:catalog_params) do + { + "name" => "Test Catalog", + "url" => "http://example.com", + "batch_support" => true, + "batch_size" => 10, + "request_method" => "POST" + } + end + + it "fails to create a connector" do + result = described_class.call(workspace:, catalog_params:) + expect(result.failure?).to eq(true) + end + end +end diff --git a/server/spec/requests/api/v1/catalogs_spec.rb b/server/spec/requests/api/v1/catalogs_spec.rb new file mode 100644 index 00000000..f85f6de2 --- /dev/null +++ b/server/spec/requests/api/v1/catalogs_spec.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe "Api::V1::CatalogsController", type: :request do + let(:workspace) { create(:workspace) } + let!(:workspace_id) { workspace.id } + let(:user) { workspace.workspace_users.first.user } + let(:viewer_role) { create(:role, :viewer) } + let(:member_role) { create(:role, :member) } + let(:admin_role) { create(:role, :admin) } + let(:connector) do + create(:connector, workspace:, connector_type: "destination", name: "klavio1", connector_name: "Klaviyo") + end + + let(:request_body) do + { + connector_id: connector.id, + catalog: { + "json_schema" => { + "input" => [ + { + "name" => "val1", + "type" => "string" + } + ], + "output" => [ + { + "name" => "val1", + "type" => "string" + } + ] + } + } + } + end + + before do + user.confirm + end + describe "POST #create" do + context "when it is an unauthenticated user for create catalog" do + it "returns unauthorized" do + post "/api/v1/catalogs" + expect(response).to have_http_status(:unauthorized) + end + end + + context "user is admin" do + it "creates catalog" do + workspace.workspace_users.first.update(role: admin_role) + + post "/api/v1/catalogs", params: request_body.to_json, headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:created) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash.dig(:data, :id)).to be_present + expect(response_hash.dig(:data, :type)).to eq("catalogs") + expect(response_hash.dig(:data, :attributes, :workspace_id)).to eq(workspace.id) + expect(response_hash.dig(:data, :attributes, :connector_id)).to eq(connector.id) + expect(response_hash.dig(:data, :attributes, :catalog, :streams).first["name"]).to eq(connector.name) + expect(response_hash.dig(:data, :attributes, :catalog, + :streams).first["json_schema"]).to eq(request_body[:catalog]["json_schema"]) + end + end + + context "user is member" do + it "shouldn't create catalog" do + workspace.workspace_users.first.update(role: member_role) + + post "/api/v1/catalogs", params: request_body.to_json, headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:created) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash.dig(:data, :id)).to be_present + expect(response_hash.dig(:data, :type)).to eq("catalogs") + expect(response_hash.dig(:data, :attributes, :workspace_id)).to eq(workspace.id) + expect(response_hash.dig(:data, :attributes, :connector_id)).to eq(connector.id) + expect(response_hash.dig(:data, :attributes, :catalog, :streams).first["name"]).to eq(connector.name) + expect(response_hash.dig(:data, :attributes, :catalog, + :streams).first["json_schema"]).to eq(request_body[:catalog]["json_schema"]) + end + end + + context "user is viewer" do + it "shouldn't create catalog" do + workspace.workspace_users.first.update(role: viewer_role) + + post "/api/v1/catalogs", params: request_body.to_json, headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:unauthorized) + end + end + end +end From 4036587d33abc361c26d75ac0fcc68e6b772276f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 14:33:22 +0530 Subject: [PATCH 47/74] feat(CE): calalog update api (#333) Co-authored-by: datafloyd --- server/app/contracts/catalog_contracts.rb | 10 ++ .../controllers/api/v1/catalogs_controller.rb | 27 ++- .../interactors/catalogs/update_catalog.rb | 15 ++ server/config/routes.rb | 2 +- .../spec/contracts/catalog_contracts_spec.rb | 154 ++++++++++++++++++ .../catalogs/update_catalog_spec.rb | 92 +++++++++++ ...gs_spec.rb => catalogs_controller_spec.rb} | 87 ++++++++++ 7 files changed, 385 insertions(+), 2 deletions(-) create mode 100644 server/app/interactors/catalogs/update_catalog.rb create mode 100644 server/spec/interactors/catalogs/update_catalog_spec.rb rename server/spec/requests/api/v1/{catalogs_spec.rb => catalogs_controller_spec.rb} (52%) diff --git a/server/app/contracts/catalog_contracts.rb b/server/app/contracts/catalog_contracts.rb index 7e0c7ad0..7ebceec1 100644 --- a/server/app/contracts/catalog_contracts.rb +++ b/server/app/contracts/catalog_contracts.rb @@ -9,4 +9,14 @@ class Create < Dry::Validation::Contract end end end + + class Update < Dry::Validation::Contract + params do + required(:id).filled(:integer) + required(:connector_id).filled(:integer) + required(:catalog).hash do + required(:json_schema).filled(:hash) + end + end + end end diff --git a/server/app/controllers/api/v1/catalogs_controller.rb b/server/app/controllers/api/v1/catalogs_controller.rb index 9aa840bd..c93d60e5 100644 --- a/server/app/controllers/api/v1/catalogs_controller.rb +++ b/server/app/controllers/api/v1/catalogs_controller.rb @@ -4,7 +4,8 @@ module Api module V1 class CatalogsController < ApplicationController include Catalogs - before_action :set_connector, only: %i[create] + before_action :set_connector, only: %i[create update] + before_action :set_catalog, only: %i[update] def create authorize current_workspace, policy_class: ConnectorPolicy @@ -25,6 +26,26 @@ def create end end + def update + authorize current_workspace, policy_class: ConnectorPolicy + result = UpdateCatalog.call( + connector: @connector, + catalog: @catalog, + catalog_params: catalog_params.to_h + ) + + if result.success? + @catalog = result.catalog + render json: @catalog, status: :created + else + render_error( + message: "Catalog update failed", + status: :unprocessable_entity, + details: format_errors(result.catalog) + ) + end + end + private def set_connector @@ -36,6 +57,10 @@ def set_connector ) end + def set_catalog + @catalog = @connector.catalog + end + def catalog_params params.require(:catalog).permit(json_schema: {}) end diff --git a/server/app/interactors/catalogs/update_catalog.rb b/server/app/interactors/catalogs/update_catalog.rb new file mode 100644 index 00000000..64f165e6 --- /dev/null +++ b/server/app/interactors/catalogs/update_catalog.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Catalogs + class UpdateCatalog < CreateCatalog + def call + validate_catalog_params! + unless context.catalog.update( + catalog: catalog_params, catalog_hash: generate_catalog_hash, + connector_id: context.connector.id, workspace_id: context.connector.workspace_id + ) + context.fail!(model: context.model) + end + end + end +end diff --git a/server/config/routes.rb b/server/config/routes.rb index 3fb17bdb..a031c347 100644 --- a/server/config/routes.rb +++ b/server/config/routes.rb @@ -25,7 +25,7 @@ post :query_source end end - resources :catalogs, only: %i[create] + resources :catalogs, only: %i[create update] resources :models resources :syncs do collection do diff --git a/server/spec/contracts/catalog_contracts_spec.rb b/server/spec/contracts/catalog_contracts_spec.rb index 2e2ed7a7..e31e7392 100644 --- a/server/spec/contracts/catalog_contracts_spec.rb +++ b/server/spec/contracts/catalog_contracts_spec.rb @@ -97,3 +97,157 @@ end end end + +RSpec.describe CatalogContracts::Update do + subject(:contract) { described_class.new } + + describe "validations" do + context "when valid parameters are provided" do + let(:valid_params) do + { + id: 1, + connector_id: 2, + catalog: { + json_schema: { + type: "object", + properties: { + name: { type: "string" } + }, + required: ["name"] + } + } + } + end + + it "passes validation" do + result = contract.call(valid_params) + expect(result).to be_success + end + end + + context "when id is missing" do + let(:invalid_params) do + { + connector_id: 2, + catalog: { + json_schema: { + type: "object", + properties: { + name: { type: "string" } + }, + required: ["name"] + } + } + } + end + + it "fails validation" do + result = contract.call(invalid_params) + expect(result).to_not be_success + expect(result.errors[:id]).to include("is missing") + end + end + + context "when connector_id is missing" do + let(:invalid_params) do + { + id: 1, + catalog: { + json_schema: { + type: "object", + properties: { + name: { type: "string" } + }, + required: ["name"] + } + } + } + end + + it "fails validation" do + result = contract.call(invalid_params) + expect(result).to_not be_success + expect(result.errors[:connector_id]).to include("is missing") + end + end + + context "when catalog is missing" do + let(:invalid_params) do + { + id: 1, + connector_id: 2 + } + end + + it "fails validation" do + result = contract.call(invalid_params) + expect(result).to_not be_success + expect(result.errors[:catalog]).to include("is missing") + end + end + + context "when json_schema is missing" do + let(:invalid_params) do + { + id: 1, + connector_id: 2, + catalog: {} + } + end + + it "fails validation" do + result = contract.call(invalid_params) + expect(result).to_not be_success + expect(result.errors[:catalog][:json_schema]).to include("is missing") + end + end + + context "when id is not an integer" do + let(:invalid_params) do + { + id: "not_an_integer", + connector_id: 2, + catalog: { + json_schema: { + type: "object", + properties: { + name: { type: "string" } + }, + required: ["name"] + } + } + } + end + + it "fails validation" do + result = contract.call(invalid_params) + expect(result).to_not be_success + expect(result.errors[:id]).to include("must be an integer") + end + end + + context "when connector_id is not an integer" do + let(:invalid_params) do + { + id: 1, + connector_id: "not_an_integer", + catalog: { + json_schema: { + type: "object", + properties: { + name: { type: "string" } + }, + required: ["name"] + } + } + } + end + + it "fails validation" do + result = contract.call(invalid_params) + expect(result).to_not be_success + expect(result.errors[:connector_id]).to include("must be an integer") + end + end + end +end diff --git a/server/spec/interactors/catalogs/update_catalog_spec.rb b/server/spec/interactors/catalogs/update_catalog_spec.rb new file mode 100644 index 00000000..556d39f1 --- /dev/null +++ b/server/spec/interactors/catalogs/update_catalog_spec.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe Catalogs::UpdateCatalog do + let(:workspace) { create(:workspace) } + let(:connector) { create(:connector, workspace:, connector_name: "Klaviyo", connector_type: "destination") } + let(:existing_catalog) { create(:catalog, connector:, catalog: { "streams" => [] }) } + + context "with valid params" do + let(:catalog_params) do + { + "name" => "Updated Catalog", + "url" => "http://updated-example.com", + "json_schema" => { "type" => "object" }, + "batch_support" => true, + "batch_size" => 20, + "request_method" => "PUT" + } + end + + it "updates the existing catalog" do + result = described_class.call( + connector:, + catalog: existing_catalog, + catalog_params: + ) + + expect(result.success?).to eq(true) + expect(result.catalog.catalog["streams"].first).to include(catalog_params) + expect(result.catalog.workspace_id).to eql(connector.workspace_id) + expect(result.catalog.connector_id).to eql(connector.id) + + default_catalog = connector.pull_catalog + expect(result.catalog.catalog["request_rate_limit"]).to eql(default_catalog[:request_rate_limit]) + expect(result.catalog.catalog["request_rate_limit_unit"]).to eql(default_catalog[:request_rate_limit_unit]) + expect(result.catalog.catalog["request_rate_concurrency"]).to eql(default_catalog[:request_rate_concurrency]) + end + end + + context "with invalid params" do + let(:catalog_params) do + { + "name" => "Updated Catalog", + "url" => "http://updated-example.com", + # Missing required json_schema + "batch_support" => true, + "batch_size" => 20, + "request_method" => "PUT" + } + end + + it "fails to update the catalog" do + result = described_class.call( + connector:, + catalog: existing_catalog, + catalog_params: + ) + + expect(result.failure?).to eq(true) + expect(result.error).to eq("json_schema must be present in catalog_params") + end + end + + context "when update operation fails" do + let(:catalog_params) do + { + + "name" => "Invalid Update", + "url" => "http://invalid-update.com", + "json_schema" => { "type" => "object" }, + "batch_support" => true, + "batch_size" => 20, + "request_method" => "PUT" + } + end + + before do + allow_any_instance_of(Catalog).to receive(:update).and_return(false) + end + + it "fails with an appropriate error" do + result = described_class.call( + connector:, + catalog: existing_catalog, + catalog_params: + ) + + expect(result.failure?).to eq(true) + end + end +end diff --git a/server/spec/requests/api/v1/catalogs_spec.rb b/server/spec/requests/api/v1/catalogs_controller_spec.rb similarity index 52% rename from server/spec/requests/api/v1/catalogs_spec.rb rename to server/spec/requests/api/v1/catalogs_controller_spec.rb index f85f6de2..52b684ca 100644 --- a/server/spec/requests/api/v1/catalogs_spec.rb +++ b/server/spec/requests/api/v1/catalogs_controller_spec.rb @@ -13,6 +13,10 @@ create(:connector, workspace:, connector_type: "destination", name: "klavio1", connector_name: "Klaviyo") end + let(:existing_catalog) do + create(:catalog, connector:, catalog: { "streams" => [{ "name" => "Old Catalog", "json_schema" => {} }] }) + end + let(:request_body) do { connector_id: connector.id, @@ -35,6 +39,28 @@ } end + let(:update_request_body) do + { + connector_id: connector.id, + catalog: { + "json_schema" => { + "input" => [ + { + "updated_name" => "val1", + "updated_type" => "string" + } + ], + "output" => [ + { + "name" => "val1", + "type" => "string" + } + ] + } + } + } + end + before do user.confirm end @@ -92,4 +118,65 @@ end end end + + describe "PUT #update" do + context "when it is an unauthenticated user" do + it "returns unauthorized" do + put "/api/v1/catalogs/#{existing_catalog.id}", + params: update_request_body.to_json, + headers: { "Content-Type": "application/json" } + expect(response).to have_http_status(:unauthorized) + end + end + + context "user is admin" do + it "updates the catalog" do + workspace.workspace_users.first.update(role: admin_role) + + put "/api/v1/catalogs/#{existing_catalog.id}", + params: update_request_body.to_json, + headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:created) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash.dig(:data, :id)).to be_present + expect(response_hash.dig(:data, :type)).to eq("catalogs") + expect(response_hash.dig(:data, :attributes, :workspace_id)).to eq(workspace.id) + expect(response_hash.dig(:data, :attributes, :connector_id)).to eq(connector.id) + expect(response_hash.dig(:data, :attributes, :catalog, + :streams).first["json_schema"]).to eq(update_request_body[:catalog]["json_schema"]) + end + end + + context "user is member" do + it "updates the catalog" do + workspace.workspace_users.first.update(role: member_role) + + put "/api/v1/catalogs/#{existing_catalog.id}", + params: update_request_body.to_json, + headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:created) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash.dig(:data, :id)).to be_present + expect(response_hash.dig(:data, :type)).to eq("catalogs") + expect(response_hash.dig(:data, :attributes, :workspace_id)).to eq(workspace.id) + expect(response_hash.dig(:data, :attributes, :connector_id)).to eq(connector.id) + expect(response_hash.dig(:data, :attributes, :catalog, + :streams).first["json_schema"]).to eq(update_request_body[:catalog]["json_schema"]) + end + end + + context "user is viewer" do + it "updates the catalog" do + workspace.workspace_users.first.update(role: viewer_role) + + put "/api/v1/catalogs/#{existing_catalog.id}", + params: update_request_body.to_json, + headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:unauthorized) + end + end + end end From e8814298f621d49e60722309301a5d7b77c78bc2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 16:21:13 +0530 Subject: [PATCH 48/74] feat(CE): add AI/ML query type to model (#334) Co-authored-by: datafloyd --- server/app/contracts/model_contracts.rb | 27 +++++-- .../controllers/api/v1/models_controller.rb | 17 +++-- server/app/models/model.rb | 18 +++-- server/app/serializers/model_serializer.rb | 2 +- ...40823114746_add_configuration_to_models.rb | 5 ++ server/db/schema.rb | 3 +- server/spec/contracts/model_contracts_spec.rb | 70 ++++++------------- server/spec/models/model_spec.rb | 67 ++++++++++++++++-- .../requests/api/v1/models_controller_spec.rb | 50 +++++++++++++ 9 files changed, 186 insertions(+), 73 deletions(-) create mode 100644 server/db/migrate/20240823114746_add_configuration_to_models.rb diff --git a/server/app/contracts/model_contracts.rb b/server/app/contracts/model_contracts.rb index a560eb23..a9d88e77 100644 --- a/server/app/contracts/model_contracts.rb +++ b/server/app/contracts/model_contracts.rb @@ -18,9 +18,10 @@ class Create < Dry::Validation::Contract required(:model).hash do required(:connector_id).filled(:integer) required(:name).filled(:string) - required(:query).filled(:string) + optional(:query).filled(:string) required(:query_type).filled(:string) required(:primary_key).filled(:string) + optional(:configuration).filled(:hash) end end @@ -29,8 +30,21 @@ class Create < Dry::Validation::Contract end rule(model: :query) do - regex = /\b(?:LIMIT|OFFSET)\b\s*\d*\s*;?\s*$/i - key.failure("Query validation failed: Query cannot contain LIMIT or OFFSET") if value.match?(regex) + if %w[raw_sql dbt soql].include?(values[:model][:query_type]) + + if values[:model][:query].present? + regex = /\b(?:LIMIT|OFFSET)\b\s*\d*\s*;?\s*$/i + key.failure("Query validation failed: Query cannot contain LIMIT or OFFSET") if value.match?(regex) + else + key.failure("Query is required for this query type") + end + end + end + + rule(model: :configuration) do + if values[:model][:configuration].blank? && (values[:model][:query_type] == "ai_ml") + key.failure("Configuration is required for this query type") + end end end @@ -43,6 +57,7 @@ class Update < Dry::Validation::Contract optional(:query).filled(:string) optional(:query_type).filled(:string) optional(:primary_key).filled(:string) + optional(:configuration).filled(:hash) end end @@ -53,8 +68,10 @@ class Update < Dry::Validation::Contract end rule(model: :query) do - regex = /\b(?:LIMIT|OFFSET)\b\s*\d*\s*;?\s*$/i - key.failure("Query validation failed: Query cannot contain LIMIT or OFFSET") if value.match?(regex) + if values[:model][:query].present? + regex = /\b(?:LIMIT|OFFSET)\b\s*\d*\s*;?\s*$/i + key.failure("Query validation failed: Query cannot contain LIMIT or OFFSET") if value.match?(regex) + end end end diff --git a/server/app/controllers/api/v1/models_controller.rb b/server/app/controllers/api/v1/models_controller.rb index ca5f3b1c..f39dadfa 100644 --- a/server/app/controllers/api/v1/models_controller.rb +++ b/server/app/controllers/api/v1/models_controller.rb @@ -91,12 +91,17 @@ def validate_query end def model_params - params.require(:model).permit(:connector_id, - :name, - :description, - :query, - :query_type, - :primary_key).merge(workspace_id: current_workspace.id) + params.require(:model).permit( + :connector_id, + :name, + :description, + :query, + :query_type, + :primary_key, + configuration: {} + ).merge( + workspace_id: current_workspace.id + ) end end end diff --git a/server/app/models/model.rb b/server/app/models/model.rb index 6732ac96..372effd3 100644 --- a/server/app/models/model.rb +++ b/server/app/models/model.rb @@ -18,9 +18,12 @@ class Model < ApplicationRecord validates :workspace_id, presence: true validates :connector_id, presence: true validates :name, presence: true - validates :query, presence: true - validates :primary_key, presence: true - enum :query_type, %i[raw_sql dbt soql table_selector] + + enum :query_type, %i[raw_sql dbt soql table_selector ai_ml] + + validates :query, presence: true, if: :requires_query? + # Havesting configuration + validates :configuration, presence: true, if: :requires_configuration? belongs_to :workspace belongs_to :connector @@ -28,7 +31,6 @@ class Model < ApplicationRecord has_many :syncs, dependent: :destroy default_scope { order(updated_at: :desc) } - def to_protocol Multiwoven::Integrations::Protocol::Model.new( name:, @@ -37,4 +39,12 @@ def to_protocol primary_key: ) end + + def requires_query? + %w[raw_sql dbt soql table_selector].include?(query_type) + end + + def requires_configuration? + %w[ai_ml].include?(query_type) + end end diff --git a/server/app/serializers/model_serializer.rb b/server/app/serializers/model_serializer.rb index 0127426d..eb11b504 100644 --- a/server/app/serializers/model_serializer.rb +++ b/server/app/serializers/model_serializer.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class ModelSerializer < ActiveModel::Serializer - attributes :id, :name, :description, :query, :query_type, :primary_key, :created_at, :updated_at + attributes :id, :name, :description, :query, :query_type, :configuration, :primary_key, :created_at, :updated_at attribute :connector do ConnectorSerializer.new(object.connector).attributes diff --git a/server/db/migrate/20240823114746_add_configuration_to_models.rb b/server/db/migrate/20240823114746_add_configuration_to_models.rb new file mode 100644 index 00000000..7132c704 --- /dev/null +++ b/server/db/migrate/20240823114746_add_configuration_to_models.rb @@ -0,0 +1,5 @@ +class AddConfigurationToModels < ActiveRecord::Migration[7.1] + def change + add_column :models, :configuration, :jsonb + end +end diff --git a/server/db/schema.rb b/server/db/schema.rb index d6b61860..e1967009 100644 --- a/server/db/schema.rb +++ b/server/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_07_26_095056) do +ActiveRecord::Schema[7.1].define(version: 2024_08_23_114746) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -48,6 +48,7 @@ t.datetime "created_at", null: false t.datetime "updated_at", null: false t.string "description" + t.jsonb "configuration" end create_table "organizations", force: :cascade do |t| diff --git a/server/spec/contracts/model_contracts_spec.rb b/server/spec/contracts/model_contracts_spec.rb index 360495cb..e92c6489 100644 --- a/server/spec/contracts/model_contracts_spec.rb +++ b/server/spec/contracts/model_contracts_spec.rb @@ -57,87 +57,82 @@ end it "passes validation" do - allow(PgQuery).to receive(:parse).with("SELECT * FROM table;").and_return(true) expect(contract.call(valid_inputs)).to be_success end end - context "with invalid query_type" do + context "with missing query for query_type requiring it" do let(:invalid_inputs) do { model: { connector_id: 1, name: "Model Name", - query: "SELECT FROM table;", - query_type: "test", + query_type: "raw_sql", primary_key: "id" } } end - it "fails validation due to invalid query type" do + it "fails validation due to missing query" do result = contract.call(invalid_inputs) - expect(result.errors[:model][:query_type]).to include("invalid query type") + expect(result.errors[:model][:query]).to include("Query is required for this query type") end end - context "with query containing LIMIT" do + context "with invalid query_type" do let(:invalid_inputs) do { model: { connector_id: 1, name: "Model Name", - query: "SELECT * FROM table LIMIT 10;", - query_type: "raw_sql", + query: "SELECT FROM table;", + query_type: "test", primary_key: "id" } } end - it "fails validation due to LIMIT in query" do - error_message = "Query validation failed: Query cannot contain LIMIT or OFFSET" + it "fails validation due to invalid query type" do result = contract.call(invalid_inputs) - expect(result.errors[:model][:query]).to include(error_message) + expect(result.errors[:model][:query_type]).to include("invalid query type") end end - context "with query containing OFFSET" do + context "with query containing LIMIT" do let(:invalid_inputs) do { model: { connector_id: 1, name: "Model Name", - query: "SELECT * FROM table OFFSET 10;", + query: "SELECT * FROM table LIMIT 10;", query_type: "raw_sql", primary_key: "id" } } end - it "fails validation due to OFFSET in query" do + it "fails validation due to LIMIT in query" do error_message = "Query validation failed: Query cannot contain LIMIT or OFFSET" result = contract.call(invalid_inputs) expect(result.errors[:model][:query]).to include(error_message) end end - context "with query containing LIMIT and OFFSET" do + context "with missing configuration for ai_ml query_type" do let(:invalid_inputs) do { model: { connector_id: 1, name: "Model Name", - query: "SELECT * FROM table LIMIT 10 OFFSET 5;", - query_type: "raw_sql", + query_type: "ai_ml", primary_key: "id" } } end - it "fails validation due to LIMIT and OFFSET in query" do - error_message = "Query validation failed: Query cannot contain LIMIT or OFFSET" + it "fails validation due to missing configuration" do result = contract.call(invalid_inputs) - expect(result.errors[:model][:query]).to include(error_message) + expect(result.errors[:model][:configuration]).to include("Configuration is required for this query type") end end end @@ -202,43 +197,20 @@ end end - context "with query containing OFFSET" do - let(:invalid_inputs) do - { - id: 1, - model: { - name: "Updated Model Name", - query: "SELECT * FROM updated_table OFFSET 10;", - query_type: "soql", - primary_key: "updated_id" - } - } - end - - it "fails validation due to OFFSET in query" do - error_message = "Query validation failed: Query cannot contain LIMIT or OFFSET" - result = contract.call(invalid_inputs) - expect(result.errors[:model][:query]).to include(error_message) - end - end - - context "with query containing LIMIT and OFFSET" do - let(:invalid_inputs) do + context "with valid inputs without query" do + let(:valid_inputs) do { id: 1, model: { name: "Updated Model Name", - query: "SELECT * FROM updated_table LIMIT 10 OFFSET 5;", - query_type: "soql", + query_type: "ai_ml", primary_key: "updated_id" } } end - it "fails validation due to LIMIT and OFFSET in query" do - error_message = "Query validation failed: Query cannot contain LIMIT or OFFSET" - result = contract.call(invalid_inputs) - expect(result.errors[:model][:query]).to include(error_message) + it "passes validation" do + expect(contract.call(valid_inputs)).to be_success end end end diff --git a/server/spec/models/model_spec.rb b/server/spec/models/model_spec.rb index 6e16e3a5..10d28184 100644 --- a/server/spec/models/model_spec.rb +++ b/server/spec/models/model_spec.rb @@ -14,20 +14,74 @@ # created_at :datetime not null # updated_at :datetime not null # + require "rails_helper" RSpec.describe Model, type: :model do describe "associations" do it { should belong_to(:workspace) } it { should belong_to(:connector) } + it { should have_many(:syncs).dependent(:destroy) } end describe "validations" do + let(:source) do + create(:connector, connector_type: "source", connector_name: "Snowflake") + end + it { should validate_presence_of(:workspace_id) } it { should validate_presence_of(:connector_id) } it { should validate_presence_of(:name) } - it { should validate_presence_of(:query) } - it { should have_many(:syncs).dependent(:destroy) } + + context "when query_type requires query" do + it "validates presence of query" do + model = Model.new( + name: "test_model", + query_type: :raw_sql, connector_id: source.id, + workspace_id: source.workspace_id + ) + model.query = nil + expect(model).not_to be_valid + expect(model.errors[:query]).to include("can't be blank") + end + end + + context "when query_type does not require query" do + it "does not validate presence of query" do + model = Model.new( + name: "test_model", + query_type: :ai_ml, connector_id: source.id, + workspace_id: source.workspace_id, + configuration: { "field1" => "value1" } + ) + model.query = nil + expect(model).to be_valid + end + end + + context "when query_type requires configuration" do + it "validates presence of configuration" do + model = Model.new( + name: "test_model", + query_type: :ai_ml, connector_id: source.id, + workspace_id: source.workspace_id + ) + model.configuration = nil + expect(model).not_to be_valid + end + end + + context "when query_type does not require configuration" do + it "does not validate presence of configuration" do + model = Model.new( + name: "test_model", + query_type: :raw_sql, connector_id: source.id, + workspace_id: source.workspace_id, query: "test_query" + ) + model.configuration = nil + expect(model).to be_valid + end + end end describe "#to_protocol" do @@ -46,7 +100,6 @@ expect(protocol_model.query).to eq(model.query) expect(protocol_model.query_type).to eq(model.query_type) expect(protocol_model.primary_key).to eq(model.primary_key) - expect(model).to have_many(:syncs).dependent(:destroy) end end @@ -56,14 +109,14 @@ end let(:model) { create_list(:model, 4, connector: source) } - context "when a multiple models are created" do - it "returns the model in descending order of updated_at" do + context "when multiple models are created" do + it "returns the models in descending order of updated_at" do expect(Model.all).to eq(model.sort_by(&:updated_at).reverse) end end context "when a model is updated" do - it "returns the model in descending order of updated_at" do + it "returns the models in descending order of updated_at" do model.first.update(updated_at: DateTime.current + 1.week) model.last.update(updated_at: DateTime.current - 1.week) @@ -74,7 +127,7 @@ describe "query_type" do it "defines query_type enum with specified values" do - expect(Model.query_types).to eq({ "raw_sql" => 0, "dbt" => 1, "soql" => 2, "table_selector" => 3 }) + expect(Model.query_types).to eq({ "raw_sql" => 0, "dbt" => 1, "soql" => 2, "table_selector" => 3, "ai_ml" => 4 }) end end end diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb index 8f2721ed..7ac61917 100644 --- a/server/spec/requests/api/v1/models_controller_spec.rb +++ b/server/spec/requests/api/v1/models_controller_spec.rb @@ -180,6 +180,31 @@ expect(response_hash.dig(:data, :attributes, :primary_key)).to eq(request_body.dig(:model, :primary_key)) end + context "when creating a model with query_type = ai_ml and configuration is present" do + let(:request_body) do + { + model: { + connector_id: models.first.connector_id, + name: "AI/ML Model", + query_type: "ai_ml", + primary_key: "id", + configuration: { "test" => "value" } + } + } + end + + it "creates the model and returns success" do + post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:created) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash.dig(:data, :id)).to be_present + expect(response_hash.dig(:data, :attributes, :name)).to eq(request_body.dig(:model, :name)) + expect(response_hash.dig(:data, :attributes, :query_type)).to eq("ai_ml") + expect(response_hash.dig(:data, :attributes, :configuration)).to eq(request_body.dig(:model, :configuration)) + end + end + it "returns fail viwer role" do workspace.workspace_users.first.update(role: viewer_role) post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" } @@ -244,6 +269,31 @@ expect(response_hash.dig(:data, :attributes, :primary_key)).to eq(request_body.dig(:model, :primary_key)) end + context "when updating a model with query_type = ai_ml and configuration is present" do + let(:request_body) do + { + model: { + connector_id: models.first.connector_id, + name: "Updated AI/ML Model", + query_type: "ai_ml", + primary_key: "updated_id", + configuration: { "updated_test" => "value" } + } + } + end + it "updates the model and returns success" do + put "/api/v1/models/#{models.second.id}", params: request_body.to_json, headers: + { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:ok) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash.dig(:data, :id)).to be_present + expect(response_hash.dig(:data, :id)).to eq(models.second.id.to_s) + expect(response_hash.dig(:data, :attributes, :name)).to eq(request_body.dig(:model, :name)) + expect(response_hash.dig(:data, :attributes, :query_type)).to eq("ai_ml") + expect(response_hash.dig(:data, :attributes, :configuration)).to eq(request_body.dig(:model, :configuration)) + end + end + it "returns fail for viewer role" do workspace.workspace_users.first.update(role: viewer_role) put "/api/v1/models/#{models.second.id}", params: request_body.to_json, headers: From 6b790fcdf239af4465de8db4c90167ec590326a9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:34:40 +0530 Subject: [PATCH 49/74] feat(CE): filter connectors based on the category (#338) Co-authored-by: datafloyd --- .../v1/connector_definitions_controller.rb | 12 ++-- .../filter_connectors.rb | 56 +++++++++++++++ .../filter_connectors_spec.rb | 69 +++++++++++++++++++ 3 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 server/app/interactors/connector_definitions/filter_connectors.rb create mode 100644 server/spec/interactors/connector_definitions/filter_connectors_spec.rb diff --git a/server/app/controllers/api/v1/connector_definitions_controller.rb b/server/app/controllers/api/v1/connector_definitions_controller.rb index 7b127b2d..5298de25 100644 --- a/server/app/controllers/api/v1/connector_definitions_controller.rb +++ b/server/app/controllers/api/v1/connector_definitions_controller.rb @@ -3,6 +3,7 @@ module Api module V1 class ConnectorDefinitionsController < ApplicationController + include ConnectorDefinitions before_action :set_connectors, only: %i[show index] before_action :set_connector_client, only: %i[check_connection] @@ -33,10 +34,9 @@ def check_connection private def set_connectors - @connectors = Multiwoven::Integrations::Service - .connectors - .with_indifferent_access - @connectors = @connectors[params[:type]] if params[:type] + @connectors = FilterConnectors.call( + connection_definitions_params + ).connectors end def set_connector_client @@ -46,6 +46,10 @@ def set_connector_client params[:name].camelize ).new end + + def connection_definitions_params + params.permit(:type, :catagory) + end end end end diff --git a/server/app/interactors/connector_definitions/filter_connectors.rb b/server/app/interactors/connector_definitions/filter_connectors.rb new file mode 100644 index 00000000..4d266f23 --- /dev/null +++ b/server/app/interactors/connector_definitions/filter_connectors.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module ConnectorDefinitions + class FilterConnectors + include Interactor + + # TODO: Move this to integration so that whenever a new category is introduced, + # integrations will be the single source of truth for categories + + DATA_CATEGORIES = [ + "Data Warehouse", + "Retail", + "Data Lake", + "Database", + "Marketing Automation", + "CRM", + "Ad-Tech", + "Team Collaboration", + "Productivity Tools", + "Payments", + "File Storage", + "HTTP", + "Customer Support" + ].freeze + + AI_ML_CATEGORIES = [ + "AI Model" + ].freeze + + def call + context.connectors = Multiwoven::Integrations::Service.connectors.with_indifferent_access + + filter_connectors_by_category if context.category + context.connectors = context.connectors[context.type] if context.type + end + + private + + def filter_connectors_by_category + categories = case context.category + when "data" + DATA_CATEGORIES + when "ai_ml" + AI_ML_CATEGORIES + else + [context.category] + end + context.connectors[:source] = filter_by_category(context.connectors[:source], categories) + context.connectors[:destination] = filter_by_category(context.connectors[:destination], categories) + end + + def filter_by_category(connectors, categories) + connectors.select { |connector| categories.include?(connector[:category]) } + end + end +end diff --git a/server/spec/interactors/connector_definitions/filter_connectors_spec.rb b/server/spec/interactors/connector_definitions/filter_connectors_spec.rb new file mode 100644 index 00000000..a7597e86 --- /dev/null +++ b/server/spec/interactors/connector_definitions/filter_connectors_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe ConnectorDefinitions::FilterConnectors, type: :interactor do + let(:connectors) do + { + source: [ + { name: "Snowflake", category: "Data Warehouse", connector_type: "source" }, + { name: "Redshift", category: "Data Warehouse", connector_type: "source" }, + { name: "DatabricksModel", category: "AI Model", connector_type: "source" } + ], + destination: [ + { name: "Klaviyo", category: "Marketing Automation", connector_type: "destination" }, + { name: "SalesforceCrm", category: "CRM", connector_type: "destination" }, + { name: "FacebookCustomAudience", category: "Ad-Tech", connector_type: "destination" } + ] + }.with_indifferent_access + end + + before do + allow(Multiwoven::Integrations::Service).to receive(:connectors).and_return(connectors) + end + + describe "#call" do + context "when filtering by type only" do + it "returns only the destination connectors when type is destination" do + result = described_class.call(type: "destination") + + expect(result.connectors).to match_array(connectors[:destination]) + end + + it "returns only the source connectors when type is source" do + result = described_class.call(type: "source") + + expect(result.connectors).to match_array(connectors[:source]) + end + end + + context "when filtering by category and type" do + it "returns only the AI Model connectors for source type" do + result = described_class.call(type: "source", category: "ai_ml") + + expect(result.connectors).to match_array([connectors[:source][2]]) + end + + it "returns only the Data Warehouse connectors for source type" do + result = described_class.call(type: "source", category: "data") + + expect(result.connectors).to match_array([connectors[:source][0], connectors[:source][1]]) + end + + it "returns only the Marketing Automation connectors for destination type" do + result = described_class.call(type: "destination", category: "data") + + expect(result.connectors).to match_array([connectors[:destination][0], connectors[:destination][1], + connectors[:destination][2]]) + end + end + + context "when no category is specified" do + it "returns all connectors of the specified type" do + result = described_class.call(type: "destination") + + expect(result.connectors).to match_array(connectors[:destination]) + end + end + end +end From 8824e995689179f4ad8062fc9d6215798bc0e671 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:35:41 +0530 Subject: [PATCH 50/74] fix(CE): refresh catalog only when refresh is true (#336) Co-authored-by: datafloyd --- .../app/interactors/connectors/discover_connector.rb | 2 +- .../connectors/discover_connector_spec.rb | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/server/app/interactors/connectors/discover_connector.rb b/server/app/interactors/connectors/discover_connector.rb index 66fe949e..295aeabe 100644 --- a/server/app/interactors/connectors/discover_connector.rb +++ b/server/app/interactors/connectors/discover_connector.rb @@ -7,7 +7,7 @@ class DiscoverConnector def call context.catalog = context.connector.catalog # refresh catalog when the refresh flag is true - return if context.catalog.present? && !context.refresh + return if context.catalog.present? && context.refresh != "true" catalog = context.connector.build_catalog( workspace_id: context.connector.workspace_id diff --git a/server/spec/interactors/connectors/discover_connector_spec.rb b/server/spec/interactors/connectors/discover_connector_spec.rb index 3a68b5b9..9429a710 100644 --- a/server/spec/interactors/connectors/discover_connector_spec.rb +++ b/server/spec/interactors/connectors/discover_connector_spec.rb @@ -8,7 +8,15 @@ let(:connector_client) { double("ConnectorClient") } let(:streams) { "stream_test_data" } - context "when catalog is already present" do + context "when catalog is already present with refresh param present" do + it "returns existing catalog" do + catalog = create(:catalog, connector:) + result = described_class.call(connector:, refresh: "false") + expect(result.catalog).to eq(catalog) + end + end + + context "when catalog is already present with refresh param absent" do it "returns existing catalog" do catalog = create(:catalog, connector:) result = described_class.call(connector:) @@ -20,7 +28,7 @@ it "returns refreshed catalog" do catalog = create(:catalog, connector:) allow_any_instance_of(described_class).to receive(:streams).and_return(streams) - result = described_class.call(connector:, refresh: true) + result = described_class.call(connector:, refresh: "true") expect(result.catalog).not_to eq(catalog) end end From c6dcd060e66393ad73111218ac2f9de077089f77 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Tue, 27 Aug 2024 04:40:35 -0400 Subject: [PATCH 51/74] Multiwoven release v0.22.0 (#339) Co-authored-by: github-actions --- release-notes.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/release-notes.md b/release-notes.md index fdaa3e73..35a1828a 100644 --- a/release-notes.md +++ b/release-notes.md @@ -2,27 +2,30 @@ All notable changes to this project will be documented in this file. -## [0.21.0] - 2024-08-19 +## [0.22.0] - 2024-08-26 ### 🚀 Features -- *(CE)* Destination/microsoft excel (#314) +- *(CE)* Enable/disable sync (#321) +- *(CE)* Databricks AI Model connector (#325) +- *(CE)* Catalog creation via API (#329) +- *(CE)* Calalog update api (#333) +- *(CE)* Add AI/ML query type to model (#334) ### 🐛 Bug Fixes -- *(CE)* Memory bloat issue in sync (#300) -- *(CE)* Fix discover and table url (#316) +- *(CE)* Correction in databricks model connection_spec (#331) ### 🚜 Refactor -- *(CE)* Added disable to fields +- *(CE)* Changed condition to render toast (#315) ### ⚙️ Miscellaneous Tasks -- *(CE)* Update the query_source response (#305) -- *(CE)* Fix oci8 version (#313) -- *(CE)* Update user read permission (#320) -- *(CE)* Update server gem (#312) -- *(CE)* Update connector name (#317) +- *(CE)* Add request response log for Hubspot (#323) +- *(CE)* Add request response log for Stripe (#328) +- *(CE)* Add request response log for SalesforceCrm (#326) +- *(CE)* Upgrade version to 0.9.1 (#330) +- *(CE)* Refactor error message to agreed standard (#332) From 24463f7805ed46f4034abaf9a131956a9433864b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:24:46 +0400 Subject: [PATCH 52/74] Resolve conflict in cherry-pick of 9a4860287a9bda117c8096cdd33ca37e823de242 and change the commit message (#337) Co-authored-by: datafloyd Co-authored-by: afthab vp --- .../v1/connector_definitions_controller.rb | 2 +- .../api/v1/connector_definitions_spec.rb | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/server/app/controllers/api/v1/connector_definitions_controller.rb b/server/app/controllers/api/v1/connector_definitions_controller.rb index 5298de25..a4c9491c 100644 --- a/server/app/controllers/api/v1/connector_definitions_controller.rb +++ b/server/app/controllers/api/v1/connector_definitions_controller.rb @@ -48,7 +48,7 @@ def set_connector_client end def connection_definitions_params - params.permit(:type, :catagory) + params.permit(:type, :category) end end end diff --git a/server/spec/requests/api/v1/connector_definitions_spec.rb b/server/spec/requests/api/v1/connector_definitions_spec.rb index 42d8e008..aa3956ab 100644 --- a/server/spec/requests/api/v1/connector_definitions_spec.rb +++ b/server/spec/requests/api/v1/connector_definitions_spec.rb @@ -33,6 +33,34 @@ expect(response_hash[:destination].count).to eql(service.connectors[:destination].count) end + it "returns only ai/ml sources" do + get "/api/v1/connector_definitions?type=source&category=ai_ml", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + + response_hash = JSON.parse(response.body) + + categories = response_hash.map { |item| item["category"] }.uniq + connector_types = response_hash.map { |item| item["connector_type"] }.uniq + expect(categories.count).to eql(1) + expect(connector_types.count).to eql(1) + expect(categories).to eql(["AI Model"]) + expect(connector_types).to eql(["source"]) + end + + it "returns only ai/ml connectors" do + get "/api/v1/connector_definitions?category=ai_ml", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + + response_hash = JSON.parse(response.body).with_indifferent_access + + source_categories = response_hash[:source].map { |item| item["category"] }.uniq + destination_categories = response_hash[:destination].map { |item| item["category"] }.uniq + + categories = (source_categories + destination_categories).uniq + expect(categories.count).to eql(1) + expect(categories).to eql(["AI Model"]) + end + it "returns success viewer role" do workspace.workspace_users.first.update(role: viewer_role) get "/api/v1/connector_definitions", headers: auth_headers(user, workspace_id) From f89f3638433d4027487c5126753c71559971628c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:39:46 +0530 Subject: [PATCH 53/74] chore(CE): validate catalog for query source (#340) Co-authored-by: datafloyd --- .../api/v1/connectors_controller.rb | 12 +++++++ .../api/v1/connectors_controller_spec.rb | 32 +++++++++++++++---- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/server/app/controllers/api/v1/connectors_controller.rb b/server/app/controllers/api/v1/connectors_controller.rb index b26fe46b..fe33b258 100644 --- a/server/app/controllers/api/v1/connectors_controller.rb +++ b/server/app/controllers/api/v1/connectors_controller.rb @@ -2,11 +2,13 @@ module Api module V1 + # rubocop:disable Metrics/ClassLength class ConnectorsController < ApplicationController include Connectors before_action :set_connector, only: %i[show update destroy discover query_source] # TODO: Enable this once we have query validation implemented for all the connectors # before_action :validate_query, only: %i[query_source] + before_action :validate_catalog, only: %i[query_source] after_action :event_logger def index @@ -121,6 +123,15 @@ def set_connector ) end + def validate_catalog + return if @connector.catalog.present? + + render_error( + message: "Catalog is not present for the connector", + status: :unprocessable_entity + ) + end + def validate_query Utils::QueryValidator.validate_query(@connector.connector_query_type, params[:query]) rescue StandardError => e @@ -137,5 +148,6 @@ def connector_params configuration: {}) end end + # rubocop:enable Metrics/ClassLength end end diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb index cf970563..fd430c89 100644 --- a/server/spec/requests/api/v1/connectors_controller_spec.rb +++ b/server/spec/requests/api/v1/connectors_controller_spec.rb @@ -406,7 +406,7 @@ end describe "POST /api/v1/connectors/id/query_source" do - let(:connector) { create(:connector, connector_type: "source") } + let(:connector) { create(:connector, connector_type: "source", workspace:, connector_name: "AmazonS3") } let(:query) { "SELECT * FROM table_name" } let(:limit) { 50 } let(:record1) do @@ -424,9 +424,13 @@ } end + before do + create(:catalog, connector_id: connector.id, workspace:) + end + context "when it is an unauthenticated user" do it "returns unauthorized" do - post "/api/v1/connectors/#{connectors.second.id}/query_source" + post "/api/v1/connectors/#{connector.id}/query_source" expect(response).to have_http_status(:unauthorized) end end @@ -435,7 +439,7 @@ it "returns success status for a valid query" do allow(Connectors::QuerySource).to receive(:call) .and_return(double(:context, success?: true, records: [record1, record2])) - post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access @@ -446,18 +450,32 @@ workspace.workspace_users.first.update(role: member_role) allow(Connectors::QuerySource).to receive(:call) .and_return(double(:context, success?: true, records: [record1, record2])) - post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) end + it "returns an error message for missing catalog" do + catalog = connector.catalog + catalog.connector_id = connectors.second.id + catalog.save + + allow(Connectors::QuerySource).to receive(:call).and_return(double(:context, success?: true, + records: [record1, record2])) + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: + { "Content-Type": "application/json" }.merge(auth_headers(user, workspace.id)) + expect(response).to have_http_status(:unprocessable_entity) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash.dig(:errors, 0, :detail)).to eq("Catalog is not present for the connector") + end + it "returns success status for a valid query for viewer role" do workspace.workspace_users.first.update(role: viewer_role) allow(Connectors::QuerySource).to receive(:call) .and_return(double(:context, success?: true, records: [record1, record2])) - post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access @@ -467,7 +485,7 @@ it "returns failure status for a invalid query" do allow(Connectors::QuerySource).to receive(:call).and_raise(StandardError, "query failed") - post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:bad_request) @@ -475,7 +493,7 @@ it "returns failure status for a invalid query" do request_body[:query] = "invalid" - post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:unprocessable_entity) From b93a8e2638f9c251eed2943b2a2c2569d698e90f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:40:29 +0530 Subject: [PATCH 54/74] chore(CE): add catalog presence validation for models (#341) Co-authored-by: datafloyd --- .../controllers/api/v1/models_controller.rb | 11 ++++++++ .../requests/api/v1/models_controller_spec.rb | 28 ++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/server/app/controllers/api/v1/models_controller.rb b/server/app/controllers/api/v1/models_controller.rb index f39dadfa..6b2063e6 100644 --- a/server/app/controllers/api/v1/models_controller.rb +++ b/server/app/controllers/api/v1/models_controller.rb @@ -8,6 +8,7 @@ class ModelsController < ApplicationController before_action :set_connector, only: %i[create] before_action :set_model, only: %i[show update destroy] + before_action :validate_catalog, only: %i[create update] # TODO: Enable this once we have query validation implemented for all the connectors # before_action :validate_query, only: %i[create update] after_action :event_logger @@ -75,6 +76,16 @@ def set_connector def set_model @model = current_workspace.models.find(params[:id]) + @connector = @model.connector + end + + def validate_catalog + return if connector.catalog.present? + + render_error( + message: "Catalog is not present for the connector", + status: :unprocessable_entity + ) end def validate_query diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb index 7ac61917..3301e72f 100644 --- a/server/spec/requests/api/v1/models_controller_spec.rb +++ b/server/spec/requests/api/v1/models_controller_spec.rb @@ -9,6 +9,10 @@ let(:connector) do create(:connector, workspace:, connector_type: "destination", name: "klavio1", connector_name: "Klaviyo") end + + let(:connector_without_catalog) do + create(:connector, workspace:, connector_type: "destination", name: "klavio1", connector_name: "Klaviyo") + end let!(:models) do [ create(:model, connector:, workspace:, name: "model1", query: "SELECT * FROM locations"), @@ -19,6 +23,7 @@ let(:member_role) { create(:role, :member) } before do + create(:catalog, connector:) user.confirm end @@ -152,7 +157,16 @@ expect(response_hash.dig(:data, :attributes, :primary_key)).to eq(request_body.dig(:model, :primary_key)) end - it "creates a new model and returns success" do + it "fails model creation for connector without catalog" do + workspace.workspace_users.first.update(role: member_role) + # set connector without catalog + request_body[:model][:connector_id] = connector_without_catalog.id + post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:unprocessable_entity) + end + + it " creates a new model and returns success" do workspace.workspace_users.first.update(role: member_role) post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" } .merge(auth_headers(user, workspace_id)) @@ -255,6 +269,18 @@ expect(response_hash.dig(:data, :attributes, :primary_key)).to eq(request_body.dig(:model, :primary_key)) end + it "fails model update for connector without catalog" do + workspace.workspace_users.first.update(role: member_role) + model = models.second + model.connector_id = connector_without_catalog.id + model.save! + + put "/api/v1/models/#{models.second.id}", params: request_body.to_json, + headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:unprocessable_entity) + end + it "updates the model and returns success for member role" do workspace.workspace_users.first.update(role: member_role) put "/api/v1/models/#{models.second.id}", params: request_body.to_json, headers: From 684fa4610c78bc9b584011d3cc2024a62abe27d0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:40:59 +0530 Subject: [PATCH 55/74] fix(CE): connection_spec mandatory changes (#342) Co-authored-by: afthab vp --- integrations/Gemfile.lock | 2 +- integrations/lib/multiwoven/integrations/rollout.rb | 2 +- .../integrations/source/databrics_model/config/spec.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 160d7ed9..0bcecf8b 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.9.1) + multiwoven-integrations (0.9.2) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index a2409100..1447eb3a 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.9.1" + VERSION = "0.9.2" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json b/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json index 9fb6ddac..40d6b9ec 100644 --- a/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json +++ b/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json @@ -6,7 +6,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Databricks Model", "type": "object", - "required": ["databricks_host", "token", "endpoint"], + "required": ["databricks_host", "token", "endpoint", "request_format", "response_format"], "properties": { "databricks_host": { "title": "databricks_host", From 4c900da0791447860641fde49e44425faea793df Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 22:59:28 +0530 Subject: [PATCH 56/74] fix(CE): added tag label to auto-truncate text --- ui/src/components/StatusTag/StatusTag.tsx | 40 +++++++++++------------ 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/ui/src/components/StatusTag/StatusTag.tsx b/ui/src/components/StatusTag/StatusTag.tsx index 7671bc2e..a6fe73dc 100644 --- a/ui/src/components/StatusTag/StatusTag.tsx +++ b/ui/src/components/StatusTag/StatusTag.tsx @@ -1,4 +1,4 @@ -import { Tag, Text } from '@chakra-ui/react'; +import { Tag, TagLabel } from '@chakra-ui/react'; export enum StatusTagVariants { success = 'success', @@ -79,25 +79,23 @@ export const StatusTagText = { failed: 'Failed', }; -const StatusTag = ({ status, variant = StatusTagVariants.success }: StatusTagProps) => { - return ( - - - {status} - - - ); -}; +const StatusTag = ({ status, variant = StatusTagVariants.success }: StatusTagProps) => ( + + + {status} + + +); export default StatusTag; From 7dae4f070c66b50a2d8eac2737bdf0c501636381 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:49:05 +0530 Subject: [PATCH 57/74] feat(CE): filter connectors (#343) Co-authored-by: Subin T P --- .../api/v1/connectors_controller.rb | 1 + .../api/v1/connectors_controller_spec.rb | 34 ++++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/server/app/controllers/api/v1/connectors_controller.rb b/server/app/controllers/api/v1/connectors_controller.rb index fe33b258..77bbcd08 100644 --- a/server/app/controllers/api/v1/connectors_controller.rb +++ b/server/app/controllers/api/v1/connectors_controller.rb @@ -15,6 +15,7 @@ def index @connectors = current_workspace.connectors authorize @connectors @connectors = @connectors.send(params[:type].downcase) if params[:type] + @connectors = @connectors.send(params[:category].downcase) if params[:category] @connectors = @connectors.page(params[:page] || 1) render json: @connectors, status: :ok end diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb index fd430c89..e5f194b1 100644 --- a/server/spec/requests/api/v1/connectors_controller_spec.rb +++ b/server/spec/requests/api/v1/connectors_controller_spec.rb @@ -14,6 +14,9 @@ create(:connector, workspace:, connector_type: "source", name: "redshift", connector_name: "Redshift") ] end + let!(:data_connector) { create(:connector, workspace:, connector_category: "Data Warehouse") } + let!(:ai_ml_connector) { create(:connector, workspace:, connector_category: "AI Model") } + let!(:other_connector) { create(:connector, workspace:, connector_category: "CRM") } before do user.confirm @@ -33,7 +36,7 @@ get "/api/v1/connectors", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(connectors.count) + expect(response_hash[:data].count).to eql(connectors.count + 3) expect(response_hash.dig(:data, 0, :type)).to eq("connectors") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/connectors?page=1") end @@ -43,7 +46,7 @@ get "/api/v1/connectors", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(connectors.count) + expect(response_hash[:data].count).to eql(connectors.count + 3) expect(response_hash.dig(:data, 0, :type)).to eq("connectors") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/connectors?page=1") end @@ -53,7 +56,7 @@ get "/api/v1/connectors", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(connectors.count) + expect(response_hash[:data].count).to eql(connectors.count + 3) expect(response_hash.dig(:data, 0, :type)).to eq("connectors") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/connectors?page=1") end @@ -63,7 +66,7 @@ get "/api/v1/connectors", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(connectors.count) + expect(response_hash[:data].count).to eql(connectors.count + 3) expect(response_hash.dig(:data, 0, :type)).to eq("connectors") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/connectors?page=1") end @@ -82,12 +85,33 @@ get "/api/v1/connectors?type=destination", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(1) + expect(response_hash[:data].count).to eql(4) expect(response_hash.dig(:data, 0, :type)).to eq("connectors") expect(response_hash.dig(:data, 0, :attributes, :connector_type)).to eql("destination") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/connectors?page=1") end + it "returns only data connectors" do + get "/api/v1/connectors?category=data", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + result = JSON.parse(response.body) + expect(result["data"].map { |connector| connector["id"] }).not_to include(ai_ml_connector.id.to_s) + end + + it "returns only ai_ml connectors" do + get "/api/v1/connectors?category=ai_ml", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + result = JSON.parse(response.body) + expect(result["data"].map { |connector| connector["id"] }).to eql([ai_ml_connector.id.to_s]) + end + + it "returns only ai_ml connectors for source" do + get "/api/v1/connectors?type=source&&category=ai_ml", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + result = JSON.parse(response.body) + expect(result["data"].count).to eql(0) + end + it "returns an error response for connectors" do get "/api/v1/connectors?type=destination1", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:bad_request) From 33e490bfa9d3ad41127a750d5c4112ecdbf81a7d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:49:52 +0530 Subject: [PATCH 58/74] chore(CE): refactor interactors (#344) Co-authored-by: Subin T P --- .../v1/connector_definitions_controller.rb | 2 +- ...ors.rb => filter_connector_definitions.rb} | 29 ++----------------- ...b => filter_connector_definitions_spec.rb} | 2 +- 3 files changed, 5 insertions(+), 28 deletions(-) rename server/app/interactors/connector_definitions/{filter_connectors.rb => filter_connector_definitions.rb} (60%) rename server/spec/interactors/connector_definitions/{filter_connectors_spec.rb => filter_connector_definitions_spec.rb} (96%) diff --git a/server/app/controllers/api/v1/connector_definitions_controller.rb b/server/app/controllers/api/v1/connector_definitions_controller.rb index a4c9491c..ed901652 100644 --- a/server/app/controllers/api/v1/connector_definitions_controller.rb +++ b/server/app/controllers/api/v1/connector_definitions_controller.rb @@ -34,7 +34,7 @@ def check_connection private def set_connectors - @connectors = FilterConnectors.call( + @connectors = FilterConnectorDefinitions.call( connection_definitions_params ).connectors end diff --git a/server/app/interactors/connector_definitions/filter_connectors.rb b/server/app/interactors/connector_definitions/filter_connector_definitions.rb similarity index 60% rename from server/app/interactors/connector_definitions/filter_connectors.rb rename to server/app/interactors/connector_definitions/filter_connector_definitions.rb index 4d266f23..0ebbcfee 100644 --- a/server/app/interactors/connector_definitions/filter_connectors.rb +++ b/server/app/interactors/connector_definitions/filter_connector_definitions.rb @@ -1,32 +1,9 @@ # frozen_string_literal: true module ConnectorDefinitions - class FilterConnectors + class FilterConnectorDefinitions include Interactor - # TODO: Move this to integration so that whenever a new category is introduced, - # integrations will be the single source of truth for categories - - DATA_CATEGORIES = [ - "Data Warehouse", - "Retail", - "Data Lake", - "Database", - "Marketing Automation", - "CRM", - "Ad-Tech", - "Team Collaboration", - "Productivity Tools", - "Payments", - "File Storage", - "HTTP", - "Customer Support" - ].freeze - - AI_ML_CATEGORIES = [ - "AI Model" - ].freeze - def call context.connectors = Multiwoven::Integrations::Service.connectors.with_indifferent_access @@ -39,9 +16,9 @@ def call def filter_connectors_by_category categories = case context.category when "data" - DATA_CATEGORIES + Connector::DATA_CATEGORIES when "ai_ml" - AI_ML_CATEGORIES + Connector::AI_ML_CATEGORIES else [context.category] end diff --git a/server/spec/interactors/connector_definitions/filter_connectors_spec.rb b/server/spec/interactors/connector_definitions/filter_connector_definitions_spec.rb similarity index 96% rename from server/spec/interactors/connector_definitions/filter_connectors_spec.rb rename to server/spec/interactors/connector_definitions/filter_connector_definitions_spec.rb index a7597e86..f6b83014 100644 --- a/server/spec/interactors/connector_definitions/filter_connectors_spec.rb +++ b/server/spec/interactors/connector_definitions/filter_connector_definitions_spec.rb @@ -2,7 +2,7 @@ require "rails_helper" -RSpec.describe ConnectorDefinitions::FilterConnectors, type: :interactor do +RSpec.describe ConnectorDefinitions::FilterConnectorDefinitions, type: :interactor do let(:connectors) do { source: [ From b927e706b1c8a3cd6980de1fb007afbb0ea2856d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:50:44 +0530 Subject: [PATCH 59/74] feat(CE): filter model based on the query type (#347) Co-authored-by: datafloyd --- .../controllers/api/v1/models_controller.rb | 3 +- server/app/models/model.rb | 3 ++ server/spec/models/model_spec.rb | 25 ++++++++++++++ .../requests/api/v1/models_controller_spec.rb | 34 +++++++++++++++++-- 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/server/app/controllers/api/v1/models_controller.rb b/server/app/controllers/api/v1/models_controller.rb index 6b2063e6..421d9165 100644 --- a/server/app/controllers/api/v1/models_controller.rb +++ b/server/app/controllers/api/v1/models_controller.rb @@ -14,8 +14,9 @@ class ModelsController < ApplicationController after_action :event_logger def index + filter = params[:query_type] || "all" @models = current_workspace - .models.all.page(params[:page] || 1) + .models.send(filter).page(params[:page] || 1) authorize @models render json: @models, status: :ok end diff --git a/server/app/models/model.rb b/server/app/models/model.rb index 372effd3..f7db2832 100644 --- a/server/app/models/model.rb +++ b/server/app/models/model.rb @@ -30,6 +30,9 @@ class Model < ApplicationRecord has_many :syncs, dependent: :destroy + scope :data, -> { where(query_type: %i[raw_sql dbt soql table_selector]) } + scope :ai_ml, -> { where(query_type: :ai_ml) } + default_scope { order(updated_at: :desc) } def to_protocol Multiwoven::Integrations::Protocol::Model.new( diff --git a/server/spec/models/model_spec.rb b/server/spec/models/model_spec.rb index 10d28184..1d7d23a1 100644 --- a/server/spec/models/model_spec.rb +++ b/server/spec/models/model_spec.rb @@ -130,4 +130,29 @@ expect(Model.query_types).to eq({ "raw_sql" => 0, "dbt" => 1, "soql" => 2, "table_selector" => 3, "ai_ml" => 4 }) end end + + describe "scopes" do + let(:source) { create(:connector, connector_type: "source", connector_name: "Snowflake") } + let!(:raw_sql_model) { create(:model, query_type: :raw_sql, connector: source) } + let!(:dbt_model) { create(:model, query_type: :dbt, connector: source) } + let!(:soql_model) { create(:model, query_type: :soql, connector: source) } + let!(:table_selector_model) { create(:model, query_type: :table_selector, connector: source) } + let!(:ai_ml_model) { create(:model, query_type: :ai_ml, connector: source, configuration: { test: "value" }) } + + describe ".data" do + it "returns models with query_type in [raw_sql, dbt, soql, table_selector]" do + data_models = Model.data + expect(data_models).to include(raw_sql_model, dbt_model, soql_model, table_selector_model) + expect(data_models).not_to include(ai_ml_model) + end + end + + describe ".ai_ml" do + it "returns models with query_type equal to ai_ml" do + ai_ml_models = Model.ai_ml + expect(ai_ml_models).to include(ai_ml_model) + expect(ai_ml_models).not_to include(raw_sql_model, dbt_model, soql_model, table_selector_model) + end + end + end end diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb index 3301e72f..c05177a5 100644 --- a/server/spec/requests/api/v1/models_controller_spec.rb +++ b/server/spec/requests/api/v1/models_controller_spec.rb @@ -19,6 +19,13 @@ create(:model, connector:, workspace:, name: "model2", query: "SELECT * FROM locations") ] end + + let!(:raw_sql_model) { create(:model, query_type: :raw_sql, connector:, workspace:) } + let!(:dbt_model) { create(:model, query_type: :dbt, connector:, workspace:) } + let!(:soql_model) { create(:model, query_type: :soql, connector:, workspace:) } + let!(:ai_ml_model) do + create(:model, query_type: :ai_ml, connector:, configuration: { key: "value" }, workspace:) + end let(:viewer_role) { create(:role, :viewer) } let(:member_role) { create(:role, :member) } @@ -40,7 +47,7 @@ get "/api/v1/models", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(models.count) + expect(response_hash[:data].count).to eql(6) expect(response_hash.dig(:data, 0, :type)).to eq("models") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1") end @@ -50,7 +57,7 @@ get "/api/v1/models", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(models.count) + expect(response_hash[:data].count).to eql(6) expect(response_hash.dig(:data, 0, :type)).to eq("models") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1") end @@ -60,10 +67,31 @@ get "/api/v1/models", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(models.count) + expect(response_hash[:data].count).to eql(6) expect(response_hash.dig(:data, 0, :type)).to eq("models") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1") end + + it "filters models based on the query_type parameter" do + get "/api/v1/models?query_type=data", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + expect(JSON.parse(response.body)["data"].map { |m| m["id"] }).not_to include(ai_ml_model.id) + end + + it "filters models based on a different query_type" do + get "/api/v1/models?query_type=ai_ml", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + expect(JSON.parse(response.body)["data"].map do |m| + m["id"] + end).not_to include(raw_sql_model.id, dbt_model.id, soql_model.id) + end + + it "returns all models" do + get "/api/v1/models", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + model_ids = JSON.parse(response.body)["data"].map { |m| m["id"] } + expect(model_ids.count).to eql(6) + end end end From 9b4a53d39cb0e84441b9f0245e786a379ed4bfbe Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 14:32:05 +0530 Subject: [PATCH 60/74] chore(CE): add filtering scope to connectors (#345) Co-authored-by: Subin T P --- server/app/models/connector.rb | 43 ++++++++++++++++++ ...2_populate_category_field_in_connectors.rb | 20 +++++++++ server/db/data_schema.rb | 2 +- ...003_add_connector_category_to_connector.rb | 7 +++ server/db/schema.rb | 4 +- server/spec/models/connector_spec.rb | 44 +++++++++++++++++++ 6 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 server/db/data/20240827212112_populate_category_field_in_connectors.rb create mode 100644 server/db/migrate/20240827211003_add_connector_category_to_connector.rb diff --git a/server/app/models/connector.rb b/server/app/models/connector.rb index 9b86d5e4..c2ee3c36 100644 --- a/server/app/models/connector.rb +++ b/server/app/models/connector.rb @@ -36,6 +36,36 @@ class Connector < ApplicationRecord default_scope { order(updated_at: :desc) } + before_save :set_category + before_update :set_category, if: :will_save_change_to_connector_name? + + DEFAULT_CONNECTOR_CATEGORY = "data" + + # TODO: Move this to integrations gem + DATA_CATEGORIES = [ + "Data Warehouse", + "Retail", + "Data Lake", + "Database", + "Marketing Automation", + "CRM", + "Ad-Tech", + "Team Collaboration", + "Productivity Tools", + "Payments", + "File Storage", + "HTTP", + "Customer Support", + "data" + ].freeze + + AI_ML_CATEGORIES = [ + "AI Model" + ].freeze + + scope :ai_ml, -> { where(connector_category: AI_ML_CATEGORIES) } + scope :data, -> { where(connector_category: DATA_CATEGORIES) } + def connector_definition @connector_definition ||= connector_client.new.meta_data.with_indifferent_access end @@ -90,4 +120,17 @@ def connector_query_type def pull_catalog connector_client.new.discover(configuration).catalog.to_h.with_indifferent_access end + + def set_category + unless connector_category.present? && + connector_category == DEFAULT_CONNECTOR_CATEGORY && + !will_save_change_to_connector_category? + return + end + + category_name = connector_client.new.meta_data[:data][:category] + self.connector_category = category_name if category_name.present? + rescue StandardError => e + Rails.logger.error("Failed to set category for connector ##{id}: #{e.message}") + end end diff --git a/server/db/data/20240827212112_populate_category_field_in_connectors.rb b/server/db/data/20240827212112_populate_category_field_in_connectors.rb new file mode 100644 index 00000000..3e1a5f98 --- /dev/null +++ b/server/db/data/20240827212112_populate_category_field_in_connectors.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class PopulateCategoryFieldInConnectors < ActiveRecord::Migration[7.1] + def up + Connector.find_each do |connector| + category_name = connector.connector_client.new.meta_data[:data][:category] + connector.update!(connector_category: category_name) if category_name.present? + rescue StandardError => e + Rails.logger.error("Failed to update connector ##{connector.id}: #{e.message}") + end + end + + def down + Connector.find_each do |connector| + connector.update!(connector_category: "data") + rescue StandardError => e + Rails.logger.error("Failed to revert connector ##{connector.id} to 'data': #{e.message}") + end + end +end diff --git a/server/db/data_schema.rb b/server/db/data_schema.rb index 9c182f0d..f54a3d06 100644 --- a/server/db/data_schema.rb +++ b/server/db/data_schema.rb @@ -1,3 +1,3 @@ # frozen_string_literal: true -DataMigrate::Data.define(version: 20_240_813_084_446) +DataMigrate::Data.define(version: 20_240_827_212_112) diff --git a/server/db/migrate/20240827211003_add_connector_category_to_connector.rb b/server/db/migrate/20240827211003_add_connector_category_to_connector.rb new file mode 100644 index 00000000..a16ddd10 --- /dev/null +++ b/server/db/migrate/20240827211003_add_connector_category_to_connector.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AddConnectorCategoryToConnector < ActiveRecord::Migration[7.1] + def change + add_column :connectors, :connector_category, :string, null: false, default: "data" + end +end diff --git a/server/db/schema.rb b/server/db/schema.rb index e1967009..c6672c4f 100644 --- a/server/db/schema.rb +++ b/server/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_08_23_114746) do +ActiveRecord::Schema[7.1].define(version: 2024_08_27_211003) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -33,6 +33,7 @@ t.datetime "updated_at", null: false t.string "connector_name" t.string "description" + t.string "connector_category", default: "data", null: false end create_table "data_migrations", primary_key: "version", id: :string, force: :cascade do |t| @@ -132,6 +133,7 @@ t.string "cursor_field" t.string "current_cursor_field" t.string "cron_expression" + t.string "name" t.index ["discarded_at"], name: "index_syncs_on_discarded_at" end diff --git a/server/spec/models/connector_spec.rb b/server/spec/models/connector_spec.rb index 4c7ac76e..fd814aaa 100644 --- a/server/spec/models/connector_spec.rb +++ b/server/spec/models/connector_spec.rb @@ -164,4 +164,48 @@ end end end + + describe "#set_category" do + let(:workspace) { create(:workspace) } + let(:connector) do + create(:connector, + workspace:) + end + + context "populate category from connector meta data" do + it "sets the connector_category based on the meta_data" do + connector.run_callbacks(:save) { true } + expect(connector.connector_category).to eq("Marketing Automation") + end + end + + context "catagory is set by user" do + it "does not change the connector_category" do + connector.update!(connector_category: "user_input_category") + expect(connector.connector_category).to eq("user_input_category") + end + end + end + + describe ".ai_ml" do + let!(:ai_ml_connector) { create(:connector, connector_category: "AI Model") } + let!(:non_ai_ml_connector) { create(:connector, connector_category: "Data Warehouse") } + + it "returns connectors with connector_category in AI_ML_CATEGORIES" do + result = Connector.ai_ml + expect(result).to include(ai_ml_connector) + expect(result).not_to include(non_ai_ml_connector) + end + end + + describe ".data" do + let!(:data_connector) { create(:connector, connector_category: "Data Warehouse") } + let!(:non_data_connector) { create(:connector, connector_category: "AI Model") } + + it "returns connectors with connector_category in DATA_CATEGORIES" do + result = Connector.data + expect(result).to include(data_connector) + expect(result).not_to include(non_data_connector) + end + end end From 7e304c546c152548ce32a2d80f348836c347a95f Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:00:13 -0400 Subject: [PATCH 61/74] refactor(CE): created common connector lists component --- ui/src/components/DataTable/RowsNotFound.tsx | 13 +++ .../Activate/Syncs/SyncRuns/SyncRuns.tsx | 18 +--- .../ConnectorsListColumns.tsx | 42 +++++++++ .../DestinationsList/DestinationsList.tsx | 47 +++++++--- .../Sources/SourcesList/SourcesList.tsx | 93 ++++++------------- ui/src/views/Connectors/types.ts | 3 + 6 files changed, 124 insertions(+), 92 deletions(-) create mode 100644 ui/src/components/DataTable/RowsNotFound.tsx create mode 100644 ui/src/views/Connectors/ConnectorsListColumns/ConnectorsListColumns.tsx diff --git a/ui/src/components/DataTable/RowsNotFound.tsx b/ui/src/components/DataTable/RowsNotFound.tsx new file mode 100644 index 00000000..df269f56 --- /dev/null +++ b/ui/src/components/DataTable/RowsNotFound.tsx @@ -0,0 +1,13 @@ +import EmptyState from '@/assets/images/empty-state-illustration.svg'; +import { Box, Image, Text } from '@chakra-ui/react'; + +const RowsNotFound = () => ( + + + + No rows found + + +); + +export default RowsNotFound; diff --git a/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx b/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx index cd0aa12d..9029ec05 100644 --- a/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx +++ b/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx @@ -2,12 +2,12 @@ import { useQuery } from '@tanstack/react-query'; import { useNavigate, useParams, useSearchParams } from 'react-router-dom'; import { getSyncRunsBySyncId } from '@/services/syncs'; import { useMemo, useState, useEffect } from 'react'; -import { Box, Image, Text } from '@chakra-ui/react'; +import { Box } from '@chakra-ui/react'; import Loader from '@/components/Loader'; import Pagination from '@/components/Pagination'; import { SyncRunsColumns } from './SyncRunsColumns'; import DataTable from '@/components/DataTable'; -import SyncRunEmptyImage from '@/assets/images/empty-state-illustration.svg'; +import RowsNotFound from '@/components/DataTable/RowsNotFound'; const SyncRuns = () => { const { syncId } = useParams(); @@ -51,19 +51,7 @@ const SyncRuns = () => { ) : ( {data?.data?.length === 0 || !data?.data ? ( - - - - No rows found - - + ) : ( )} diff --git a/ui/src/views/Connectors/ConnectorsListColumns/ConnectorsListColumns.tsx b/ui/src/views/Connectors/ConnectorsListColumns/ConnectorsListColumns.tsx new file mode 100644 index 00000000..a39e2a1e --- /dev/null +++ b/ui/src/views/Connectors/ConnectorsListColumns/ConnectorsListColumns.tsx @@ -0,0 +1,42 @@ +import { ConnectorItem } from '../types'; +import { ColumnDef } from '@tanstack/react-table'; +import { Text } from '@chakra-ui/react'; +import EntityItem from '@/components/EntityItem'; +import StatusTag from '@/components/StatusTag'; +import dayjs from 'dayjs'; + +export const ConnectorsListColumns: ColumnDef[] = [ + { + accessorKey: 'attributes.name', + header: () => Name, + cell: (info) => ( + + {info.getValue() as string} + + ), + }, + { + accessorKey: 'attributes', + header: () => Type, + cell: (info) => ( + + ), + }, + { + accessorKey: 'attributes.updated_at', + header: () => Updated At, + cell: (info) => ( + + {dayjs(info.getValue() as number).format('DD/MM/YY')} + + ), + }, + { + accessorKey: 'attributes.status', + header: () => Status, + cell: () => , + }, +]; diff --git a/ui/src/views/Connectors/Destinations/DestinationsList/DestinationsList.tsx b/ui/src/views/Connectors/Destinations/DestinationsList/DestinationsList.tsx index 2b2442e7..f141296d 100644 --- a/ui/src/views/Connectors/Destinations/DestinationsList/DestinationsList.tsx +++ b/ui/src/views/Connectors/Destinations/DestinationsList/DestinationsList.tsx @@ -1,29 +1,53 @@ import { Box } from '@chakra-ui/react'; import { FiPlus } from 'react-icons/fi'; import TopBar from '@/components/TopBar'; -import { useNavigate } from 'react-router-dom'; import ContentContainer from '@/components/ContentContainer'; -import DestinationsTable from './DestinationsTable'; import { useQuery } from '@tanstack/react-query'; import { DESTINATIONS_LIST_QUERY_KEY } from '../../constant'; import { getUserConnectors } from '@/services/connectors'; import NoConnectors from '../../NoConnectors'; import Loader from '@/components/Loader'; +import { useStore } from '@/stores'; +import useCustomToast from '@/hooks/useCustomToast'; +import { CustomToastStatus } from '@/components/Toast/index'; +import titleCase from '@/utils/TitleCase'; +import DataTable from '@/components/DataTable'; +import { ConnectorsListColumns } from '@/views/Connectors/ConnectorsListColumns/ConnectorsListColumns'; +import { useNavigate } from 'react-router-dom'; const DestinationsList = (): JSX.Element | null => { + const showToast = useCustomToast(); + + const activeWorkspaceId = useStore((state) => state.workspaceId); + const navigate = useNavigate(); const { data, isLoading } = useQuery({ - queryKey: DESTINATIONS_LIST_QUERY_KEY, + queryKey: [...DESTINATIONS_LIST_QUERY_KEY, activeWorkspaceId], queryFn: () => getUserConnectors('destination'), refetchOnMount: true, refetchOnWindowFocus: false, + enabled: activeWorkspaceId > 0, }); if (isLoading && !data) return ; - if (data?.data.length === 0) return ; + if (data?.data?.length === 0 || !data) return ; + + if (data?.errors) { + data.errors?.forEach((error) => { + showToast({ + duration: 5000, + isClosable: true, + position: 'bottom-right', + colorScheme: 'red', + status: CustomToastStatus.Warning, + title: titleCase(error.detail), + }); + }); + return ; + } return ( @@ -36,17 +60,14 @@ const DestinationsList = (): JSX.Element | null => { ctaButtonVariant='solid' ctaButtonWidth='fit' ctaButtonHeight='40px' - isCtaVisible /> - {isLoading || !data ? ( - - ) : ( - navigate(`/setup/destinations/${row?.id}`)} - destinationData={data} - isLoading={isLoading} + + navigate(`/setup/destinations/${row?.original?.id}`)} /> - )} + ); diff --git a/ui/src/views/Connectors/Sources/SourcesList/SourcesList.tsx b/ui/src/views/Connectors/Sources/SourcesList/SourcesList.tsx index 07f1f512..6546bdfe 100644 --- a/ui/src/views/Connectors/Sources/SourcesList/SourcesList.tsx +++ b/ui/src/views/Connectors/Sources/SourcesList/SourcesList.tsx @@ -1,50 +1,21 @@ -import { useMemo } from 'react'; import { useQuery } from '@tanstack/react-query'; -import { Box, Text } from '@chakra-ui/react'; +import { Box } from '@chakra-ui/react'; import { FiPlus } from 'react-icons/fi'; import TopBar from '@/components/TopBar'; -import { useNavigate } from 'react-router-dom'; -import { SOURCES_LIST_QUERY_KEY, CONNECTOR_LIST_COLUMNS } from '@/views/Connectors/constant'; -import Table from '@/components/Table'; +import { SOURCES_LIST_QUERY_KEY } from '@/views/Connectors/constant'; import { getUserConnectors } from '@/services/connectors'; -import { ConnectorAttributes, ConnectorTableColumnFields } from '../../types'; -import moment from 'moment'; import ContentContainer from '@/components/ContentContainer'; -import EntityItem from '@/components/EntityItem'; import Loader from '@/components/Loader'; import NoConnectors from '@/views/Connectors/NoConnectors'; -import StatusTag from '@/components/StatusTag'; - -type TableItem = { - field: ConnectorTableColumnFields; - attributes: ConnectorAttributes; -}; - -const TableItem = ({ field, attributes }: TableItem): JSX.Element => { - switch (field) { - case 'icon': - return ; - - case 'updated_at': - return ( - - {moment(attributes?.updated_at).format('DD/MM/YY')} - - ); - - case 'status': - return ; - - default: - return ( - - {attributes?.[field]} - - ); - } -}; +import { CustomToastStatus } from '@/components/Toast/index'; +import titleCase from '@/utils/TitleCase'; +import { ConnectorsListColumns } from '@/views/Connectors/ConnectorsListColumns/ConnectorsListColumns'; +import DataTable from '@/components/DataTable'; +import { useNavigate } from 'react-router-dom'; +import useCustomToast from '@/hooks/useCustomToast'; const SourcesList = (): JSX.Element | null => { + const showToast = useCustomToast(); const navigate = useNavigate(); const { data, isLoading } = useQuery({ queryKey: SOURCES_LIST_QUERY_KEY, @@ -53,31 +24,23 @@ const SourcesList = (): JSX.Element | null => { refetchOnWindowFocus: false, }); - const connectors = data?.data; - - const tableData = useMemo(() => { - if (connectors && connectors?.length > 0) { - const rows = connectors.map(({ attributes, id }) => { - return CONNECTOR_LIST_COLUMNS.reduce( - (acc, { key }) => ({ - [key]: , - id, - ...acc, - }), - {}, - ); - }); - - return { - columns: CONNECTOR_LIST_COLUMNS, - data: rows, - }; - } - }, [data]); - if (isLoading) return ; - if (!isLoading && !tableData) return ; + if (data?.data?.length === 0 || !data) return ; + + if (data?.errors) { + data.errors?.forEach((error) => { + showToast({ + duration: 5000, + isClosable: true, + position: 'bottom-right', + colorScheme: 'red', + status: CustomToastStatus.Warning, + title: titleCase(error.detail), + }); + }); + return ; + } return ( @@ -90,9 +53,11 @@ const SourcesList = (): JSX.Element | null => { ctaButtonVariant='solid' isCtaVisible /> - {tableData ? ( -
navigate(`/setup/sources/${row?.id}`)} /> - ) : null} + navigate(`/setup/sources/${row?.original?.id}`)} + /> ); diff --git a/ui/src/views/Connectors/types.ts b/ui/src/views/Connectors/types.ts index c0498c97..baf13571 100644 --- a/ui/src/views/Connectors/types.ts +++ b/ui/src/views/Connectors/types.ts @@ -1,3 +1,5 @@ +import { ErrorResponse } from '@/services/common'; + export type ConnectorTypes = 'source' | 'destination'; export type DatasourceType = { @@ -95,6 +97,7 @@ export type ConnectorInfoResponse = { export type ConnectorListResponse = { data: ConnectorItem[]; + errors?: ErrorResponse[]; }; export type ConnectorTableColumnFields = From 735eb43a75884fd0e8225fe8a15cd02489cf61c6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 09:53:29 +0530 Subject: [PATCH 62/74] fix(CE): request log redaction (#348) Co-authored-by: afthab vp --- .../app/middleware/multiwoven_server/request_response_logger.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app/middleware/multiwoven_server/request_response_logger.rb b/server/app/middleware/multiwoven_server/request_response_logger.rb index abf3d352..93b9b179 100644 --- a/server/app/middleware/multiwoven_server/request_response_logger.rb +++ b/server/app/middleware/multiwoven_server/request_response_logger.rb @@ -25,7 +25,7 @@ def log_request(env) Rails.logger.info({ request_method: request.request_method, request_url: request.url, - request_params: request.parameters, + request_params: request.filtered_parameters, request_headers: { "Workspace-Id": request.headers["Workspace-Id"] } }.to_s) end From 4d5d74041ee2adcbc79d1f0f2c32e61b98e29148 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 09:53:58 +0530 Subject: [PATCH 63/74] fix(CE): add more fields to log redaction (#349) Co-authored-by: datafloyd --- .../initializers/filter_parameter_logging.rb | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/server/config/initializers/filter_parameter_logging.rb b/server/config/initializers/filter_parameter_logging.rb index c2d89e28..240bab25 100644 --- a/server/config/initializers/filter_parameter_logging.rb +++ b/server/config/initializers/filter_parameter_logging.rb @@ -4,5 +4,27 @@ # Use this to limit dissemination of sensitive information. # See the ActiveSupport::ParameterFilter documentation for supported notations and behaviors. Rails.application.config.filter_parameters += [ - :passw, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn + :password, + :password_confirmation, + :secret, + :token, + :_key, + :crypt, + :salt, + :certificate, + :otp, + :ssn, + :credit_card_number, + :card_number, + :cvv, + :card_verification, + :expiration_date, + :authenticity_token, + :api_key, + :access_token, + :refresh_token, + :pin, + :current_password, + :new_password, + :ssn_last4, ] From b53ec8795d995fa021849a4a47d77d95e58ffc6b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 09:46:40 +0530 Subject: [PATCH 64/74] feat(CE): add aws sagemaker model source connector (#352) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- integrations/Gemfile | 4 + integrations/Gemfile.lock | 15 +- integrations/lib/multiwoven/integrations.rb | 3 + .../multiwoven/integrations/core/constants.rb | 3 - .../lib/multiwoven/integrations/rollout.rb | 3 +- .../source/aws_sagemaker_model/client.rb | 79 ++++++++++ .../aws_sagemaker_model/config/catalog.json | 6 + .../aws_sagemaker_model/config/meta.json | 15 ++ .../aws_sagemaker_model/config/spec.json | 50 ++++++ .../source/aws_sagemaker_model/icon.svg | 18 +++ integrations/multiwoven-integrations.gemspec | 2 + .../source/aws_sagemaker_model/client_spec.rb | 146 ++++++++++++++++++ 12 files changed, 339 insertions(+), 5 deletions(-) create mode 100644 integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/client.rb create mode 100644 integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/catalog.json create mode 100644 integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/meta.json create mode 100644 integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/spec.json create mode 100644 integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/icon.svg create mode 100644 integrations/spec/multiwoven/integrations/source/aws_sagemaker_model/client_spec.rb diff --git a/integrations/Gemfile b/integrations/Gemfile index 757d6502..970287a8 100644 --- a/integrations/Gemfile +++ b/integrations/Gemfile @@ -69,6 +69,10 @@ gem "aws-sdk-sts" gem "ruby-oci8", "~> 2.2.12" +gem "aws-sdk-sagemaker" + +gem "aws-sdk-sagemakerruntime" + group :development, :test do gem "simplecov", require: false gem "simplecov_json_formatter", require: false diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 0bcecf8b..5fc041b6 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,12 +7,14 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.9.2) + multiwoven-integrations (0.10.0) activesupport async-websocket aws-sdk-athena + aws-sdk-cloudwatchlogs aws-sdk-s3 aws-sdk-sts + aws-sigv4 csv dry-schema dry-struct @@ -67,6 +69,9 @@ GEM aws-sdk-athena (1.83.0) aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) + aws-sdk-cloudwatchlogs (1.82.0) + aws-sdk-core (~> 3, >= 3.193.0) + aws-sigv4 (~> 1.1) aws-sdk-core (3.196.1) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) @@ -79,6 +84,12 @@ GEM aws-sdk-core (~> 3, >= 3.194.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.8) + aws-sdk-sagemaker (1.229.0) + aws-sdk-core (~> 3, >= 3.188.0) + aws-sigv4 (~> 1.1) + aws-sdk-sagemakerruntime (1.63.0) + aws-sdk-core (~> 3, >= 3.193.0) + aws-sigv4 (~> 1.1) aws-sdk-sts (1.11.0) aws-sdk-core (~> 3, >= 3.110.0) aws-sigv4 (~> 1.1) @@ -338,6 +349,8 @@ DEPENDENCIES async-websocket (~> 0.8.0) aws-sdk-athena aws-sdk-s3 + aws-sdk-sagemaker + aws-sdk-sagemakerruntime aws-sdk-sts byebug csv diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb index 02f2e262..bf214828 100644 --- a/integrations/lib/multiwoven/integrations.rb +++ b/integrations/lib/multiwoven/integrations.rb @@ -32,6 +32,8 @@ require "iterable-api-client" require "aws-sdk-sts" require "ruby-oci8" +require "aws-sdk-sagemaker" +require "aws-sdk-sagemakerruntime" # Service require_relative "integrations/config" @@ -63,6 +65,7 @@ require_relative "integrations/source/maria_db/client" require_relative "integrations/source/oracle_db/client" require_relative "integrations/source/databrics_model/client" +require_relative "integrations/source/aws_sagemaker_model/client" # Destination require_relative "integrations/destination/klaviyo/client" diff --git a/integrations/lib/multiwoven/integrations/core/constants.rb b/integrations/lib/multiwoven/integrations/core/constants.rb index 95e369dc..4d774a5f 100644 --- a/integrations/lib/multiwoven/integrations/core/constants.rb +++ b/integrations/lib/multiwoven/integrations/core/constants.rb @@ -50,9 +50,6 @@ module Constants DATABRICKS_HEALTH_URL = "https://%s/api/2.0/serving-endpoints/%s" DATABRICKS_SERVING_URL = "https://%s/serving-endpoints/%s/invocations" - AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"] - AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"] - # HTTP HTTP_GET = "GET" HTTP_POST = "POST" diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 1447eb3a..fbba1dfc 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.9.2" + VERSION = "0.10.0" ENABLED_SOURCES = %w[ Snowflake @@ -17,6 +17,7 @@ module Integrations MariaDB Oracle DatabricksModel + AwsSagemakerModel ].freeze ENABLED_DESTINATIONS = %w[ diff --git a/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/client.rb b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/client.rb new file mode 100644 index 00000000..f5ecce11 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/client.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Multiwoven::Integrations::Source + module AwsSagemakerModel + include Multiwoven::Integrations::Core + class Client < SourceConnector + def check_connection(connection_config) + connection_config = connection_config.with_indifferent_access + create_connection(connection_config) + response = @client.describe_endpoint(endpoint_name: connection_config[:endpoint_name]) + if response.endpoint_status == "InService" + success_status + else + failure_status + end + rescue StandardError => e + ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message + end + + def discover(_connection_config) + catalog_json = read_json(CATALOG_SPEC_PATH) + catalog = build_catalog(catalog_json) + catalog.to_multiwoven_message + rescue StandardError => e + handle_exception(e, { + context: "AWS:SAGEMAKER MODEL:DISCOVER:EXCEPTION", + type: "error" + }) + end + + def read(sync_config) + connection_config = sync_config.source.connection_specification + connection_config = connection_config.with_indifferent_access + payload = sync_config.model.query + create_connection(connection_config) + run_model(connection_config, payload) + rescue StandardError => e + handle_exception(e, { + context: "AWS:SAGEMAKER MODEL:READ:EXCEPTION", + type: "error", + sync_id: sync_config.sync_id, + sync_run_id: sync_config.sync_run_id + }) + end + + private + + def create_connection(connection_config) + @client = Aws::SageMaker::Client.new( + region: connection_config[:region], + access_key_id: connection_config[:access_key], + secret_access_key: connection_config[:secret_access_key] + ) + + @client_runtime = Aws::SageMakerRuntime::Client.new( + region: connection_config[:region], + access_key_id: connection_config[:access_key], + secret_access_key: connection_config[:secret_access_key] + ) + end + + def run_model(connection_config, payload) + response = @client_runtime.invoke_endpoint( + endpoint_name: connection_config[:endpoint_name], + content_type: "application/json", + body: payload + ) + process_response(response) + rescue StandardError => e + handle_exception(e, context: "AWS:SAGEMAKER MODEL:RUN_MODEL:EXCEPTION", type: "error") + end + + def process_response(response) + data = JSON.parse(response.body.read) + [RecordMessage.new(data: { response: data }, emitted_at: Time.now.to_i).to_multiwoven_message] + end + end + end +end diff --git a/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/catalog.json b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/catalog.json new file mode 100644 index 00000000..dacb788b --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/catalog.json @@ -0,0 +1,6 @@ +{ + "request_rate_limit": 600, + "request_rate_limit_unit": "minute", + "request_rate_concurrency": 10, + "streams": [] +} diff --git a/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/meta.json b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/meta.json new file mode 100644 index 00000000..9aeb49ba --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/meta.json @@ -0,0 +1,15 @@ +{ + "data": { + "name": "AwsSagemakerModel", + "title": "AWS Sagemaker Model", + "connector_type": "source", + "category": "AI Model", + "documentation_url": "https://docs.mutliwoven.com", + "github_issue_label": "source-aws-sagemaker-model", + "icon": "icon.svg", + "license": "MIT", + "release_stage": "alpha", + "support_level": "community", + "tags": ["language:ruby", "multiwoven"] + } +} diff --git a/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/spec.json b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/spec.json new file mode 100644 index 00000000..2253b4c1 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/spec.json @@ -0,0 +1,50 @@ +{ + "documentation_url": "https://docs.multiwoven.com/integrations/sources/aws_sagemaker-model", + "stream_type": "user_defined", + "connector_query_type": "ai_ml", + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AWS Sagemaker Model", + "type": "object", + "required": ["access_key", "secret_access_key", "region", "endpoint_name", "request_format", "response_format"], + "properties": { + "access_key": { + "description": "The AWS Access Key ID to use for authentication", + "type": "string", + "title": "Personal Access Key", + "order": 0 + }, + "secret_access_key": { + "description": "The AWS Secret Access Key to use for authentication", + "type": "string", + "multiwoven_secret": true, + "title": "Secret Access Key", + "order": 1 + }, + "region": { + "description": "AWS region", + "type": "string", + "title": "Region", + "order": 2 + }, + "endpoint_name": { + "description": "Endpoint name for AWS Sagemaker", + "type": "string", + "title": "Endpoint name", + "order": 3 + }, + "request_format": { + "description": "Sample Request Format", + "type": "string", + "title": "Request Format", + "order": 4 + }, + "response_format": { + "description": "Sample Response Format", + "type": "string", + "title": "Response Format", + "order": 5 + } + } + } +} \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/icon.svg b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/icon.svg new file mode 100644 index 00000000..542ea19d --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/icon.svg @@ -0,0 +1,18 @@ + + + + Icon-Architecture/64/Arch_AWS-SageMaker_64 + Created with Sketch. + + + + + + + + + + + + + \ No newline at end of file diff --git a/integrations/multiwoven-integrations.gemspec b/integrations/multiwoven-integrations.gemspec index 498a2d19..f312920d 100644 --- a/integrations/multiwoven-integrations.gemspec +++ b/integrations/multiwoven-integrations.gemspec @@ -36,8 +36,10 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "activesupport" spec.add_runtime_dependency "async-websocket" spec.add_runtime_dependency "aws-sdk-athena" + spec.add_runtime_dependency "aws-sdk-cloudwatchlogs" spec.add_runtime_dependency "aws-sdk-s3" spec.add_runtime_dependency "aws-sdk-sts" + spec.add_runtime_dependency "aws-sigv4" spec.add_runtime_dependency "csv" spec.add_runtime_dependency "dry-schema" spec.add_runtime_dependency "dry-struct" diff --git a/integrations/spec/multiwoven/integrations/source/aws_sagemaker_model/client_spec.rb b/integrations/spec/multiwoven/integrations/source/aws_sagemaker_model/client_spec.rb new file mode 100644 index 00000000..44b74eaf --- /dev/null +++ b/integrations/spec/multiwoven/integrations/source/aws_sagemaker_model/client_spec.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +RSpec.describe Multiwoven::Integrations::Source::AwsSagemakerModel::Client do + let(:client) { Multiwoven::Integrations::Source::AwsSagemakerModel::Client.new } + let(:payload) do + { + mode: "embedding", + text_inputs: [ + "Hi" + ] + } + end + let(:sync_config) do + { + "source": { + "name": "AWS Sagemaker Model", + "type": "source", + "connection_specification": { + "access_key": ENV["ATHENA_ACCESS"], + "secret_access_key": ENV["ATHENA_SECRET"], + "region": "us-east-2", + "endpoint_name": "Test-Endpoint", + "request_format": {}, + "response_format": {} + } + }, + "destination": { + "name": "Sample Destination Connector", + "type": "destination", + "connection_specification": { + "example_destination_key": "example_destination_value" + } + }, + "model": { + "name": "Anthena Account", + "query": payload.to_json, + "query_type": "raw_sql", + "primary_key": "id" + }, + "stream": { + "name": "example_stream", + "action": "create", + "json_schema": { "field1": "type1" }, + "supported_sync_modes": %w[full_refresh incremental], + "source_defined_cursor": true, + "default_cursor_field": ["field1"], + "source_defined_primary_key": [["field1"], ["field2"]], + "namespace": "exampleNamespace", + "url": "https://api.example.com/data", + "method": "GET" + }, + "sync_mode": "full_refresh", + "cursor_field": "timestamp", + "destination_sync_mode": "upsert", + "sync_id": "1" + } + end + + let(:runtime_client) { instance_double(Aws::SageMakerRuntime::Client) } + let(:sagemaker_client) { instance_double(Aws::SageMaker::Client) } + let(:response) { instance_double(Aws::SageMaker::Types::DescribeEndpointOutput, endpoint_status: "InService") } + let(:body_double) { instance_double(StringIO, read: "[1,2,3]") } + let(:endpoint_response) { instance_double(Aws::SageMakerRuntime::Types::InvokeEndpointOutput, body: body_double) } + + describe "#check_connection" do + before do + allow(Aws::SageMakerRuntime::Client).to receive(:new).and_return(runtime_client) + allow(Aws::SageMaker::Client).to receive(:new).and_return(sagemaker_client) + allow(sagemaker_client).to receive(:describe_endpoint).and_return(response) + end + context "when the connection is successful" do + it "returns a succeeded connection status" do + message = client.check_connection(sync_config[:source][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("succeeded") + expect(result.message).to be_nil + end + end + + context "when the connection fails" do + it "returns a failed connection status with an error message" do + allow_any_instance_of(Multiwoven::Integrations::Source::AwsSagemakerModel::Client).to receive(:create_connection).and_raise(StandardError, "Connection failed") + message = client.check_connection(sync_config[:source][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("failed") + expect(result.message).to include("Connection failed") + end + end + end + + describe "#discover" do + it "successfully returns the catalog message" do + message = client.discover(nil) + catalog = message.catalog + expect(catalog).to be_a(Multiwoven::Integrations::Protocol::Catalog) + expect(catalog.request_rate_limit).to eql(600) + expect(catalog.request_rate_limit_unit).to eql("minute") + expect(catalog.request_rate_concurrency).to eql(10) + end + + it "handles exceptions during discovery" do + allow(client).to receive(:read_json).and_raise(StandardError.new("test error")) + expect(client).to receive(:handle_exception).with( + an_instance_of(StandardError), + hash_including(context: "AWS:SAGEMAKER MODEL:DISCOVER:EXCEPTION", type: "error") + ) + client.discover(nil) + end + end + + # read and #discover tests for AWS Athena + describe "#read" do + context "when the read is successful" do + it "reads records successfully" do + s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json) + allow(Aws::SageMakerRuntime::Client).to receive(:new).and_return(runtime_client) + allow(Aws::SageMaker::Client).to receive(:new).and_return(sagemaker_client) + allow(runtime_client).to receive(:invoke_endpoint).and_return(endpoint_response) + records = client.read(s_config) + expect(records).to be_an(Array) + expect(records.first.record).to be_a(Multiwoven::Integrations::Protocol::RecordMessage) + expect(records.first.record.data).to eq({ response: [1, 2, 3] }) + end + end + + context "when the read is failed" do + it "handles exceptions during reading" do + s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json) + error_instance = StandardError.new("test error") + allow(Aws::SageMakerRuntime::Client).to receive(:new).and_return(runtime_client) + allow(Aws::SageMaker::Client).to receive(:new).and_return(sagemaker_client) + allow(client).to receive(:run_model).and_raise(error_instance) + expect(client).to receive(:handle_exception).with( + error_instance, + { + context: "AWS:SAGEMAKER MODEL:READ:EXCEPTION", + sync_id: "1", + sync_run_id: nil, + type: "error" + } + ) + client.read(s_config) + end + end + end +end From 274c1999d9890de7aeacfd5c13566010f47fb479 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 09:47:45 +0530 Subject: [PATCH 65/74] chore(CE): gem server update 0.10.0 (#351) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- server/Gemfile | 2 +- server/Gemfile.lock | 37 +++++++++++++++++++++---------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/server/Gemfile b/server/Gemfile index c4e15073..77280cb9 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -13,7 +13,7 @@ gem "interactor", "~> 3.0" gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git" -gem "multiwoven-integrations", "~> 0.9.1" +gem "multiwoven-integrations", "~> 0.10.0" gem "temporal-ruby", github: "coinbase/temporal-ruby" diff --git a/server/Gemfile.lock b/server/Gemfile.lock index 383d6078..afb6c195 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -108,21 +108,23 @@ GEM appsignal (3.7.5) rack ast (2.4.2) - async (2.15.3) + async (2.16.1) console (~> 1.26) fiber-annotation io-event (~> 1.6, >= 1.6.5) - async-http (0.70.0) + async-http (0.72.0) async (>= 2.10.2) async-pool (~> 0.7) io-endpoint (~> 0.11) io-stream (~> 0.4) - protocol-http (~> 0.28) - protocol-http1 (~> 0.19) + protocol-http (~> 0.29) + protocol-http1 (~> 0.20) protocol-http2 (~> 0.18) traces (>= 0.10) - async-pool (0.8.0) + async-pool (0.8.1) async (>= 1.25) + metrics + traces async-websocket (0.28.0) async-http (~> 0.54) protocol-http (>= 0.28.1) @@ -1773,7 +1775,7 @@ GEM fugit (1.10.1) et-orbi (~> 1, >= 1.2.7) raabro (~> 1.4) - git (2.1.1) + git (2.2.0) activesupport (>= 5.0) addressable (~> 2.8) process_executer (~> 1.1) @@ -1781,7 +1783,7 @@ GEM gli (2.21.5) globalid (1.2.1) activesupport (>= 6.1) - google-apis-bigquery_v2 (0.76.0) + google-apis-bigquery_v2 (0.77.0) google-apis-core (>= 0.15.0, < 2.a) google-apis-core (0.15.1) addressable (~> 2.5, >= 2.5.1) @@ -1804,7 +1806,7 @@ GEM google-cloud-core (1.7.1) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) - google-cloud-env (2.1.1) + google-cloud-env (2.2.0) faraday (>= 1.0, < 3.a) google-cloud-errors (1.4.0) google-protobuf (3.25.2-aarch64-linux) @@ -1847,7 +1849,7 @@ GEM inflection (1.0.0) interactor (3.1.2) io-console (0.7.1) - io-endpoint (0.13.0) + io-endpoint (0.13.1) io-event (1.6.5) io-stream (0.4.0) irb (1.11.1) @@ -1893,17 +1895,20 @@ GEM net-pop net-smtp marcel (1.0.2) + metrics (0.10.2) mini_mime (1.1.5) minitest (5.21.2) msgpack (1.7.2) multi_json (1.15.0) multipart-post (2.4.1) - multiwoven-integrations (0.9.1) + multiwoven-integrations (0.10.0) activesupport async-websocket aws-sdk-athena + aws-sdk-cloudwatchlogs aws-sdk-s3 aws-sdk-sts + aws-sigv4 csv dry-schema dry-struct @@ -1973,8 +1978,8 @@ GEM premailer (~> 1.7, >= 1.7.9) process_executer (1.1.0) protocol-hpack (1.5.0) - protocol-http (0.28.2) - protocol-http1 (0.19.1) + protocol-http (0.29.0) + protocol-http1 (0.20.0) protocol-http (~> 0.22) protocol-http2 (0.18.0) protocol-hpack (~> 1.4) @@ -2123,7 +2128,7 @@ GEM faraday-multipart gli hashie - sorbet-runtime (0.5.11532) + sorbet-runtime (0.5.11553) stringio (3.1.0) stripe (12.5.0) strong_migrations (1.8.0) @@ -2131,7 +2136,7 @@ GEM thor (1.3.0) timecop (0.9.8) timeout (0.4.1) - traces (0.11.1) + traces (0.13.1) trailblazer-option (0.1.2) typhoeus (1.4.1) ethon (>= 0.9.0) @@ -2143,7 +2148,7 @@ GEM unf_ext (0.0.9.1) unicode-display_width (2.5.0) uniform_notifier (1.16.0) - uri (0.13.0) + uri (0.13.1) uri_template (0.7.0) warden (1.2.9) rack (>= 2.0.9) @@ -2197,7 +2202,7 @@ DEPENDENCIES jwt kaminari liquid - multiwoven-integrations (~> 0.9.1) + multiwoven-integrations (~> 0.10.0) mysql2 newrelic_rpm parallel From 9e404651bd25129faa40f4d19dc8580b28cfadd1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 09:48:13 +0530 Subject: [PATCH 66/74] chore(CE): data app model migration (#350) Co-authored-by: afthab vp --- .../20240829110045_create_data_apps.rb | 13 ++++++++++ .../20240829131951_create_visual_component.rb | 15 ++++++++++++ server/db/schema.rb | 24 ++++++++++++++++++- 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 server/db/migrate/20240829110045_create_data_apps.rb create mode 100644 server/db/migrate/20240829131951_create_visual_component.rb diff --git a/server/db/migrate/20240829110045_create_data_apps.rb b/server/db/migrate/20240829110045_create_data_apps.rb new file mode 100644 index 00000000..7177b4e1 --- /dev/null +++ b/server/db/migrate/20240829110045_create_data_apps.rb @@ -0,0 +1,13 @@ +class CreateDataApps < ActiveRecord::Migration[7.1] + def change + create_table :data_apps do |t| + t.string :name, null: false + t.integer :status, null: false + t.integer :workspace_id, null: false + t.text :description + t.json :meta_data + + t.timestamps + end + end +end diff --git a/server/db/migrate/20240829131951_create_visual_component.rb b/server/db/migrate/20240829131951_create_visual_component.rb new file mode 100644 index 00000000..8abcc15b --- /dev/null +++ b/server/db/migrate/20240829131951_create_visual_component.rb @@ -0,0 +1,15 @@ +class CreateVisualComponent < ActiveRecord::Migration[7.1] + def change + create_table :visual_components do |t| + t.integer :component_type, null: false + t.string :name, null: false + t.integer :workspace_id, null: false + t.integer :data_app_id, null: false + t.integer :model_id, null: false + t.jsonb :properties + t.jsonb :feedback_config + + t.timestamps + end + end +end diff --git a/server/db/schema.rb b/server/db/schema.rb index c6672c4f..936958b6 100644 --- a/server/db/schema.rb +++ b/server/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_08_27_211003) do +ActiveRecord::Schema[7.1].define(version: 2024_08_29_131951) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -36,6 +36,16 @@ t.string "connector_category", default: "data", null: false end + create_table "data_apps", force: :cascade do |t| + t.string "name", null: false + t.integer "status", null: false + t.integer "workspace_id", null: false + t.text "description" + t.json "meta_data" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "data_migrations", primary_key: "version", id: :string, force: :cascade do |t| end @@ -175,6 +185,18 @@ t.index ["unlock_token"], name: "index_users_on_unlock_token", unique: true end + create_table "visual_components", force: :cascade do |t| + t.integer "component_type", null: false + t.string "name", null: false + t.integer "workspace_id", null: false + t.integer "data_app_id", null: false + t.integer "model_id", null: false + t.jsonb "properties" + t.jsonb "feedback_config" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "workspace_users", force: :cascade do |t| t.bigint "user_id", null: false t.bigint "workspace_id" From 529e56b54d8b25075725f4d46193b59518a49a60 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Mon, 2 Sep 2024 21:41:59 -0400 Subject: [PATCH 67/74] Multiwoven release v0.23.0 (#354) Co-authored-by: github-actions --- release-notes.md | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/release-notes.md b/release-notes.md index 35a1828a..868d77d0 100644 --- a/release-notes.md +++ b/release-notes.md @@ -2,30 +2,34 @@ All notable changes to this project will be documented in this file. -## [0.22.0] - 2024-08-26 +## [0.23.0] - 2024-09-02 ### 🚀 Features -- *(CE)* Enable/disable sync (#321) -- *(CE)* Databricks AI Model connector (#325) -- *(CE)* Catalog creation via API (#329) -- *(CE)* Calalog update api (#333) -- *(CE)* Add AI/ML query type to model (#334) +- *(CE)* Filter connectors based on the category (#338) +- *(CE)* Filter connectors (#343) +- *(CE)* Filter model based on the query type (#347) +- *(CE)* Add aws sagemaker model source connector (#352) ### 🐛 Bug Fixes -- *(CE)* Correction in databricks model connection_spec (#331) +- *(CE)* Refresh catalog only when refresh is true (#336) +- *(CE)* Connection_spec mandatory changes (#342) +- *(CE)* Added tag label to auto-truncate text +- *(CE)* Request log redaction (#348) +- *(CE)* Add more fields to log redaction (#349) ### 🚜 Refactor -- *(CE)* Changed condition to render toast (#315) +- *(CE)* Created common connector lists component ### ⚙️ Miscellaneous Tasks -- *(CE)* Add request response log for Hubspot (#323) -- *(CE)* Add request response log for Stripe (#328) -- *(CE)* Add request response log for SalesforceCrm (#326) -- *(CE)* Upgrade version to 0.9.1 (#330) -- *(CE)* Refactor error message to agreed standard (#332) +- *(CE)* Validate catalog for query source (#340) +- *(CE)* Add catalog presence validation for models (#341) +- *(CE)* Refactor interactors (#344) +- *(CE)* Add filtering scope to connectors (#345) +- *(CE)* Gem server update 0.10.0 (#351) +- *(CE)* Data app model migration (#350) From 50d3e085495e33f2550ccea4786568e5495efb86 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 11:51:23 +0530 Subject: [PATCH 68/74] feat(CE): data app model changes (#353) Co-authored-by: afthab vp --- server/app/models/data_app.rb | 21 +++++++++++++++++++ server/app/models/model.rb | 1 + server/app/models/visual_component.rb | 14 +++++++++++++ server/app/models/workspace.rb | 1 + server/spec/models/data_app_spec.rb | 23 +++++++++++++++++++++ server/spec/models/model_spec.rb | 1 + server/spec/models/visual_component_spec.rb | 16 ++++++++++++++ server/spec/models/workspace_spec.rb | 1 + 8 files changed, 78 insertions(+) create mode 100644 server/app/models/data_app.rb create mode 100644 server/app/models/visual_component.rb create mode 100644 server/spec/models/data_app_spec.rb create mode 100644 server/spec/models/visual_component_spec.rb diff --git a/server/app/models/data_app.rb b/server/app/models/data_app.rb new file mode 100644 index 00000000..628561cc --- /dev/null +++ b/server/app/models/data_app.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class DataApp < ApplicationRecord + validates :workspace_id, presence: true + validates :status, presence: true + validates :name, presence: true + + enum :status, %i[inactive active draft] + + belongs_to :workspace + has_many :visual_components, dependent: :destroy + has_many :models, through: :visual_components + + after_initialize :set_default_status, if: :new_record? + + private + + def set_default_status + self.status ||= :draft + end +end diff --git a/server/app/models/model.rb b/server/app/models/model.rb index f7db2832..9a4f991c 100644 --- a/server/app/models/model.rb +++ b/server/app/models/model.rb @@ -29,6 +29,7 @@ class Model < ApplicationRecord belongs_to :connector has_many :syncs, dependent: :destroy + has_many :visual_components, dependent: :destroy scope :data, -> { where(query_type: %i[raw_sql dbt soql table_selector]) } scope :ai_ml, -> { where(query_type: :ai_ml) } diff --git a/server/app/models/visual_component.rb b/server/app/models/visual_component.rb new file mode 100644 index 00000000..e0737f2a --- /dev/null +++ b/server/app/models/visual_component.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class VisualComponent < ApplicationRecord + validates :workspace_id, presence: true + validates :component_type, presence: true + validates :model_id, presence: true + validates :data_app_id, presence: true + + enum :component_type, %i[pie bar data_table] + + belongs_to :workspace + belongs_to :data_app + belongs_to :model +end diff --git a/server/app/models/workspace.rb b/server/app/models/workspace.rb index b9bc40a2..97410b97 100644 --- a/server/app/models/workspace.rb +++ b/server/app/models/workspace.rb @@ -20,6 +20,7 @@ class Workspace < ApplicationRecord has_many :catalogs, dependent: :nullify has_many :syncs, dependent: :nullify has_many :sync_runs, dependent: :nullify + has_many :data_apps, dependent: :nullify belongs_to :organization STATUS_ACTIVE = "active" diff --git a/server/spec/models/data_app_spec.rb b/server/spec/models/data_app_spec.rb new file mode 100644 index 00000000..1877f18e --- /dev/null +++ b/server/spec/models/data_app_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe DataApp, type: :model do + it { should validate_presence_of(:workspace_id) } + it { should validate_presence_of(:status) } + it { should validate_presence_of(:name) } + + it { should define_enum_for(:status).with_values(inactive: 0, active: 1, draft: 2) } + + it { should belong_to(:workspace) } + it { should have_many(:visual_components).dependent(:destroy) } + it { should have_many(:models).through(:visual_components) } + + describe "#set_default_status" do + let(:data_app) { DataApp.new } + + it "sets default status" do + expect(data_app.status).to eq("draft") + end + end +end diff --git a/server/spec/models/model_spec.rb b/server/spec/models/model_spec.rb index 1d7d23a1..b28d7a76 100644 --- a/server/spec/models/model_spec.rb +++ b/server/spec/models/model_spec.rb @@ -22,6 +22,7 @@ it { should belong_to(:workspace) } it { should belong_to(:connector) } it { should have_many(:syncs).dependent(:destroy) } + it { should have_many(:visual_components).dependent(:destroy) } end describe "validations" do diff --git a/server/spec/models/visual_component_spec.rb b/server/spec/models/visual_component_spec.rb new file mode 100644 index 00000000..0ba909dd --- /dev/null +++ b/server/spec/models/visual_component_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe VisualComponent, type: :model do + it { should validate_presence_of(:workspace_id) } + it { should validate_presence_of(:component_type) } + it { should validate_presence_of(:model_id) } + it { should validate_presence_of(:data_app_id) } + + it { should define_enum_for(:component_type).with_values(pie: 0, bar: 1, data_table: 2) } + + it { should belong_to(:workspace) } + it { should belong_to(:data_app) } + it { should belong_to(:model) } +end diff --git a/server/spec/models/workspace_spec.rb b/server/spec/models/workspace_spec.rb index 96d1ada1..5e0e1ce3 100644 --- a/server/spec/models/workspace_spec.rb +++ b/server/spec/models/workspace_spec.rb @@ -29,6 +29,7 @@ it { should have_many(:models).dependent(:nullify) } it { should have_many(:catalogs).dependent(:nullify) } it { should have_many(:syncs).dependent(:nullify) } + it { should have_many(:data_apps).dependent(:nullify) } it { should belong_to(:organization) } end From e184972d3b7f6d69db916d1c6a927aedaa8814ee Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 01:21:16 +0530 Subject: [PATCH 69/74] refactor(CE): moved common elements of sign up and sign in to separate views * Resolve conflict in cherry-pick of 9f11d0258219f006606605557beece58912e373e and change the commit message * feat(CE): added common schema * feat(CE): added privacyPolicy and TOS * refactor(CE): resolved conflicts * feat(CE): updated AuthFooter * refactor(CE): updated types and auth views --------- Co-authored-by: Tushar Selvakumar <54372016+macintushar@users.noreply.github.com> Co-authored-by: Tushar Selvakumar --- ui/src/chakra-core.config.ts | 2 + ui/src/components/HiddenInput/HiddenInput.tsx | 16 +- ui/src/services/authentication.ts | 9 +- .../Authentication/AuthCard/AuthCard.tsx | 38 +++ ui/src/views/Authentication/AuthCard/index.ts | 1 + .../Authentication/AuthFooter/AuthFooter.tsx | 18 +- .../AuthViews/SignInAuthView.tsx | 118 +++++++ ui/src/views/Authentication/SignIn/SignIn.tsx | 233 +------------ ui/src/views/Authentication/SignUp/SignUp.tsx | 311 +++--------------- ui/src/views/Authentication/types.ts | 1 - 10 files changed, 243 insertions(+), 504 deletions(-) create mode 100644 ui/src/views/Authentication/AuthCard/AuthCard.tsx create mode 100644 ui/src/views/Authentication/AuthCard/index.ts create mode 100644 ui/src/views/Authentication/AuthViews/SignInAuthView.tsx diff --git a/ui/src/chakra-core.config.ts b/ui/src/chakra-core.config.ts index ebc49a4a..28ff9115 100644 --- a/ui/src/chakra-core.config.ts +++ b/ui/src/chakra-core.config.ts @@ -293,4 +293,6 @@ export const defaultExtension = { }, brandName: 'Multiwoven', logoUrl: '', + termsOfServiceUrl: 'https://multiwoven.com/terms-of-service', + privacyPolicyUrl: 'https://multiwoven.com/privacy-policy', }; diff --git a/ui/src/components/HiddenInput/HiddenInput.tsx b/ui/src/components/HiddenInput/HiddenInput.tsx index 2164b938..ea7c4fcd 100644 --- a/ui/src/components/HiddenInput/HiddenInput.tsx +++ b/ui/src/components/HiddenInput/HiddenInput.tsx @@ -17,6 +17,14 @@ const HiddenInput = (props: InputProps): JSX.Element => { return ( + { onClick={onClickReveal} /> - ); }; diff --git a/ui/src/services/authentication.ts b/ui/src/services/authentication.ts index 5b832b2f..657e9636 100644 --- a/ui/src/services/authentication.ts +++ b/ui/src/services/authentication.ts @@ -1,4 +1,4 @@ -import { multiwovenFetch } from './common'; +import { ErrorResponse, multiwovenFetch } from './common'; export type SignUpPayload = { email: string; @@ -28,6 +28,12 @@ export type SignInResponse = { errors?: SignInErrorResponse[]; }; +export type AuthErrorResponse = { + status: number; + title: string; + detail: string; +}; + export type AuthResponse = { type: string; id: string; @@ -44,6 +50,7 @@ export type AuthResponse = { export type ApiResponse = { data?: T; status: number; + errors?: ErrorResponse[]; }; export const signUp = async (payload: SignUpPayload) => diff --git a/ui/src/views/Authentication/AuthCard/AuthCard.tsx b/ui/src/views/Authentication/AuthCard/AuthCard.tsx new file mode 100644 index 00000000..2c7583cd --- /dev/null +++ b/ui/src/views/Authentication/AuthCard/AuthCard.tsx @@ -0,0 +1,38 @@ +import { Box, Container, Flex, Image, Stack } from '@chakra-ui/react'; +import { AuthCardProps } from '../types'; + +const AuthCard = ({ children, brandName, logoUrl }: AuthCardProps): JSX.Element => ( + <> + + + + + + + + + + {children} + + + + + +); + +export default AuthCard; diff --git a/ui/src/views/Authentication/AuthCard/index.ts b/ui/src/views/Authentication/AuthCard/index.ts new file mode 100644 index 00000000..37915af0 --- /dev/null +++ b/ui/src/views/Authentication/AuthCard/index.ts @@ -0,0 +1 @@ +export { default } from './AuthCard'; diff --git a/ui/src/views/Authentication/AuthFooter/AuthFooter.tsx b/ui/src/views/Authentication/AuthFooter/AuthFooter.tsx index 3f6e6e2d..52cbc369 100644 --- a/ui/src/views/Authentication/AuthFooter/AuthFooter.tsx +++ b/ui/src/views/Authentication/AuthFooter/AuthFooter.tsx @@ -1,7 +1,17 @@ import { Box, HStack, Text } from '@chakra-ui/layout'; import { Link } from 'react-router-dom'; -const AuthFooter = (): JSX.Element => { +type AuthFooterProps = { + brandName: string; + privacyPolicyUrl: string; + termsOfServiceUrl: string; +}; + +const AuthFooter = ({ + brandName, + privacyPolicyUrl, + termsOfServiceUrl, +}: AuthFooterProps): JSX.Element => { return ( { > - © Multiwoven Inc. All rights reserved. + {`© ${brandName} Inc. All rights reserved.`} - + Terms of use @@ -24,7 +34,7 @@ const AuthFooter = (): JSX.Element => { • - + Privacy Policy diff --git a/ui/src/views/Authentication/AuthViews/SignInAuthView.tsx b/ui/src/views/Authentication/AuthViews/SignInAuthView.tsx new file mode 100644 index 00000000..2b4544ab --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/SignInAuthView.tsx @@ -0,0 +1,118 @@ +import AuthCard from '../AuthCard'; +import { Form, Formik } from 'formik'; +import { Button, Checkbox, HStack, Heading, Stack, Text } from '@chakra-ui/react'; +import { FormField, PasswordField } from '@/components/Fields'; +import { Link } from 'react-router-dom'; +import { SignInAuthViewProps } from '../types'; +import { SignInSchema } from '@/constants/schemas'; +import { useEffect } from 'react'; +import { useStore } from '@/stores'; +import Cookies from 'js-cookie'; + +export const SignInAuthView = ({ + brandName, + logoUrl, + handleSubmit, + submitting, +}: SignInAuthViewProps) => { + // clears the state when the sign in auth loads + useEffect(() => { + Cookies?.remove('authToken'); + useStore.getState().clearState(); + }, []); + + return ( + <> + + handleSubmit(values)} + validationSchema={SignInSchema} + > + {({ getFieldProps, touched, errors }) => ( +
+ + + {"Let's activate your data"} + + + {`Sign In to your ${brandName} account`} + + + + + + + + + + Stay signed in + + + + + Forgot Password? + + + + + + + + + + {"Don't have an account?"}{' '} + + + + Sign Up + + + + + + )} + + + + ); +}; diff --git a/ui/src/views/Authentication/SignIn/SignIn.tsx b/ui/src/views/Authentication/SignIn/SignIn.tsx index 5e413c1b..82ca37f0 100644 --- a/ui/src/views/Authentication/SignIn/SignIn.tsx +++ b/ui/src/views/Authentication/SignIn/SignIn.tsx @@ -1,106 +1,14 @@ import { useState } from 'react'; -import { Formik, Form, ErrorMessage, FormikTouched, FormikErrors, FieldInputProps } from 'formik'; -import * as Yup from 'yup'; -import { Link, useNavigate } from 'react-router-dom'; -import { - Box, - Button, - FormControl, - Input, - Heading, - Text, - Container, - Stack, - Flex, - HStack, - Image, - Checkbox, -} from '@chakra-ui/react'; -import MultiwovenIcon from '@/assets/images/icon-white.svg'; -import { SignInErrorResponse, SignInPayload, signIn } from '@/services/authentication'; +import { useNavigate } from 'react-router-dom'; +import { AuthErrorResponse, SignInPayload, signIn } from '@/services/authentication'; import Cookies from 'js-cookie'; import titleCase from '@/utils/TitleCase'; import AuthFooter from '../AuthFooter'; -import HiddenInput from '@/components/HiddenInput'; import { CustomToastStatus } from '@/components/Toast/index'; import useCustomToast from '@/hooks/useCustomToast'; import mwTheme from '@/chakra.config'; import { useMutation } from '@tanstack/react-query'; - -const SignInSchema = Yup.object().shape({ - email: Yup.string().email('Please enter a valid email address').required('Email is required'), - password: Yup.string() - .min(8, 'Password must be at least 8 characters') - .required('Password is required'), -}); - -interface SignInFormProps { - name: string; - type: string; - placeholder?: string; - getFieldProps: ( - nameOrOptions: - | string - | { - name: string; - value?: any; - onChange?: (e: any) => void; - onBlur?: (e: any) => void; - }, - ) => FieldInputProps; - touched: FormikTouched; - errors: FormikErrors; -} - -const FormField = ({ - name, - type, - getFieldProps, - touched, - errors, - placeholder, -}: SignInFormProps) => ( - - - - - - -); - -const PasswordField = ({ - name, - type, - getFieldProps, - touched, - errors, - placeholder, -}: SignInFormProps) => ( - - - - - - -); +import { SignInAuthView } from '../AuthViews/SignInAuthView'; const SignIn = (): JSX.Element => { const [submitting, setSubmitting] = useState(false); @@ -130,7 +38,7 @@ const SignIn = (): JSX.Element => { }); navigate('/', { replace: true }); } else { - result.data?.errors?.forEach((error: SignInErrorResponse) => { + result?.errors?.forEach((error: AuthErrorResponse) => { showToast({ duration: 5000, isClosable: true, @@ -154,130 +62,21 @@ const SignIn = (): JSX.Element => { } }; - const { logoUrl, brandName } = mwTheme; + const { logoUrl, brandName, privacyPolicyUrl, termsOfServiceUrl } = mwTheme; return ( <> - - handleSubmit(values)} - validationSchema={SignInSchema} - > - {({ getFieldProps, touched, errors }) => ( -
- - - - - - - - - - - {"Let's activate your data"} - - - {`Sign In to your ${brandName} account`} - - - - - - - ={' '} - - - - Stay signed in - - - {/* - Forgot Password? - */} - - - - - - - - {"Don't have an account?"}{' '} - - - - Sign Up - - - - - - - - - )} - - - + + ); }; diff --git a/ui/src/views/Authentication/SignUp/SignUp.tsx b/ui/src/views/Authentication/SignUp/SignUp.tsx index c34ebe36..13fc6956 100644 --- a/ui/src/views/Authentication/SignUp/SignUp.tsx +++ b/ui/src/views/Authentication/SignUp/SignUp.tsx @@ -1,125 +1,27 @@ -import { useState } from 'react'; -import { Formik, Form, ErrorMessage, FormikTouched, FormikErrors, FieldInputProps } from 'formik'; -import * as Yup from 'yup'; -import { Link, useNavigate } from 'react-router-dom'; -import { - Box, - Button, - FormControl, - Input, - Heading, - Text, - Container, - Stack, - Flex, - HStack, - Image, -} from '@chakra-ui/react'; -import MultiwovenIcon from '@/assets/images/icon-white.svg'; -import { SignUpPayload, signUp } from '@/services/authentication'; -import Cookies from 'js-cookie'; -import titleCase from '@/utils/TitleCase'; +import { useNavigate } from 'react-router-dom'; +import { Stack, Heading } from '@chakra-ui/react'; +import mwTheme from '@/chakra.config'; import AuthFooter from '../AuthFooter'; -import HiddenInput from '@/components/HiddenInput'; -import { CustomToastStatus } from '@/components/Toast/index'; +import { SignUpAuthView } from '@/views/Authentication/AuthViews/SignUpAuthView'; +import AuthCard from '../AuthCard'; +import { CustomToastStatus } from '@/components/Toast'; +import { SignUpPayload, signUp } from '@/services/authentication'; +import { useState } from 'react'; import useCustomToast from '@/hooks/useCustomToast'; -import mwTheme from '@/chakra.config'; import { useMutation } from '@tanstack/react-query'; - -const SignUpSchema = Yup.object().shape({ - company_name: Yup.string().required('Company name is required'), - name: Yup.string().required('Name is required'), - email: Yup.string().email('Invalid email address').required('Email is required'), - password: Yup.string() - .min(8, 'Password must be at least 8 characters') - .required('Password is required'), - password_confirmation: Yup.string() - .oneOf([Yup.ref('password'), ''], 'Passwords must match') - .required('Confirm Password is required'), -}); - -interface SignUpFormProps { - name: string; - type: string; - placeholder?: string; - getFieldProps: ( - nameOrOptions: - | string - | { - name: string; - value?: any; - onChange?: (e: any) => void; - onBlur?: (e: any) => void; - }, - ) => FieldInputProps; - touched: FormikTouched; - errors: FormikErrors; -} - -const FormField = ({ - name, - type, - getFieldProps, - touched, - errors, - placeholder, -}: SignUpFormProps) => ( - - - - - - -); - -const PasswordField = ({ - name, - type, - getFieldProps, - touched, - errors, - placeholder, -}: SignUpFormProps) => ( - - - - - - -); - -type SignUpErrors = { - source: { - [key: string]: string; - }; -}; +import { useAPIErrorsToast, useErrorToast } from '@/hooks/useErrorToast'; +// import isValidEmailDomain from '@/utils/isValidEmailDomain'; const SignUp = (): JSX.Element => { const [submitting, setSubmitting] = useState(false); const navigate = useNavigate(); const showToast = useCustomToast(); + const apiErrorToast = useAPIErrorsToast(); + const errorToast = useErrorToast(); const { mutateAsync } = useMutation({ mutationFn: (values: SignUpPayload) => signUp(values), - mutationKey: ['signIn'], + mutationKey: ['signUp'], }); const handleSubmit = async (values: any) => { @@ -128,12 +30,6 @@ const SignUp = (): JSX.Element => { const result = await mutateAsync(values); if (result.data?.attributes) { - const token = result.data.attributes.token; - Cookies.set('authToken', token, { - secure: true, - sameSite: 'Lax', - }); - showToast({ title: 'Account created.', status: CustomToastStatus.Success, @@ -142,179 +38,48 @@ const SignUp = (): JSX.Element => { position: 'bottom-right', }); - navigate('/'); + navigate(`/sign-up/success?email=${values.email}`); } else { - result.data?.errors?.map((error: SignUpErrors) => { - Object.keys(error.source).map((error_key) => { - showToast({ - title: titleCase(error_key) + ' ' + error.source[error_key], - status: CustomToastStatus.Warning, - duration: 5000, - isClosable: true, - position: 'bottom-right', - colorScheme: 'red', - }); - }); - }); + apiErrorToast(result.errors || []); } } catch (error) { - showToast({ - title: 'An error occured. Please try again later.', - status: CustomToastStatus.Error, - duration: 5000, - isClosable: true, - position: 'bottom-right', - colorScheme: 'red', - }); + errorToast('An error occured. Please try again later.', true, null, true); } finally { setSubmitting(false); } }; - const { logoUrl, brandName } = mwTheme; + const { logoUrl, brandName, privacyPolicyUrl, termsOfServiceUrl } = mwTheme; return ( <> - - + + + Get started with {brandName} + + + handleSubmit(values)} - validationSchema={SignUpSchema} - > - {({ getFieldProps, touched, errors }) => ( -
- - - - - - - - - - - {`Get started with ${brandName}`} - - - Sign up and create your account - - - - - - - - - - - - By creating an account, I agree to the{' '} - - - - Terms - - - - and - - - - Privacy Policy - - - - - - - - - Do you have an account?{' '} - - - - Sign In - - - - - - - - - - )} - - - + /> + + ); }; diff --git a/ui/src/views/Authentication/types.ts b/ui/src/views/Authentication/types.ts index 3d2185d9..f0168859 100644 --- a/ui/src/views/Authentication/types.ts +++ b/ui/src/views/Authentication/types.ts @@ -1,5 +1,4 @@ import { ReactNode } from 'react'; - export type AuthCardProps = { children: ReactNode; brandName: string; From 5a2b57ca48f0a4faa7217d83590bf996bd35743d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:15:23 +0530 Subject: [PATCH 70/74] chore(CE): add data-app permissions to roles (#357) Co-authored-by: afthab vp --- ...055705_add_data_app_permission_to_roles.rb | 60 +++++++++++++++++++ server/db/data_schema.rb | 2 +- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 server/db/data/20240903055705_add_data_app_permission_to_roles.rb diff --git a/server/db/data/20240903055705_add_data_app_permission_to_roles.rb b/server/db/data/20240903055705_add_data_app_permission_to_roles.rb new file mode 100644 index 00000000..16bf2961 --- /dev/null +++ b/server/db/data/20240903055705_add_data_app_permission_to_roles.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +class AddDataAppPermissionToRoles < ActiveRecord::Migration[7.1] + def change # rubocop:disable Metrics/MethodLength + admin_role = Role.find_by(role_name: "Admin") + member_role = Role.find_by(role_name: "Member") + viewer_role = Role.find_by(role_name: "Viewer") + + admin_role&.update!( + policies: { + permissions: { + connector_definition: { create: true, read: true, update: true, delete: true }, + connector: { create: true, read: true, update: true, delete: true }, + model: { create: true, read: true, update: true, delete: true }, + report: { create: true, read: true, update: true, delete: true }, + sync_record: { create: true, read: true, update: true, delete: true }, + sync_run: { create: true, read: true, update: true, delete: true }, + sync: { create: true, read: true, update: true, delete: true }, + user: { create: true, read: true, update: true, delete: true }, + workspace: { create: true, read: true, update: true, delete: true }, + data_app: { create: true, read: true, update: true, delete: true } + } + } + ) + + member_role&.update!( + policies: { + permissions: { + connector_definition: { create: true, read: true, update: true, delete: true }, + connector: { create: true, read: true, update: true, delete: true }, + model: { create: true, read: true, update: true, delete: true }, + report: { create: true, read: true, update: true, delete: true }, + sync_record: { create: true, read: true, update: true, delete: true }, + sync_run: { create: true, read: true, update: true, delete: true }, + sync: { create: true, read: true, update: true, delete: true }, + user: { create: false, read: true, update: false, delete: false }, + workspace: { create: false, read: true, update: false, delete: false }, + data_app: { create: true, read: true, update: true, delete: true } + } + } + ) + + viewer_role&.update!( + policies: { + permissions: { + connector_definition: { create: false, read: true, update: false, delete: false }, + connector: { create: false, read: true, update: false, delete: false }, + model: { create: false, read: true, update: false, delete: false }, + report: { create: false, read: true, update: false, delete: false }, + sync_record: { create: false, read: true, update: false, delete: false }, + sync_run: { create: false, read: true, update: false, delete: false }, + sync: { create: false, read: true, update: false, delete: false }, + user: { create: false, read: true, update: false, delete: false }, + workspace: { create: false, read: true, update: false, delete: false }, + data_app: { create: false, read: true, update: false, delete: false } + } + } + ) + end +end diff --git a/server/db/data_schema.rb b/server/db/data_schema.rb index f54a3d06..ba8761bd 100644 --- a/server/db/data_schema.rb +++ b/server/db/data_schema.rb @@ -1,3 +1,3 @@ # frozen_string_literal: true -DataMigrate::Data.define(version: 20_240_827_212_112) +DataMigrate::Data.define(version: 20_240_903_055_705) From d11a7801fe61009cf6a02285bfc434276dbeb0cc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:16:38 +0530 Subject: [PATCH 71/74] fix(CE): validate_catalog disabled query_source (#356) Co-authored-by: afthab vp --- .../api/v1/connectors_controller.rb | 3 ++- .../api/v1/connectors_controller_spec.rb | 26 +++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/server/app/controllers/api/v1/connectors_controller.rb b/server/app/controllers/api/v1/connectors_controller.rb index 77bbcd08..821a99f7 100644 --- a/server/app/controllers/api/v1/connectors_controller.rb +++ b/server/app/controllers/api/v1/connectors_controller.rb @@ -8,7 +8,8 @@ class ConnectorsController < ApplicationController before_action :set_connector, only: %i[show update destroy discover query_source] # TODO: Enable this once we have query validation implemented for all the connectors # before_action :validate_query, only: %i[query_source] - before_action :validate_catalog, only: %i[query_source] + # TODO: Enable this for ai_ml sources + # before_action :validate_catalog, only: %i[query_source] after_action :event_logger def index diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb index e5f194b1..02f9da27 100644 --- a/server/spec/requests/api/v1/connectors_controller_spec.rb +++ b/server/spec/requests/api/v1/connectors_controller_spec.rb @@ -481,19 +481,19 @@ expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) end - it "returns an error message for missing catalog" do - catalog = connector.catalog - catalog.connector_id = connectors.second.id - catalog.save - - allow(Connectors::QuerySource).to receive(:call).and_return(double(:context, success?: true, - records: [record1, record2])) - post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: - { "Content-Type": "application/json" }.merge(auth_headers(user, workspace.id)) - expect(response).to have_http_status(:unprocessable_entity) - response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash.dig(:errors, 0, :detail)).to eq("Catalog is not present for the connector") - end + # it "returns an error message for missing catalog" do + # catalog = connector.catalog + # catalog.connector_id = connectors.second.id + # catalog.save + + # allow(Connectors::QuerySource).to receive(:call).and_return(double(:context, success?: true, + # records: [record1, record2])) + # post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: + # { "Content-Type": "application/json" }.merge(auth_headers(user, workspace.id)) + # expect(response).to have_http_status(:unprocessable_entity) + # response_hash = JSON.parse(response.body).with_indifferent_access + # expect(response_hash.dig(:errors, 0, :detail)).to eq("Catalog is not present for the connector") + # end it "returns success status for a valid query for viewer role" do workspace.workspace_users.first.update(role: viewer_role) From fb86473751b382814ce55592c8ef126c1dba2761 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 12:50:56 +0400 Subject: [PATCH 72/74] fix(CE): enable catalog validation only for ai models (#355) * Resolve conflict in cherry-pick of 6de5e956cb5cd7d54544cfb096802d8e00e15b5f and change the commit message * fix(CE): resolve conflicts * fix(CE): remove duplicate * fix(CE): fix spec --------- Co-authored-by: datafloyd Co-authored-by: afthab vp --- .../api/v1/connectors_controller.rb | 3 +- server/app/models/connector.rb | 4 ++ server/spec/models/connector_spec.rb | 5 ++ .../api/v1/connectors_controller_spec.rb | 46 +++++++++++++------ 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/server/app/controllers/api/v1/connectors_controller.rb b/server/app/controllers/api/v1/connectors_controller.rb index 821a99f7..5d8e9e67 100644 --- a/server/app/controllers/api/v1/connectors_controller.rb +++ b/server/app/controllers/api/v1/connectors_controller.rb @@ -9,7 +9,7 @@ class ConnectorsController < ApplicationController # TODO: Enable this once we have query validation implemented for all the connectors # before_action :validate_query, only: %i[query_source] # TODO: Enable this for ai_ml sources - # before_action :validate_catalog, only: %i[query_source] + before_action :validate_catalog, only: %i[query_source] after_action :event_logger def index @@ -126,6 +126,7 @@ def set_connector end def validate_catalog + return unless @connector.ai_model? return if @connector.catalog.present? render_error( diff --git a/server/app/models/connector.rb b/server/app/models/connector.rb index c2ee3c36..d48bc955 100644 --- a/server/app/models/connector.rb +++ b/server/app/models/connector.rb @@ -133,4 +133,8 @@ def set_category rescue StandardError => e Rails.logger.error("Failed to set category for connector ##{id}: #{e.message}") end + + def ai_model? + connector_category == "AI Model" + end end diff --git a/server/spec/models/connector_spec.rb b/server/spec/models/connector_spec.rb index fd814aaa..8a1c1981 100644 --- a/server/spec/models/connector_spec.rb +++ b/server/spec/models/connector_spec.rb @@ -196,6 +196,11 @@ expect(result).to include(ai_ml_connector) expect(result).not_to include(non_ai_ml_connector) end + + it "check whether connector is ai model or not" do + expect(ai_ml_connector.ai_model?).to eq(true) + expect(non_ai_ml_connector.ai_model?).to eq(false) + end end describe ".data" do diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb index 02f9da27..693988f0 100644 --- a/server/spec/requests/api/v1/connectors_controller_spec.rb +++ b/server/spec/requests/api/v1/connectors_controller_spec.rb @@ -481,19 +481,39 @@ expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) end - # it "returns an error message for missing catalog" do - # catalog = connector.catalog - # catalog.connector_id = connectors.second.id - # catalog.save - - # allow(Connectors::QuerySource).to receive(:call).and_return(double(:context, success?: true, - # records: [record1, record2])) - # post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: - # { "Content-Type": "application/json" }.merge(auth_headers(user, workspace.id)) - # expect(response).to have_http_status(:unprocessable_entity) - # response_hash = JSON.parse(response.body).with_indifferent_access - # expect(response_hash.dig(:errors, 0, :detail)).to eq("Catalog is not present for the connector") - # end + it "returns an error message for missing catalog for ai connectors" do + catalog = connector.catalog + catalog.connector_id = connectors.second.id + catalog.save + # rubocop:disable Rails/SkipsModelValidations + connector.update_column(:connector_category, "AI Model") + # rubocop:enable Rails/SkipsModelValidations + + allow(Connectors::QuerySource).to receive(:call).and_return(double(:context, success?: true, + records: [record1, record2])) + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: + { "Content-Type": "application/json" }.merge(auth_headers(user, workspace.id)) + expect(response).to have_http_status(:unprocessable_entity) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash.dig(:errors, 0, :detail)).to eq("Catalog is not present for the connector") + end + + it "should not return error message for missing catalog for data connectors" do + catalog = connector.catalog + catalog.connector_id = connectors.second.id + # rubocop:disable Rails/SkipsModelValidations + connector.update_column(:connector_category, "Data Warehouse") + # rubocop:enable Rails/SkipsModelValidations + catalog.save + + allow(Connectors::QuerySource).to receive(:call).and_return(double(:context, success?: true, + records: [record1, record2])) + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: + { "Content-Type": "application/json" }.merge(auth_headers(user, workspace.id)) + expect(response).to have_http_status(:ok) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) + end it "returns success status for a valid query for viewer role" do workspace.workspace_users.first.update(role: viewer_role) From b20ab3865ef8d2ef2594723ac22ad66a74e067f6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 12:45:32 +0530 Subject: [PATCH 73/74] fix(CE): disable catalog validation for data models (#358) Co-authored-by: datafloyd --- .../controllers/api/v1/models_controller.rb | 1 + .../requests/api/v1/models_controller_spec.rb | 27 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/server/app/controllers/api/v1/models_controller.rb b/server/app/controllers/api/v1/models_controller.rb index 421d9165..5c81e006 100644 --- a/server/app/controllers/api/v1/models_controller.rb +++ b/server/app/controllers/api/v1/models_controller.rb @@ -81,6 +81,7 @@ def set_model end def validate_catalog + return unless connector.ai_model? return if connector.catalog.present? render_error( diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb index c05177a5..229d7144 100644 --- a/server/spec/requests/api/v1/models_controller_spec.rb +++ b/server/spec/requests/api/v1/models_controller_spec.rb @@ -189,11 +189,22 @@ workspace.workspace_users.first.update(role: member_role) # set connector without catalog request_body[:model][:connector_id] = connector_without_catalog.id + # rubocop:disable Rails/SkipsModelValidations + connector_without_catalog.update_column(:connector_category, "AI Model") + # rubocop:enable Rails/SkipsModelValidations post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" } .merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:unprocessable_entity) end + it "shouldn't fail model creation for connector without catalog for data sources" do + workspace.workspace_users.first.update(role: member_role) + request_body[:model][:connector_id] = connector_without_catalog.id + post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:created) + end + it " creates a new model and returns success" do workspace.workspace_users.first.update(role: member_role) post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" } @@ -297,11 +308,14 @@ expect(response_hash.dig(:data, :attributes, :primary_key)).to eq(request_body.dig(:model, :primary_key)) end - it "fails model update for connector without catalog" do + it "fails model update for connector without catalog for ai model" do workspace.workspace_users.first.update(role: member_role) model = models.second model.connector_id = connector_without_catalog.id model.save! + # rubocop:disable Rails/SkipsModelValidations + connector_without_catalog.update_column(:connector_category, "AI Model") + # rubocop:enable Rails/SkipsModelValidations put "/api/v1/models/#{models.second.id}", params: request_body.to_json, headers: { "Content-Type": "application/json" } @@ -309,6 +323,17 @@ expect(response).to have_http_status(:unprocessable_entity) end + it "shouldn't fail model update for connector without catalog for data connector" do + workspace.workspace_users.first.update(role: member_role) + model = models.second + model.connector_id = connector_without_catalog.id + model.save! + put "/api/v1/models/#{models.second.id}", params: request_body.to_json, + headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:ok) + end + it "updates the model and returns success for member role" do workspace.workspace_users.first.update(role: member_role) put "/api/v1/models/#{models.second.id}", params: request_body.to_json, headers: From 0cf9c0976ddbaa8aa0b0cd047ed232fc36c9deed Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 15:05:28 +0530 Subject: [PATCH 74/74] feat(CE): verify user after signup --- ui/src/routes/main.tsx | 43 ++++++++ ui/src/services/authentication.ts | 74 ++++++++++--- .../Authentication/AuthCard/AuthCard.tsx | 7 +- .../AuthViews/ForgotPasswordAuthView.tsx | 71 ++++++++++++ .../ForgotPasswordSuccessAuthView.tsx | 45 ++++++++ .../AuthViews/ResetPasswordAuthView.tsx | 83 ++++++++++++++ .../ResetPasswordSuccessAuthView.tsx | 44 ++++++++ .../AuthViews/SignUpVerificationAuthView.tsx | 36 +++++++ .../AuthViews/VerifyUserAuthView.tsx | 36 +++++++ .../AuthViews/VerifyUserFailedAuthView.tsx | 33 ++++++ .../AuthViews/VerifyUserSuccessAuthView.tsx | 37 +++++++ .../ForgotPassword/ForgotPassword.tsx | 96 +++++++++++++++++ .../Authentication/ForgotPassword/index.ts | 1 + .../ResetPassword/ResetPassword.tsx | 102 ++++++++++++++++++ .../Authentication/ResetPassword/index.ts | 1 + ui/src/views/Authentication/SignUp/SignUp.tsx | 2 + .../SignUp/SignUpVerification.tsx | 72 +++++++++++++ .../Authentication/VerifyUser/VerifyUser.tsx | 88 +++++++++++++++ .../views/Authentication/VerifyUser/index.ts | 1 + ui/src/views/Authentication/types.ts | 56 ++++++---- ui/src/views/Sidebar/Sidebar.tsx | 9 +- 21 files changed, 894 insertions(+), 43 deletions(-) create mode 100644 ui/src/views/Authentication/AuthViews/ForgotPasswordAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/ForgotPasswordSuccessAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/ResetPasswordAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/ResetPasswordSuccessAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/SignUpVerificationAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/VerifyUserAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/VerifyUserFailedAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/VerifyUserSuccessAuthView.tsx create mode 100644 ui/src/views/Authentication/ForgotPassword/ForgotPassword.tsx create mode 100644 ui/src/views/Authentication/ForgotPassword/index.ts create mode 100644 ui/src/views/Authentication/ResetPassword/ResetPassword.tsx create mode 100644 ui/src/views/Authentication/ResetPassword/index.ts create mode 100644 ui/src/views/Authentication/SignUp/SignUpVerification.tsx create mode 100644 ui/src/views/Authentication/VerifyUser/VerifyUser.tsx create mode 100644 ui/src/views/Authentication/VerifyUser/index.ts diff --git a/ui/src/routes/main.tsx b/ui/src/routes/main.tsx index f1563519..2292bb12 100644 --- a/ui/src/routes/main.tsx +++ b/ui/src/routes/main.tsx @@ -3,6 +3,13 @@ const AboutUs = lazy(() => import('@/views/AboutUs')); const Dashboard = lazy(() => import('@/views/Dashboard')); const SignIn = lazy(() => import('@/views/Authentication/SignIn')); const SignUp = lazy(() => import('@/views/Authentication/SignUp')); + +const SignUpVerification = lazy(() => import('@/views/Authentication/SignUp/SignUpVerification')); +const VerifyUser = lazy(() => import('@/views/Authentication/VerifyUser')); + +const ForgotPassword = lazy(() => import('@/views/Authentication/ForgotPassword')); +const ResetPassword = lazy(() => import('@/views/Authentication/ResetPassword')); + const Models = lazy(() => import('@/views/Models')); const SetupConnectors = lazy(() => import('@/views/Connectors/SetupConnectors')); @@ -102,4 +109,40 @@ export const AUTH_ROUTES: MAIN_PAGE_ROUTES_ITEM[] = [ ), }, + { + name: 'Sign Up Success', + url: '/sign-up/success', + component: ( + + + + ), + }, + { + name: 'Verify User', + url: '/verify-user', + component: ( + + + + ), + }, + { + name: 'Forgot Password', + url: '/forgot-password', + component: ( + + + + ), + }, + { + name: 'Reset Password', + url: '/reset-password', + component: ( + + + + ), + }, ]; diff --git a/ui/src/services/authentication.ts b/ui/src/services/authentication.ts index 657e9636..bcc9311d 100644 --- a/ui/src/services/authentication.ts +++ b/ui/src/services/authentication.ts @@ -13,7 +13,7 @@ export type SignInPayload = { password: string; }; -export type SignInErrorResponse = { +export type AuthErrorResponse = { status: number; title: string; detail: string; @@ -25,13 +25,7 @@ export type SignInResponse = { attributes: { token: string; }; - errors?: SignInErrorResponse[]; -}; - -export type AuthErrorResponse = { - status: number; - title: string; - detail: string; + errors?: AuthErrorResponse[]; }; export type AuthResponse = { @@ -40,11 +34,7 @@ export type AuthResponse = { attributes: { token: string; }; - errors?: Array<{ - source: { - [key: string]: string; - }; - }>; + errors?: AuthErrorResponse[]; }; export type ApiResponse = { @@ -53,8 +43,37 @@ export type ApiResponse = { errors?: ErrorResponse[]; }; +export type ForgotPasswordPayload = { + email: string; +}; + +export type MessageResponse = { + type: string; + id: number; + attributes: { + message: string; + }; +}; + +export type ResetPasswordPayload = { + reset_password_token: string; + password: string; + password_confirmation: string; +}; + +export type SignUpResponse = { + type: string; + id: string; + attributes: { + created_at: string; + email: string; + name: string; + }; + errors?: AuthErrorResponse[]; +}; + export const signUp = async (payload: SignUpPayload) => - multiwovenFetch>({ + multiwovenFetch>({ method: 'post', url: '/signup', data: payload, @@ -66,3 +85,30 @@ export const signIn = async (payload: SignInPayload) => url: '/login', data: payload, }); + +export const forgotPassword = async (payload: ForgotPasswordPayload) => + multiwovenFetch>({ + method: 'post', + url: '/forgot_password', + data: payload, + }); + +export const resetPassword = async (payload: ResetPasswordPayload) => + multiwovenFetch>({ + method: 'post', + url: '/reset_password', + data: payload, + }); + +export const verifyUser = async (confirmation_token: string) => + multiwovenFetch>({ + method: 'get', + url: `/verify_user?confirmation_token=${confirmation_token}`, + }); + +export const resendUserVerification = async (payload: ForgotPasswordPayload) => + multiwovenFetch>({ + method: 'post', + url: `/resend_verification`, + data: payload, + }); diff --git a/ui/src/views/Authentication/AuthCard/AuthCard.tsx b/ui/src/views/Authentication/AuthCard/AuthCard.tsx index 2c7583cd..b51145ab 100644 --- a/ui/src/views/Authentication/AuthCard/AuthCard.tsx +++ b/ui/src/views/Authentication/AuthCard/AuthCard.tsx @@ -1,5 +1,6 @@ import { Box, Container, Flex, Image, Stack } from '@chakra-ui/react'; import { AuthCardProps } from '../types'; +import multiwovenLogo from '@/assets/images/icon-white.svg'; const AuthCard = ({ children, brandName, logoUrl }: AuthCardProps): JSX.Element => ( <> @@ -17,7 +18,11 @@ const AuthCard = ({ children, brandName, logoUrl }: AuthCardProps): JSX.Element borderRadius='11px' mx='auto' > - + { + return ( + <> + + + {({ getFieldProps, touched, errors }) => ( +
+ + + Reset your password + + + Enter your email address and we will send you instructions to reset your password. + + + + + + + + + + + + + Back to {brandName} + + + + + + )} + + + + ); +}; diff --git a/ui/src/views/Authentication/AuthViews/ForgotPasswordSuccessAuthView.tsx b/ui/src/views/Authentication/AuthViews/ForgotPasswordSuccessAuthView.tsx new file mode 100644 index 00000000..b666aa89 --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/ForgotPasswordSuccessAuthView.tsx @@ -0,0 +1,45 @@ +import AuthCard from '../AuthCard'; +import { Button, Heading, Stack, Text } from '@chakra-ui/react'; + +type ForgotPasswordSuccessAuthViewProps = { + brandName: string; + logoUrl: string; + email: string; + handleEmailResend: (values: any) => void; + submitting: boolean; +}; + +export const ForgotPasswordSuccessAuthView = ({ + brandName, + logoUrl, + email, + handleEmailResend, + submitting, +}: ForgotPasswordSuccessAuthViewProps) => { + return ( + <> + + + + Check your email + + + Please check the email address {email} for instructions to reset your password. + + + + + + + + ); +}; diff --git a/ui/src/views/Authentication/AuthViews/ResetPasswordAuthView.tsx b/ui/src/views/Authentication/AuthViews/ResetPasswordAuthView.tsx new file mode 100644 index 00000000..08912681 --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/ResetPasswordAuthView.tsx @@ -0,0 +1,83 @@ +import AuthCard from '../AuthCard'; +import { Form, Formik } from 'formik'; +import { Button, HStack, Heading, Stack, Text } from '@chakra-ui/react'; +import { PasswordField } from '@/components/Fields'; +import { Link } from 'react-router-dom'; +import { ResetPasswordAuthViewProps } from '../types'; +import { ResetPasswordSchema } from '@/constants/schemas'; + +export const ResetPasswordAuthView = ({ + brandName, + logoUrl, + handleSubmit, + submitting, +}: ResetPasswordAuthViewProps) => { + return ( + <> + + + {({ getFieldProps, touched, errors }) => ( +
+ + + {'Reset your password'} + + + {`Enter a new password below to change your password`} + + + + + + + + + + + + + + Back to {brandName} + + + + + + )} + + + + ); +}; diff --git a/ui/src/views/Authentication/AuthViews/ResetPasswordSuccessAuthView.tsx b/ui/src/views/Authentication/AuthViews/ResetPasswordSuccessAuthView.tsx new file mode 100644 index 00000000..5828e2c8 --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/ResetPasswordSuccessAuthView.tsx @@ -0,0 +1,44 @@ +import { useNavigate } from 'react-router-dom'; +import AuthCard from '../AuthCard'; +import { Button, Heading, Stack, Text } from '@chakra-ui/react'; + +type ResetPasswordSuccessAuthViewProps = { + brandName: string; + logoUrl: string; + submitting: boolean; +}; + +export const ResetPasswordSuccessAuthView = ({ + brandName, + logoUrl, + submitting, +}: ResetPasswordSuccessAuthViewProps) => { + const navigate = useNavigate(); + + return ( + <> + + + + Password Changed! + + + Your password has been changed successfully. + + + + + + + + ); +}; diff --git a/ui/src/views/Authentication/AuthViews/SignUpVerificationAuthView.tsx b/ui/src/views/Authentication/AuthViews/SignUpVerificationAuthView.tsx new file mode 100644 index 00000000..878f28df --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/SignUpVerificationAuthView.tsx @@ -0,0 +1,36 @@ +import AuthCard from '../AuthCard'; +import { Button, Heading, Stack, Text } from '@chakra-ui/react'; +import { VerificationAuthViewProps } from '../types'; + +export const SignUpVerificationAuthView = ({ + brandName, + logoUrl, + email, + handleEmailResend, + submitting, +}: VerificationAuthViewProps) => ( + <> + + + + Check your email + + + To complete sign up, click the verification button in the email we sent to {email}. + + + + + + + +); diff --git a/ui/src/views/Authentication/AuthViews/VerifyUserAuthView.tsx b/ui/src/views/Authentication/AuthViews/VerifyUserAuthView.tsx new file mode 100644 index 00000000..8f2b14da --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/VerifyUserAuthView.tsx @@ -0,0 +1,36 @@ +import AuthCard from '../AuthCard'; +import { Button, Heading, Stack, Text } from '@chakra-ui/react'; +import { VerificationAuthViewProps } from '../types'; + +export const VerifyUserAuthView = ({ + brandName, + logoUrl, + email, + handleEmailResend, + submitting, +}: VerificationAuthViewProps) => ( + <> + + + + Check your email + + + To complete sign up, click the verification button in the email we sent to {email}. + + + + + + + +); diff --git a/ui/src/views/Authentication/AuthViews/VerifyUserFailedAuthView.tsx b/ui/src/views/Authentication/AuthViews/VerifyUserFailedAuthView.tsx new file mode 100644 index 00000000..31a7a86c --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/VerifyUserFailedAuthView.tsx @@ -0,0 +1,33 @@ +import AuthCard from '../AuthCard'; +import { Button, Heading, Stack, Text } from '@chakra-ui/react'; +import { VerifyUserFailedAuthViewProps } from '../types'; + +export const VerifyUserFailedAuthView = ({ + brandName, + logoUrl, + resendEmail, +}: VerifyUserFailedAuthViewProps) => ( + <> + + + + Link expired{' '} + + + To verify your email and complete sign up, please resend the verification email. + + + + + + + +); diff --git a/ui/src/views/Authentication/AuthViews/VerifyUserSuccessAuthView.tsx b/ui/src/views/Authentication/AuthViews/VerifyUserSuccessAuthView.tsx new file mode 100644 index 00000000..69b64d23 --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/VerifyUserSuccessAuthView.tsx @@ -0,0 +1,37 @@ +import { useNavigate } from 'react-router-dom'; +import AuthCard from '../AuthCard'; +import { Button, Heading, Stack, Text } from '@chakra-ui/react'; +import { VerifyUserSuccessAuthViewProps } from '../types'; + +export const VerifyUserSuccessAuthView = ({ + brandName, + logoUrl, +}: VerifyUserSuccessAuthViewProps) => { + const navigate = useNavigate(); + + return ( + <> + + + + Email verified! + + + Your account has been successfully verified. + + + + + + + + ); +}; diff --git a/ui/src/views/Authentication/ForgotPassword/ForgotPassword.tsx b/ui/src/views/Authentication/ForgotPassword/ForgotPassword.tsx new file mode 100644 index 00000000..d9563234 --- /dev/null +++ b/ui/src/views/Authentication/ForgotPassword/ForgotPassword.tsx @@ -0,0 +1,96 @@ +import { useState } from 'react'; +import { ForgotPasswordPayload, forgotPassword } from '@/services/authentication'; +import titleCase from '@/utils/TitleCase'; +import AuthFooter from '../AuthFooter'; +import { CustomToastStatus } from '@/components/Toast/index'; +import useCustomToast from '@/hooks/useCustomToast'; +import { useMutation } from '@tanstack/react-query'; + +import { ForgotPasswordAuthView } from '@/views/Authentication/AuthViews/ForgotPasswordAuthView'; + +import { ErrorResponse } from '@/services/common'; +import { ForgotPasswordSuccessAuthView } from '@/views/Authentication/AuthViews/ForgotPasswordSuccessAuthView'; +import mwTheme from '@/chakra.config'; + +const ForgotPassword = (): JSX.Element => { + const [submitting, setSubmitting] = useState(false); + const [success, setSuccess] = useState(false); + const [userEmail, setUserEmail] = useState(''); + + const showToast = useCustomToast(); + + const { mutateAsync } = useMutation({ + mutationFn: (values: ForgotPasswordPayload) => forgotPassword(values), + mutationKey: ['forgot-password'], + }); + + const handleSubmit = async (values: ForgotPasswordPayload) => { + setSubmitting(true); + try { + const result = await mutateAsync(values); + + if (result.data?.attributes.message) { + showToast({ + duration: 3000, + isClosable: true, + position: 'bottom-right', + title: result.data.attributes.message, + status: CustomToastStatus.Success, + }); + setUserEmail(values.email); + setSuccess(true); + } else { + result?.errors?.forEach((error: ErrorResponse) => { + showToast({ + duration: 5000, + isClosable: true, + position: 'bottom-right', + colorScheme: 'red', + status: CustomToastStatus.Warning, + title: titleCase(error.detail), + }); + }); + } + } catch (error) { + showToast({ + duration: 3000, + isClosable: true, + position: 'bottom-right', + title: 'There was an error connecting to the server. Please try again later.', + status: CustomToastStatus.Error, + }); + } finally { + setSubmitting(false); + } + }; + + const { logoUrl, brandName, privacyPolicyUrl, termsOfServiceUrl } = mwTheme; + + return ( + <> + {success ? ( + + ) : ( + + )} + + + ); +}; + +export default ForgotPassword; diff --git a/ui/src/views/Authentication/ForgotPassword/index.ts b/ui/src/views/Authentication/ForgotPassword/index.ts new file mode 100644 index 00000000..f7e2960d --- /dev/null +++ b/ui/src/views/Authentication/ForgotPassword/index.ts @@ -0,0 +1 @@ +export { default } from './ForgotPassword'; diff --git a/ui/src/views/Authentication/ResetPassword/ResetPassword.tsx b/ui/src/views/Authentication/ResetPassword/ResetPassword.tsx new file mode 100644 index 00000000..3b77f658 --- /dev/null +++ b/ui/src/views/Authentication/ResetPassword/ResetPassword.tsx @@ -0,0 +1,102 @@ +import { useState } from 'react'; +import { ResetPasswordPayload, resetPassword } from '@/services/authentication'; +import titleCase from '@/utils/TitleCase'; +import AuthFooter from '../AuthFooter'; +import { CustomToastStatus } from '@/components/Toast/index'; +import useCustomToast from '@/hooks/useCustomToast'; +import { useMutation } from '@tanstack/react-query'; + +import { ErrorResponse } from '@/services/common'; + +import { ResetPasswordAuthView } from '@/views/Authentication/AuthViews/ResetPasswordAuthView'; +import { ResetPasswordSuccessAuthView } from '@/views/Authentication/AuthViews/ResetPasswordSuccessAuthView'; +import { useSearchParams } from 'react-router-dom'; +import mwTheme from '@/chakra.config'; +import { ResetPasswordFormPayload } from '../types'; + +const ResetPassword = (): JSX.Element => { + const [submitting, setSubmitting] = useState(false); + const [success, setSuccess] = useState(false); + + const showToast = useCustomToast(); + + const [searchParams] = useSearchParams(); + const token = searchParams.get('reset_password_token'); + + const { mutateAsync } = useMutation({ + mutationFn: (values: ResetPasswordPayload) => resetPassword(values), + mutationKey: ['Reset-password'], + }); + + const handleSubmit = async (values: ResetPasswordFormPayload) => { + setSubmitting(true); + + if (token !== null) { + const payload = { ...values, reset_password_token: token }; + try { + const result = await mutateAsync(payload); + + if (result.data?.attributes.message) { + showToast({ + duration: 3000, + isClosable: true, + position: 'bottom-right', + title: result.data.attributes.message, + status: CustomToastStatus.Success, + }); + setSuccess(true); + } else { + result?.errors?.forEach((error: ErrorResponse) => { + showToast({ + duration: 5000, + isClosable: true, + position: 'bottom-right', + colorScheme: 'red', + status: CustomToastStatus.Warning, + title: titleCase(error.detail) + ' Please try resetting your password again.', + }); + }); + } + } catch (error) { + showToast({ + duration: 3000, + isClosable: true, + position: 'bottom-right', + title: 'There was an error connecting to the server. Please try again later.', + status: CustomToastStatus.Error, + }); + } finally { + setSubmitting(false); + } + } + }; + + const { logoUrl, brandName, privacyPolicyUrl, termsOfServiceUrl } = mwTheme; + + return ( + <> + {success ? ( + + ) : ( + + )} + + + + ); +}; + +export default ResetPassword; diff --git a/ui/src/views/Authentication/ResetPassword/index.ts b/ui/src/views/Authentication/ResetPassword/index.ts new file mode 100644 index 00000000..8e628c60 --- /dev/null +++ b/ui/src/views/Authentication/ResetPassword/index.ts @@ -0,0 +1 @@ +export { default } from './ResetPassword'; diff --git a/ui/src/views/Authentication/SignUp/SignUp.tsx b/ui/src/views/Authentication/SignUp/SignUp.tsx index 13fc6956..c336a940 100644 --- a/ui/src/views/Authentication/SignUp/SignUp.tsx +++ b/ui/src/views/Authentication/SignUp/SignUp.tsx @@ -1,5 +1,6 @@ import { useNavigate } from 'react-router-dom'; import { Stack, Heading } from '@chakra-ui/react'; + import mwTheme from '@/chakra.config'; import AuthFooter from '../AuthFooter'; import { SignUpAuthView } from '@/views/Authentication/AuthViews/SignUpAuthView'; @@ -9,6 +10,7 @@ import { SignUpPayload, signUp } from '@/services/authentication'; import { useState } from 'react'; import useCustomToast from '@/hooks/useCustomToast'; import { useMutation } from '@tanstack/react-query'; + import { useAPIErrorsToast, useErrorToast } from '@/hooks/useErrorToast'; // import isValidEmailDomain from '@/utils/isValidEmailDomain'; diff --git a/ui/src/views/Authentication/SignUp/SignUpVerification.tsx b/ui/src/views/Authentication/SignUp/SignUpVerification.tsx new file mode 100644 index 00000000..07d5e12d --- /dev/null +++ b/ui/src/views/Authentication/SignUp/SignUpVerification.tsx @@ -0,0 +1,72 @@ +import { useSearchParams } from 'react-router-dom'; +import mwTheme from '@/chakra.config'; +import AuthFooter from '../AuthFooter'; +import { CustomToastStatus } from '@/components/Toast'; +import { resendUserVerification } from '@/services/authentication'; +import { useState } from 'react'; +import useCustomToast from '@/hooks/useCustomToast'; +import { useMutation } from '@tanstack/react-query'; +import { SignUpVerificationAuthView } from '../AuthViews/SignUpVerificationAuthView'; +import { useAPIErrorsToast, useErrorToast } from '@/hooks/useErrorToast'; + +const SignUpVerification = (): JSX.Element => { + const [submitting, setSubmitting] = useState(false); + const showToast = useCustomToast(); + const apiErrorToast = useAPIErrorsToast(); + const errorToast = useErrorToast(); + + const [searchParams] = useSearchParams(); + const email = searchParams.get('email') || ''; + + const { mutateAsync } = useMutation({ + mutationFn: (email: string) => resendUserVerification({ email }), + mutationKey: ['verify-user'], + }); + + const handleEmailResend = async (email: string) => { + setSubmitting(true); + try { + const result = await mutateAsync(email); + + if (result.data?.attributes) { + showToast({ + title: result.data.attributes.message, + status: CustomToastStatus.Success, + duration: 3000, + isClosable: true, + position: 'bottom-right', + }); + + setSubmitting(false); + } else { + apiErrorToast(result.errors || []); + } + } catch (error) { + errorToast('An error occured. Please try again later.', true, null, true); + } finally { + setSubmitting(false); + } + }; + + const { logoUrl, brandName, privacyPolicyUrl, termsOfServiceUrl } = mwTheme; + + return ( + <> + handleEmailResend(email)} + submitting={submitting} + /> + + + + ); +}; + +export default SignUpVerification; diff --git a/ui/src/views/Authentication/VerifyUser/VerifyUser.tsx b/ui/src/views/Authentication/VerifyUser/VerifyUser.tsx new file mode 100644 index 00000000..efbc08ef --- /dev/null +++ b/ui/src/views/Authentication/VerifyUser/VerifyUser.tsx @@ -0,0 +1,88 @@ +import { useEffect, useState } from 'react'; +import { useSearchParams } from 'react-router-dom'; +import mwTheme from '@/chakra.config'; +import AuthFooter from '../AuthFooter'; +import { CustomToastStatus } from '@/components/Toast'; +import { resendUserVerification, verifyUser } from '@/services/authentication'; +import useCustomToast from '@/hooks/useCustomToast'; +import { useMutation } from '@tanstack/react-query'; +import { VerifyUserSuccessAuthView } from '../AuthViews/VerifyUserSuccessAuthView'; +import { VerifyUserFailedAuthView } from '../AuthViews/VerifyUserFailedAuthView'; +import Loader from '@/components/Loader'; +import { useAPIErrorsToast, useErrorToast } from '@/hooks/useErrorToast'; + +const VerifyUser = (): JSX.Element => { + const [success, setSuccess] = useState(false); + const [submitting, setSubmitting] = useState(true); // Add this to handle the final state + + const showToast = useCustomToast(); + const apiErrorToast = useAPIErrorsToast(); + const errorToast = useErrorToast(); + + const [searchParams] = useSearchParams(); + const confirmation_token = searchParams.get('confirmation_token'); + const email = searchParams.get('email') || ''; + + const { mutateAsync } = useMutation({ + mutationFn: (email: string) => resendUserVerification({ email }), + mutationKey: ['resend-verification'], + }); + + const { logoUrl, brandName, privacyPolicyUrl, termsOfServiceUrl } = mwTheme; + + const verifyUserToken = async () => { + if (!confirmation_token) { + setSubmitting(false); + return; + } + + try { + const result = await verifyUser(confirmation_token); + if (result.data?.attributes) { + showToast({ + title: 'Account verified.', + status: CustomToastStatus.Success, + duration: 3000, + isClosable: true, + position: 'bottom-right', + }); + setSuccess(true); + } else { + apiErrorToast(result.errors || []); + } + } catch (error) { + errorToast('An error occured. Please try again later.', true, null, true); + } finally { + setSubmitting(false); + } + }; + + useEffect(() => { + verifyUserToken(); + }, []); + + if (submitting) { + return ; + } + + return ( + <> + {success ? ( + + ) : ( + mutateAsync(email)} + /> + )} + + + ); +}; + +export default VerifyUser; diff --git a/ui/src/views/Authentication/VerifyUser/index.ts b/ui/src/views/Authentication/VerifyUser/index.ts new file mode 100644 index 00000000..dc1e1664 --- /dev/null +++ b/ui/src/views/Authentication/VerifyUser/index.ts @@ -0,0 +1 @@ +export { default } from './VerifyUser'; diff --git a/ui/src/views/Authentication/types.ts b/ui/src/views/Authentication/types.ts index f0168859..a4e4b673 100644 --- a/ui/src/views/Authentication/types.ts +++ b/ui/src/views/Authentication/types.ts @@ -1,28 +1,35 @@ import { ReactNode } from 'react'; -export type AuthCardProps = { - children: ReactNode; + +type DefaultAuthViewProps = { brandName: string; logoUrl: string; }; +type Email = { + email: string; +}; + +type Password = { + password: string; + password_confirmation: string; +}; + +export type AuthCardProps = { + children: ReactNode; +} & DefaultAuthViewProps; + export type SignInAuthViewProps = { - brandName: string; - logoUrl: string; handleSubmit: (values: any) => void; submitting: boolean; -}; +} & DefaultAuthViewProps; type InitialValues = { company_name: string; name: string; - email: string; - password: string; - password_confirmation: string; -}; +} & Email & + Password; export type SignUpAuthViewProps = { - brandName: string; - logoUrl: string; handleSubmit: (values: any) => void; submitting: boolean; initialValues?: InitialValues; @@ -30,23 +37,28 @@ export type SignUpAuthViewProps = { termsOfServiceUrl: string; isCompanyNameDisabled?: boolean; isEmailDisabled?: boolean; -}; +} & DefaultAuthViewProps; -export type ResetPasswordFormPayload = { - password: string; - password_confirmation: string; -}; +export type ResetPasswordFormPayload = Password; -export type ForgotPasswordFormPayload = { - email: string; -}; +export type ForgotPasswordFormPayload = Email; type ChangePasswordProps = { - brandName: string; - logoUrl: string; handleSubmit: (values: T) => void; submitting: boolean; -}; +} & DefaultAuthViewProps; export type ForgotPasswordAuthViewProps = ChangePasswordProps; export type ResetPasswordAuthViewProps = ChangePasswordProps; + +export type VerificationAuthViewProps = { + handleEmailResend: (email: string) => void; + submitting: boolean; +} & DefaultAuthViewProps & + Email; + +export type VerifyUserFailedAuthViewProps = { + resendEmail: () => void; +} & DefaultAuthViewProps; + +export type VerifyUserSuccessAuthViewProps = DefaultAuthViewProps; diff --git a/ui/src/views/Sidebar/Sidebar.tsx b/ui/src/views/Sidebar/Sidebar.tsx index 081168dd..5e798b52 100644 --- a/ui/src/views/Sidebar/Sidebar.tsx +++ b/ui/src/views/Sidebar/Sidebar.tsx @@ -1,6 +1,6 @@ import { Box, Flex, Stack, Text, Divider } from '@chakra-ui/react'; import { NavLink } from 'react-router-dom'; -import IconImage from '../../assets/images/multiwoven-logo.svg'; +import IconImage from '@/assets/images/multiwoven-logo.svg'; import { FiSettings, FiDatabase, @@ -15,8 +15,6 @@ import { NavButton } from './navButton'; import Profile from './Profile'; import Workspace from './Workspace/Workspace'; -import { useConfigStore } from '@/stores/useConfigStore'; - type MenuItem = { title: string; link: string; @@ -76,7 +74,7 @@ const renderMenuSection = (section: MenuSection, index: number) => ( )} {section.menu.map((menuItem, idx) => ( - + {({ isActive }) => ( ( ); const Sidebar = (): JSX.Element => { - const { logoUrl } = useConfigStore.getState().configs; return ( { - IconImage + IconImage