From 6b790fcdf239af4465de8db4c90167ec590326a9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:34:40 +0530 Subject: [PATCH 01/31] feat(CE): filter connectors based on the category (#338) Co-authored-by: datafloyd --- .../v1/connector_definitions_controller.rb | 12 ++-- .../filter_connectors.rb | 56 +++++++++++++++ .../filter_connectors_spec.rb | 69 +++++++++++++++++++ 3 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 server/app/interactors/connector_definitions/filter_connectors.rb create mode 100644 server/spec/interactors/connector_definitions/filter_connectors_spec.rb diff --git a/server/app/controllers/api/v1/connector_definitions_controller.rb b/server/app/controllers/api/v1/connector_definitions_controller.rb index 7b127b2d..5298de25 100644 --- a/server/app/controllers/api/v1/connector_definitions_controller.rb +++ b/server/app/controllers/api/v1/connector_definitions_controller.rb @@ -3,6 +3,7 @@ module Api module V1 class ConnectorDefinitionsController < ApplicationController + include ConnectorDefinitions before_action :set_connectors, only: %i[show index] before_action :set_connector_client, only: %i[check_connection] @@ -33,10 +34,9 @@ def check_connection private def set_connectors - @connectors = Multiwoven::Integrations::Service - .connectors - .with_indifferent_access - @connectors = @connectors[params[:type]] if params[:type] + @connectors = FilterConnectors.call( + connection_definitions_params + ).connectors end def set_connector_client @@ -46,6 +46,10 @@ def set_connector_client params[:name].camelize ).new end + + def connection_definitions_params + params.permit(:type, :catagory) + end end end end diff --git a/server/app/interactors/connector_definitions/filter_connectors.rb b/server/app/interactors/connector_definitions/filter_connectors.rb new file mode 100644 index 00000000..4d266f23 --- /dev/null +++ b/server/app/interactors/connector_definitions/filter_connectors.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module ConnectorDefinitions + class FilterConnectors + include Interactor + + # TODO: Move this to integration so that whenever a new category is introduced, + # integrations will be the single source of truth for categories + + DATA_CATEGORIES = [ + "Data Warehouse", + "Retail", + "Data Lake", + "Database", + "Marketing Automation", + "CRM", + "Ad-Tech", + "Team Collaboration", + "Productivity Tools", + "Payments", + "File Storage", + "HTTP", + "Customer Support" + ].freeze + + AI_ML_CATEGORIES = [ + "AI Model" + ].freeze + + def call + context.connectors = Multiwoven::Integrations::Service.connectors.with_indifferent_access + + filter_connectors_by_category if context.category + context.connectors = context.connectors[context.type] if context.type + end + + private + + def filter_connectors_by_category + categories = case context.category + when "data" + DATA_CATEGORIES + when "ai_ml" + AI_ML_CATEGORIES + else + [context.category] + end + context.connectors[:source] = filter_by_category(context.connectors[:source], categories) + context.connectors[:destination] = filter_by_category(context.connectors[:destination], categories) + end + + def filter_by_category(connectors, categories) + connectors.select { |connector| categories.include?(connector[:category]) } + end + end +end diff --git a/server/spec/interactors/connector_definitions/filter_connectors_spec.rb b/server/spec/interactors/connector_definitions/filter_connectors_spec.rb new file mode 100644 index 00000000..a7597e86 --- /dev/null +++ b/server/spec/interactors/connector_definitions/filter_connectors_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe ConnectorDefinitions::FilterConnectors, type: :interactor do + let(:connectors) do + { + source: [ + { name: "Snowflake", category: "Data Warehouse", connector_type: "source" }, + { name: "Redshift", category: "Data Warehouse", connector_type: "source" }, + { name: "DatabricksModel", category: "AI Model", connector_type: "source" } + ], + destination: [ + { name: "Klaviyo", category: "Marketing Automation", connector_type: "destination" }, + { name: "SalesforceCrm", category: "CRM", connector_type: "destination" }, + { name: "FacebookCustomAudience", category: "Ad-Tech", connector_type: "destination" } + ] + }.with_indifferent_access + end + + before do + allow(Multiwoven::Integrations::Service).to receive(:connectors).and_return(connectors) + end + + describe "#call" do + context "when filtering by type only" do + it "returns only the destination connectors when type is destination" do + result = described_class.call(type: "destination") + + expect(result.connectors).to match_array(connectors[:destination]) + end + + it "returns only the source connectors when type is source" do + result = described_class.call(type: "source") + + expect(result.connectors).to match_array(connectors[:source]) + end + end + + context "when filtering by category and type" do + it "returns only the AI Model connectors for source type" do + result = described_class.call(type: "source", category: "ai_ml") + + expect(result.connectors).to match_array([connectors[:source][2]]) + end + + it "returns only the Data Warehouse connectors for source type" do + result = described_class.call(type: "source", category: "data") + + expect(result.connectors).to match_array([connectors[:source][0], connectors[:source][1]]) + end + + it "returns only the Marketing Automation connectors for destination type" do + result = described_class.call(type: "destination", category: "data") + + expect(result.connectors).to match_array([connectors[:destination][0], connectors[:destination][1], + connectors[:destination][2]]) + end + end + + context "when no category is specified" do + it "returns all connectors of the specified type" do + result = described_class.call(type: "destination") + + expect(result.connectors).to match_array(connectors[:destination]) + end + end + end +end From 8824e995689179f4ad8062fc9d6215798bc0e671 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:35:41 +0530 Subject: [PATCH 02/31] fix(CE): refresh catalog only when refresh is true (#336) Co-authored-by: datafloyd --- .../app/interactors/connectors/discover_connector.rb | 2 +- .../connectors/discover_connector_spec.rb | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/server/app/interactors/connectors/discover_connector.rb b/server/app/interactors/connectors/discover_connector.rb index 66fe949e..295aeabe 100644 --- a/server/app/interactors/connectors/discover_connector.rb +++ b/server/app/interactors/connectors/discover_connector.rb @@ -7,7 +7,7 @@ class DiscoverConnector def call context.catalog = context.connector.catalog # refresh catalog when the refresh flag is true - return if context.catalog.present? && !context.refresh + return if context.catalog.present? && context.refresh != "true" catalog = context.connector.build_catalog( workspace_id: context.connector.workspace_id diff --git a/server/spec/interactors/connectors/discover_connector_spec.rb b/server/spec/interactors/connectors/discover_connector_spec.rb index 3a68b5b9..9429a710 100644 --- a/server/spec/interactors/connectors/discover_connector_spec.rb +++ b/server/spec/interactors/connectors/discover_connector_spec.rb @@ -8,7 +8,15 @@ let(:connector_client) { double("ConnectorClient") } let(:streams) { "stream_test_data" } - context "when catalog is already present" do + context "when catalog is already present with refresh param present" do + it "returns existing catalog" do + catalog = create(:catalog, connector:) + result = described_class.call(connector:, refresh: "false") + expect(result.catalog).to eq(catalog) + end + end + + context "when catalog is already present with refresh param absent" do it "returns existing catalog" do catalog = create(:catalog, connector:) result = described_class.call(connector:) @@ -20,7 +28,7 @@ it "returns refreshed catalog" do catalog = create(:catalog, connector:) allow_any_instance_of(described_class).to receive(:streams).and_return(streams) - result = described_class.call(connector:, refresh: true) + result = described_class.call(connector:, refresh: "true") expect(result.catalog).not_to eq(catalog) end end From c6dcd060e66393ad73111218ac2f9de077089f77 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Tue, 27 Aug 2024 04:40:35 -0400 Subject: [PATCH 03/31] Multiwoven release v0.22.0 (#339) Co-authored-by: github-actions --- release-notes.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/release-notes.md b/release-notes.md index fdaa3e73..35a1828a 100644 --- a/release-notes.md +++ b/release-notes.md @@ -2,27 +2,30 @@ All notable changes to this project will be documented in this file. -## [0.21.0] - 2024-08-19 +## [0.22.0] - 2024-08-26 ### 🚀 Features -- *(CE)* Destination/microsoft excel (#314) +- *(CE)* Enable/disable sync (#321) +- *(CE)* Databricks AI Model connector (#325) +- *(CE)* Catalog creation via API (#329) +- *(CE)* Calalog update api (#333) +- *(CE)* Add AI/ML query type to model (#334) ### 🐛 Bug Fixes -- *(CE)* Memory bloat issue in sync (#300) -- *(CE)* Fix discover and table url (#316) +- *(CE)* Correction in databricks model connection_spec (#331) ### 🚜 Refactor -- *(CE)* Added disable to fields +- *(CE)* Changed condition to render toast (#315) ### ⚙️ Miscellaneous Tasks -- *(CE)* Update the query_source response (#305) -- *(CE)* Fix oci8 version (#313) -- *(CE)* Update user read permission (#320) -- *(CE)* Update server gem (#312) -- *(CE)* Update connector name (#317) +- *(CE)* Add request response log for Hubspot (#323) +- *(CE)* Add request response log for Stripe (#328) +- *(CE)* Add request response log for SalesforceCrm (#326) +- *(CE)* Upgrade version to 0.9.1 (#330) +- *(CE)* Refactor error message to agreed standard (#332) From 24463f7805ed46f4034abaf9a131956a9433864b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:24:46 +0400 Subject: [PATCH 04/31] Resolve conflict in cherry-pick of 9a4860287a9bda117c8096cdd33ca37e823de242 and change the commit message (#337) Co-authored-by: datafloyd Co-authored-by: afthab vp --- .../v1/connector_definitions_controller.rb | 2 +- .../api/v1/connector_definitions_spec.rb | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/server/app/controllers/api/v1/connector_definitions_controller.rb b/server/app/controllers/api/v1/connector_definitions_controller.rb index 5298de25..a4c9491c 100644 --- a/server/app/controllers/api/v1/connector_definitions_controller.rb +++ b/server/app/controllers/api/v1/connector_definitions_controller.rb @@ -48,7 +48,7 @@ def set_connector_client end def connection_definitions_params - params.permit(:type, :catagory) + params.permit(:type, :category) end end end diff --git a/server/spec/requests/api/v1/connector_definitions_spec.rb b/server/spec/requests/api/v1/connector_definitions_spec.rb index 42d8e008..aa3956ab 100644 --- a/server/spec/requests/api/v1/connector_definitions_spec.rb +++ b/server/spec/requests/api/v1/connector_definitions_spec.rb @@ -33,6 +33,34 @@ expect(response_hash[:destination].count).to eql(service.connectors[:destination].count) end + it "returns only ai/ml sources" do + get "/api/v1/connector_definitions?type=source&category=ai_ml", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + + response_hash = JSON.parse(response.body) + + categories = response_hash.map { |item| item["category"] }.uniq + connector_types = response_hash.map { |item| item["connector_type"] }.uniq + expect(categories.count).to eql(1) + expect(connector_types.count).to eql(1) + expect(categories).to eql(["AI Model"]) + expect(connector_types).to eql(["source"]) + end + + it "returns only ai/ml connectors" do + get "/api/v1/connector_definitions?category=ai_ml", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + + response_hash = JSON.parse(response.body).with_indifferent_access + + source_categories = response_hash[:source].map { |item| item["category"] }.uniq + destination_categories = response_hash[:destination].map { |item| item["category"] }.uniq + + categories = (source_categories + destination_categories).uniq + expect(categories.count).to eql(1) + expect(categories).to eql(["AI Model"]) + end + it "returns success viewer role" do workspace.workspace_users.first.update(role: viewer_role) get "/api/v1/connector_definitions", headers: auth_headers(user, workspace_id) From f89f3638433d4027487c5126753c71559971628c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:39:46 +0530 Subject: [PATCH 05/31] chore(CE): validate catalog for query source (#340) Co-authored-by: datafloyd --- .../api/v1/connectors_controller.rb | 12 +++++++ .../api/v1/connectors_controller_spec.rb | 32 +++++++++++++++---- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/server/app/controllers/api/v1/connectors_controller.rb b/server/app/controllers/api/v1/connectors_controller.rb index b26fe46b..fe33b258 100644 --- a/server/app/controllers/api/v1/connectors_controller.rb +++ b/server/app/controllers/api/v1/connectors_controller.rb @@ -2,11 +2,13 @@ module Api module V1 + # rubocop:disable Metrics/ClassLength class ConnectorsController < ApplicationController include Connectors before_action :set_connector, only: %i[show update destroy discover query_source] # TODO: Enable this once we have query validation implemented for all the connectors # before_action :validate_query, only: %i[query_source] + before_action :validate_catalog, only: %i[query_source] after_action :event_logger def index @@ -121,6 +123,15 @@ def set_connector ) end + def validate_catalog + return if @connector.catalog.present? + + render_error( + message: "Catalog is not present for the connector", + status: :unprocessable_entity + ) + end + def validate_query Utils::QueryValidator.validate_query(@connector.connector_query_type, params[:query]) rescue StandardError => e @@ -137,5 +148,6 @@ def connector_params configuration: {}) end end + # rubocop:enable Metrics/ClassLength end end diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb index cf970563..fd430c89 100644 --- a/server/spec/requests/api/v1/connectors_controller_spec.rb +++ b/server/spec/requests/api/v1/connectors_controller_spec.rb @@ -406,7 +406,7 @@ end describe "POST /api/v1/connectors/id/query_source" do - let(:connector) { create(:connector, connector_type: "source") } + let(:connector) { create(:connector, connector_type: "source", workspace:, connector_name: "AmazonS3") } let(:query) { "SELECT * FROM table_name" } let(:limit) { 50 } let(:record1) do @@ -424,9 +424,13 @@ } end + before do + create(:catalog, connector_id: connector.id, workspace:) + end + context "when it is an unauthenticated user" do it "returns unauthorized" do - post "/api/v1/connectors/#{connectors.second.id}/query_source" + post "/api/v1/connectors/#{connector.id}/query_source" expect(response).to have_http_status(:unauthorized) end end @@ -435,7 +439,7 @@ it "returns success status for a valid query" do allow(Connectors::QuerySource).to receive(:call) .and_return(double(:context, success?: true, records: [record1, record2])) - post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access @@ -446,18 +450,32 @@ workspace.workspace_users.first.update(role: member_role) allow(Connectors::QuerySource).to receive(:call) .and_return(double(:context, success?: true, records: [record1, record2])) - post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) end + it "returns an error message for missing catalog" do + catalog = connector.catalog + catalog.connector_id = connectors.second.id + catalog.save + + allow(Connectors::QuerySource).to receive(:call).and_return(double(:context, success?: true, + records: [record1, record2])) + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: + { "Content-Type": "application/json" }.merge(auth_headers(user, workspace.id)) + expect(response).to have_http_status(:unprocessable_entity) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash.dig(:errors, 0, :detail)).to eq("Catalog is not present for the connector") + end + it "returns success status for a valid query for viewer role" do workspace.workspace_users.first.update(role: viewer_role) allow(Connectors::QuerySource).to receive(:call) .and_return(double(:context, success?: true, records: [record1, record2])) - post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access @@ -467,7 +485,7 @@ it "returns failure status for a invalid query" do allow(Connectors::QuerySource).to receive(:call).and_raise(StandardError, "query failed") - post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:bad_request) @@ -475,7 +493,7 @@ it "returns failure status for a invalid query" do request_body[:query] = "invalid" - post "/api/v1/connectors/#{connectors.second.id}/query_source", params: request_body.to_json, headers: + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: { "Content-Type": "application/json" }.merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:unprocessable_entity) From b93a8e2638f9c251eed2943b2a2c2569d698e90f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:40:29 +0530 Subject: [PATCH 06/31] chore(CE): add catalog presence validation for models (#341) Co-authored-by: datafloyd --- .../controllers/api/v1/models_controller.rb | 11 ++++++++ .../requests/api/v1/models_controller_spec.rb | 28 ++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/server/app/controllers/api/v1/models_controller.rb b/server/app/controllers/api/v1/models_controller.rb index f39dadfa..6b2063e6 100644 --- a/server/app/controllers/api/v1/models_controller.rb +++ b/server/app/controllers/api/v1/models_controller.rb @@ -8,6 +8,7 @@ class ModelsController < ApplicationController before_action :set_connector, only: %i[create] before_action :set_model, only: %i[show update destroy] + before_action :validate_catalog, only: %i[create update] # TODO: Enable this once we have query validation implemented for all the connectors # before_action :validate_query, only: %i[create update] after_action :event_logger @@ -75,6 +76,16 @@ def set_connector def set_model @model = current_workspace.models.find(params[:id]) + @connector = @model.connector + end + + def validate_catalog + return if connector.catalog.present? + + render_error( + message: "Catalog is not present for the connector", + status: :unprocessable_entity + ) end def validate_query diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb index 7ac61917..3301e72f 100644 --- a/server/spec/requests/api/v1/models_controller_spec.rb +++ b/server/spec/requests/api/v1/models_controller_spec.rb @@ -9,6 +9,10 @@ let(:connector) do create(:connector, workspace:, connector_type: "destination", name: "klavio1", connector_name: "Klaviyo") end + + let(:connector_without_catalog) do + create(:connector, workspace:, connector_type: "destination", name: "klavio1", connector_name: "Klaviyo") + end let!(:models) do [ create(:model, connector:, workspace:, name: "model1", query: "SELECT * FROM locations"), @@ -19,6 +23,7 @@ let(:member_role) { create(:role, :member) } before do + create(:catalog, connector:) user.confirm end @@ -152,7 +157,16 @@ expect(response_hash.dig(:data, :attributes, :primary_key)).to eq(request_body.dig(:model, :primary_key)) end - it "creates a new model and returns success" do + it "fails model creation for connector without catalog" do + workspace.workspace_users.first.update(role: member_role) + # set connector without catalog + request_body[:model][:connector_id] = connector_without_catalog.id + post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:unprocessable_entity) + end + + it " creates a new model and returns success" do workspace.workspace_users.first.update(role: member_role) post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" } .merge(auth_headers(user, workspace_id)) @@ -255,6 +269,18 @@ expect(response_hash.dig(:data, :attributes, :primary_key)).to eq(request_body.dig(:model, :primary_key)) end + it "fails model update for connector without catalog" do + workspace.workspace_users.first.update(role: member_role) + model = models.second + model.connector_id = connector_without_catalog.id + model.save! + + put "/api/v1/models/#{models.second.id}", params: request_body.to_json, + headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:unprocessable_entity) + end + it "updates the model and returns success for member role" do workspace.workspace_users.first.update(role: member_role) put "/api/v1/models/#{models.second.id}", params: request_body.to_json, headers: From 684fa4610c78bc9b584011d3cc2024a62abe27d0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:40:59 +0530 Subject: [PATCH 07/31] fix(CE): connection_spec mandatory changes (#342) Co-authored-by: afthab vp --- integrations/Gemfile.lock | 2 +- integrations/lib/multiwoven/integrations/rollout.rb | 2 +- .../integrations/source/databrics_model/config/spec.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 160d7ed9..0bcecf8b 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.9.1) + multiwoven-integrations (0.9.2) activesupport async-websocket aws-sdk-athena diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index a2409100..1447eb3a 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.9.1" + VERSION = "0.9.2" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json b/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json index 9fb6ddac..40d6b9ec 100644 --- a/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json +++ b/integrations/lib/multiwoven/integrations/source/databrics_model/config/spec.json @@ -6,7 +6,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Databricks Model", "type": "object", - "required": ["databricks_host", "token", "endpoint"], + "required": ["databricks_host", "token", "endpoint", "request_format", "response_format"], "properties": { "databricks_host": { "title": "databricks_host", From 4c900da0791447860641fde49e44425faea793df Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 22:59:28 +0530 Subject: [PATCH 08/31] fix(CE): added tag label to auto-truncate text --- ui/src/components/StatusTag/StatusTag.tsx | 40 +++++++++++------------ 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/ui/src/components/StatusTag/StatusTag.tsx b/ui/src/components/StatusTag/StatusTag.tsx index 7671bc2e..a6fe73dc 100644 --- a/ui/src/components/StatusTag/StatusTag.tsx +++ b/ui/src/components/StatusTag/StatusTag.tsx @@ -1,4 +1,4 @@ -import { Tag, Text } from '@chakra-ui/react'; +import { Tag, TagLabel } from '@chakra-ui/react'; export enum StatusTagVariants { success = 'success', @@ -79,25 +79,23 @@ export const StatusTagText = { failed: 'Failed', }; -const StatusTag = ({ status, variant = StatusTagVariants.success }: StatusTagProps) => { - return ( - - - {status} - - - ); -}; +const StatusTag = ({ status, variant = StatusTagVariants.success }: StatusTagProps) => ( + + + {status} + + +); export default StatusTag; From 7dae4f070c66b50a2d8eac2737bdf0c501636381 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:49:05 +0530 Subject: [PATCH 09/31] feat(CE): filter connectors (#343) Co-authored-by: Subin T P --- .../api/v1/connectors_controller.rb | 1 + .../api/v1/connectors_controller_spec.rb | 34 ++++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/server/app/controllers/api/v1/connectors_controller.rb b/server/app/controllers/api/v1/connectors_controller.rb index fe33b258..77bbcd08 100644 --- a/server/app/controllers/api/v1/connectors_controller.rb +++ b/server/app/controllers/api/v1/connectors_controller.rb @@ -15,6 +15,7 @@ def index @connectors = current_workspace.connectors authorize @connectors @connectors = @connectors.send(params[:type].downcase) if params[:type] + @connectors = @connectors.send(params[:category].downcase) if params[:category] @connectors = @connectors.page(params[:page] || 1) render json: @connectors, status: :ok end diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb index fd430c89..e5f194b1 100644 --- a/server/spec/requests/api/v1/connectors_controller_spec.rb +++ b/server/spec/requests/api/v1/connectors_controller_spec.rb @@ -14,6 +14,9 @@ create(:connector, workspace:, connector_type: "source", name: "redshift", connector_name: "Redshift") ] end + let!(:data_connector) { create(:connector, workspace:, connector_category: "Data Warehouse") } + let!(:ai_ml_connector) { create(:connector, workspace:, connector_category: "AI Model") } + let!(:other_connector) { create(:connector, workspace:, connector_category: "CRM") } before do user.confirm @@ -33,7 +36,7 @@ get "/api/v1/connectors", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(connectors.count) + expect(response_hash[:data].count).to eql(connectors.count + 3) expect(response_hash.dig(:data, 0, :type)).to eq("connectors") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/connectors?page=1") end @@ -43,7 +46,7 @@ get "/api/v1/connectors", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(connectors.count) + expect(response_hash[:data].count).to eql(connectors.count + 3) expect(response_hash.dig(:data, 0, :type)).to eq("connectors") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/connectors?page=1") end @@ -53,7 +56,7 @@ get "/api/v1/connectors", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(connectors.count) + expect(response_hash[:data].count).to eql(connectors.count + 3) expect(response_hash.dig(:data, 0, :type)).to eq("connectors") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/connectors?page=1") end @@ -63,7 +66,7 @@ get "/api/v1/connectors", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(connectors.count) + expect(response_hash[:data].count).to eql(connectors.count + 3) expect(response_hash.dig(:data, 0, :type)).to eq("connectors") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/connectors?page=1") end @@ -82,12 +85,33 @@ get "/api/v1/connectors?type=destination", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(1) + expect(response_hash[:data].count).to eql(4) expect(response_hash.dig(:data, 0, :type)).to eq("connectors") expect(response_hash.dig(:data, 0, :attributes, :connector_type)).to eql("destination") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/connectors?page=1") end + it "returns only data connectors" do + get "/api/v1/connectors?category=data", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + result = JSON.parse(response.body) + expect(result["data"].map { |connector| connector["id"] }).not_to include(ai_ml_connector.id.to_s) + end + + it "returns only ai_ml connectors" do + get "/api/v1/connectors?category=ai_ml", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + result = JSON.parse(response.body) + expect(result["data"].map { |connector| connector["id"] }).to eql([ai_ml_connector.id.to_s]) + end + + it "returns only ai_ml connectors for source" do + get "/api/v1/connectors?type=source&&category=ai_ml", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + result = JSON.parse(response.body) + expect(result["data"].count).to eql(0) + end + it "returns an error response for connectors" do get "/api/v1/connectors?type=destination1", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:bad_request) From 33e490bfa9d3ad41127a750d5c4112ecdbf81a7d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:49:52 +0530 Subject: [PATCH 10/31] chore(CE): refactor interactors (#344) Co-authored-by: Subin T P --- .../v1/connector_definitions_controller.rb | 2 +- ...ors.rb => filter_connector_definitions.rb} | 29 ++----------------- ...b => filter_connector_definitions_spec.rb} | 2 +- 3 files changed, 5 insertions(+), 28 deletions(-) rename server/app/interactors/connector_definitions/{filter_connectors.rb => filter_connector_definitions.rb} (60%) rename server/spec/interactors/connector_definitions/{filter_connectors_spec.rb => filter_connector_definitions_spec.rb} (96%) diff --git a/server/app/controllers/api/v1/connector_definitions_controller.rb b/server/app/controllers/api/v1/connector_definitions_controller.rb index a4c9491c..ed901652 100644 --- a/server/app/controllers/api/v1/connector_definitions_controller.rb +++ b/server/app/controllers/api/v1/connector_definitions_controller.rb @@ -34,7 +34,7 @@ def check_connection private def set_connectors - @connectors = FilterConnectors.call( + @connectors = FilterConnectorDefinitions.call( connection_definitions_params ).connectors end diff --git a/server/app/interactors/connector_definitions/filter_connectors.rb b/server/app/interactors/connector_definitions/filter_connector_definitions.rb similarity index 60% rename from server/app/interactors/connector_definitions/filter_connectors.rb rename to server/app/interactors/connector_definitions/filter_connector_definitions.rb index 4d266f23..0ebbcfee 100644 --- a/server/app/interactors/connector_definitions/filter_connectors.rb +++ b/server/app/interactors/connector_definitions/filter_connector_definitions.rb @@ -1,32 +1,9 @@ # frozen_string_literal: true module ConnectorDefinitions - class FilterConnectors + class FilterConnectorDefinitions include Interactor - # TODO: Move this to integration so that whenever a new category is introduced, - # integrations will be the single source of truth for categories - - DATA_CATEGORIES = [ - "Data Warehouse", - "Retail", - "Data Lake", - "Database", - "Marketing Automation", - "CRM", - "Ad-Tech", - "Team Collaboration", - "Productivity Tools", - "Payments", - "File Storage", - "HTTP", - "Customer Support" - ].freeze - - AI_ML_CATEGORIES = [ - "AI Model" - ].freeze - def call context.connectors = Multiwoven::Integrations::Service.connectors.with_indifferent_access @@ -39,9 +16,9 @@ def call def filter_connectors_by_category categories = case context.category when "data" - DATA_CATEGORIES + Connector::DATA_CATEGORIES when "ai_ml" - AI_ML_CATEGORIES + Connector::AI_ML_CATEGORIES else [context.category] end diff --git a/server/spec/interactors/connector_definitions/filter_connectors_spec.rb b/server/spec/interactors/connector_definitions/filter_connector_definitions_spec.rb similarity index 96% rename from server/spec/interactors/connector_definitions/filter_connectors_spec.rb rename to server/spec/interactors/connector_definitions/filter_connector_definitions_spec.rb index a7597e86..f6b83014 100644 --- a/server/spec/interactors/connector_definitions/filter_connectors_spec.rb +++ b/server/spec/interactors/connector_definitions/filter_connector_definitions_spec.rb @@ -2,7 +2,7 @@ require "rails_helper" -RSpec.describe ConnectorDefinitions::FilterConnectors, type: :interactor do +RSpec.describe ConnectorDefinitions::FilterConnectorDefinitions, type: :interactor do let(:connectors) do { source: [ From b927e706b1c8a3cd6980de1fb007afbb0ea2856d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:50:44 +0530 Subject: [PATCH 11/31] feat(CE): filter model based on the query type (#347) Co-authored-by: datafloyd --- .../controllers/api/v1/models_controller.rb | 3 +- server/app/models/model.rb | 3 ++ server/spec/models/model_spec.rb | 25 ++++++++++++++ .../requests/api/v1/models_controller_spec.rb | 34 +++++++++++++++++-- 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/server/app/controllers/api/v1/models_controller.rb b/server/app/controllers/api/v1/models_controller.rb index 6b2063e6..421d9165 100644 --- a/server/app/controllers/api/v1/models_controller.rb +++ b/server/app/controllers/api/v1/models_controller.rb @@ -14,8 +14,9 @@ class ModelsController < ApplicationController after_action :event_logger def index + filter = params[:query_type] || "all" @models = current_workspace - .models.all.page(params[:page] || 1) + .models.send(filter).page(params[:page] || 1) authorize @models render json: @models, status: :ok end diff --git a/server/app/models/model.rb b/server/app/models/model.rb index 372effd3..f7db2832 100644 --- a/server/app/models/model.rb +++ b/server/app/models/model.rb @@ -30,6 +30,9 @@ class Model < ApplicationRecord has_many :syncs, dependent: :destroy + scope :data, -> { where(query_type: %i[raw_sql dbt soql table_selector]) } + scope :ai_ml, -> { where(query_type: :ai_ml) } + default_scope { order(updated_at: :desc) } def to_protocol Multiwoven::Integrations::Protocol::Model.new( diff --git a/server/spec/models/model_spec.rb b/server/spec/models/model_spec.rb index 10d28184..1d7d23a1 100644 --- a/server/spec/models/model_spec.rb +++ b/server/spec/models/model_spec.rb @@ -130,4 +130,29 @@ expect(Model.query_types).to eq({ "raw_sql" => 0, "dbt" => 1, "soql" => 2, "table_selector" => 3, "ai_ml" => 4 }) end end + + describe "scopes" do + let(:source) { create(:connector, connector_type: "source", connector_name: "Snowflake") } + let!(:raw_sql_model) { create(:model, query_type: :raw_sql, connector: source) } + let!(:dbt_model) { create(:model, query_type: :dbt, connector: source) } + let!(:soql_model) { create(:model, query_type: :soql, connector: source) } + let!(:table_selector_model) { create(:model, query_type: :table_selector, connector: source) } + let!(:ai_ml_model) { create(:model, query_type: :ai_ml, connector: source, configuration: { test: "value" }) } + + describe ".data" do + it "returns models with query_type in [raw_sql, dbt, soql, table_selector]" do + data_models = Model.data + expect(data_models).to include(raw_sql_model, dbt_model, soql_model, table_selector_model) + expect(data_models).not_to include(ai_ml_model) + end + end + + describe ".ai_ml" do + it "returns models with query_type equal to ai_ml" do + ai_ml_models = Model.ai_ml + expect(ai_ml_models).to include(ai_ml_model) + expect(ai_ml_models).not_to include(raw_sql_model, dbt_model, soql_model, table_selector_model) + end + end + end end diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb index 3301e72f..c05177a5 100644 --- a/server/spec/requests/api/v1/models_controller_spec.rb +++ b/server/spec/requests/api/v1/models_controller_spec.rb @@ -19,6 +19,13 @@ create(:model, connector:, workspace:, name: "model2", query: "SELECT * FROM locations") ] end + + let!(:raw_sql_model) { create(:model, query_type: :raw_sql, connector:, workspace:) } + let!(:dbt_model) { create(:model, query_type: :dbt, connector:, workspace:) } + let!(:soql_model) { create(:model, query_type: :soql, connector:, workspace:) } + let!(:ai_ml_model) do + create(:model, query_type: :ai_ml, connector:, configuration: { key: "value" }, workspace:) + end let(:viewer_role) { create(:role, :viewer) } let(:member_role) { create(:role, :member) } @@ -40,7 +47,7 @@ get "/api/v1/models", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(models.count) + expect(response_hash[:data].count).to eql(6) expect(response_hash.dig(:data, 0, :type)).to eq("models") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1") end @@ -50,7 +57,7 @@ get "/api/v1/models", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(models.count) + expect(response_hash[:data].count).to eql(6) expect(response_hash.dig(:data, 0, :type)).to eq("models") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1") end @@ -60,10 +67,31 @@ get "/api/v1/models", headers: auth_headers(user, workspace_id) expect(response).to have_http_status(:ok) response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash[:data].count).to eql(models.count) + expect(response_hash[:data].count).to eql(6) expect(response_hash.dig(:data, 0, :type)).to eq("models") expect(response_hash.dig(:links, :first)).to include("http://www.example.com/api/v1/models?page=1") end + + it "filters models based on the query_type parameter" do + get "/api/v1/models?query_type=data", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + expect(JSON.parse(response.body)["data"].map { |m| m["id"] }).not_to include(ai_ml_model.id) + end + + it "filters models based on a different query_type" do + get "/api/v1/models?query_type=ai_ml", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + expect(JSON.parse(response.body)["data"].map do |m| + m["id"] + end).not_to include(raw_sql_model.id, dbt_model.id, soql_model.id) + end + + it "returns all models" do + get "/api/v1/models", headers: auth_headers(user, workspace_id) + expect(response).to have_http_status(:ok) + model_ids = JSON.parse(response.body)["data"].map { |m| m["id"] } + expect(model_ids.count).to eql(6) + end end end From 9b4a53d39cb0e84441b9f0245e786a379ed4bfbe Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 14:32:05 +0530 Subject: [PATCH 12/31] chore(CE): add filtering scope to connectors (#345) Co-authored-by: Subin T P --- server/app/models/connector.rb | 43 ++++++++++++++++++ ...2_populate_category_field_in_connectors.rb | 20 +++++++++ server/db/data_schema.rb | 2 +- ...003_add_connector_category_to_connector.rb | 7 +++ server/db/schema.rb | 4 +- server/spec/models/connector_spec.rb | 44 +++++++++++++++++++ 6 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 server/db/data/20240827212112_populate_category_field_in_connectors.rb create mode 100644 server/db/migrate/20240827211003_add_connector_category_to_connector.rb diff --git a/server/app/models/connector.rb b/server/app/models/connector.rb index 9b86d5e4..c2ee3c36 100644 --- a/server/app/models/connector.rb +++ b/server/app/models/connector.rb @@ -36,6 +36,36 @@ class Connector < ApplicationRecord default_scope { order(updated_at: :desc) } + before_save :set_category + before_update :set_category, if: :will_save_change_to_connector_name? + + DEFAULT_CONNECTOR_CATEGORY = "data" + + # TODO: Move this to integrations gem + DATA_CATEGORIES = [ + "Data Warehouse", + "Retail", + "Data Lake", + "Database", + "Marketing Automation", + "CRM", + "Ad-Tech", + "Team Collaboration", + "Productivity Tools", + "Payments", + "File Storage", + "HTTP", + "Customer Support", + "data" + ].freeze + + AI_ML_CATEGORIES = [ + "AI Model" + ].freeze + + scope :ai_ml, -> { where(connector_category: AI_ML_CATEGORIES) } + scope :data, -> { where(connector_category: DATA_CATEGORIES) } + def connector_definition @connector_definition ||= connector_client.new.meta_data.with_indifferent_access end @@ -90,4 +120,17 @@ def connector_query_type def pull_catalog connector_client.new.discover(configuration).catalog.to_h.with_indifferent_access end + + def set_category + unless connector_category.present? && + connector_category == DEFAULT_CONNECTOR_CATEGORY && + !will_save_change_to_connector_category? + return + end + + category_name = connector_client.new.meta_data[:data][:category] + self.connector_category = category_name if category_name.present? + rescue StandardError => e + Rails.logger.error("Failed to set category for connector ##{id}: #{e.message}") + end end diff --git a/server/db/data/20240827212112_populate_category_field_in_connectors.rb b/server/db/data/20240827212112_populate_category_field_in_connectors.rb new file mode 100644 index 00000000..3e1a5f98 --- /dev/null +++ b/server/db/data/20240827212112_populate_category_field_in_connectors.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class PopulateCategoryFieldInConnectors < ActiveRecord::Migration[7.1] + def up + Connector.find_each do |connector| + category_name = connector.connector_client.new.meta_data[:data][:category] + connector.update!(connector_category: category_name) if category_name.present? + rescue StandardError => e + Rails.logger.error("Failed to update connector ##{connector.id}: #{e.message}") + end + end + + def down + Connector.find_each do |connector| + connector.update!(connector_category: "data") + rescue StandardError => e + Rails.logger.error("Failed to revert connector ##{connector.id} to 'data': #{e.message}") + end + end +end diff --git a/server/db/data_schema.rb b/server/db/data_schema.rb index 9c182f0d..f54a3d06 100644 --- a/server/db/data_schema.rb +++ b/server/db/data_schema.rb @@ -1,3 +1,3 @@ # frozen_string_literal: true -DataMigrate::Data.define(version: 20_240_813_084_446) +DataMigrate::Data.define(version: 20_240_827_212_112) diff --git a/server/db/migrate/20240827211003_add_connector_category_to_connector.rb b/server/db/migrate/20240827211003_add_connector_category_to_connector.rb new file mode 100644 index 00000000..a16ddd10 --- /dev/null +++ b/server/db/migrate/20240827211003_add_connector_category_to_connector.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AddConnectorCategoryToConnector < ActiveRecord::Migration[7.1] + def change + add_column :connectors, :connector_category, :string, null: false, default: "data" + end +end diff --git a/server/db/schema.rb b/server/db/schema.rb index e1967009..c6672c4f 100644 --- a/server/db/schema.rb +++ b/server/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_08_23_114746) do +ActiveRecord::Schema[7.1].define(version: 2024_08_27_211003) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -33,6 +33,7 @@ t.datetime "updated_at", null: false t.string "connector_name" t.string "description" + t.string "connector_category", default: "data", null: false end create_table "data_migrations", primary_key: "version", id: :string, force: :cascade do |t| @@ -132,6 +133,7 @@ t.string "cursor_field" t.string "current_cursor_field" t.string "cron_expression" + t.string "name" t.index ["discarded_at"], name: "index_syncs_on_discarded_at" end diff --git a/server/spec/models/connector_spec.rb b/server/spec/models/connector_spec.rb index 4c7ac76e..fd814aaa 100644 --- a/server/spec/models/connector_spec.rb +++ b/server/spec/models/connector_spec.rb @@ -164,4 +164,48 @@ end end end + + describe "#set_category" do + let(:workspace) { create(:workspace) } + let(:connector) do + create(:connector, + workspace:) + end + + context "populate category from connector meta data" do + it "sets the connector_category based on the meta_data" do + connector.run_callbacks(:save) { true } + expect(connector.connector_category).to eq("Marketing Automation") + end + end + + context "catagory is set by user" do + it "does not change the connector_category" do + connector.update!(connector_category: "user_input_category") + expect(connector.connector_category).to eq("user_input_category") + end + end + end + + describe ".ai_ml" do + let!(:ai_ml_connector) { create(:connector, connector_category: "AI Model") } + let!(:non_ai_ml_connector) { create(:connector, connector_category: "Data Warehouse") } + + it "returns connectors with connector_category in AI_ML_CATEGORIES" do + result = Connector.ai_ml + expect(result).to include(ai_ml_connector) + expect(result).not_to include(non_ai_ml_connector) + end + end + + describe ".data" do + let!(:data_connector) { create(:connector, connector_category: "Data Warehouse") } + let!(:non_data_connector) { create(:connector, connector_category: "AI Model") } + + it "returns connectors with connector_category in DATA_CATEGORIES" do + result = Connector.data + expect(result).to include(data_connector) + expect(result).not_to include(non_data_connector) + end + end end From 7e304c546c152548ce32a2d80f348836c347a95f Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:00:13 -0400 Subject: [PATCH 13/31] refactor(CE): created common connector lists component --- ui/src/components/DataTable/RowsNotFound.tsx | 13 +++ .../Activate/Syncs/SyncRuns/SyncRuns.tsx | 18 +--- .../ConnectorsListColumns.tsx | 42 +++++++++ .../DestinationsList/DestinationsList.tsx | 47 +++++++--- .../Sources/SourcesList/SourcesList.tsx | 93 ++++++------------- ui/src/views/Connectors/types.ts | 3 + 6 files changed, 124 insertions(+), 92 deletions(-) create mode 100644 ui/src/components/DataTable/RowsNotFound.tsx create mode 100644 ui/src/views/Connectors/ConnectorsListColumns/ConnectorsListColumns.tsx diff --git a/ui/src/components/DataTable/RowsNotFound.tsx b/ui/src/components/DataTable/RowsNotFound.tsx new file mode 100644 index 00000000..df269f56 --- /dev/null +++ b/ui/src/components/DataTable/RowsNotFound.tsx @@ -0,0 +1,13 @@ +import EmptyState from '@/assets/images/empty-state-illustration.svg'; +import { Box, Image, Text } from '@chakra-ui/react'; + +const RowsNotFound = () => ( + + empty-table + + No rows found + + +); + +export default RowsNotFound; diff --git a/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx b/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx index cd0aa12d..9029ec05 100644 --- a/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx +++ b/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx @@ -2,12 +2,12 @@ import { useQuery } from '@tanstack/react-query'; import { useNavigate, useParams, useSearchParams } from 'react-router-dom'; import { getSyncRunsBySyncId } from '@/services/syncs'; import { useMemo, useState, useEffect } from 'react'; -import { Box, Image, Text } from '@chakra-ui/react'; +import { Box } from '@chakra-ui/react'; import Loader from '@/components/Loader'; import Pagination from '@/components/Pagination'; import { SyncRunsColumns } from './SyncRunsColumns'; import DataTable from '@/components/DataTable'; -import SyncRunEmptyImage from '@/assets/images/empty-state-illustration.svg'; +import RowsNotFound from '@/components/DataTable/RowsNotFound'; const SyncRuns = () => { const { syncId } = useParams(); @@ -51,19 +51,7 @@ const SyncRuns = () => { ) : ( {data?.data?.length === 0 || !data?.data ? ( - - - - No rows found - - + ) : ( )} diff --git a/ui/src/views/Connectors/ConnectorsListColumns/ConnectorsListColumns.tsx b/ui/src/views/Connectors/ConnectorsListColumns/ConnectorsListColumns.tsx new file mode 100644 index 00000000..a39e2a1e --- /dev/null +++ b/ui/src/views/Connectors/ConnectorsListColumns/ConnectorsListColumns.tsx @@ -0,0 +1,42 @@ +import { ConnectorItem } from '../types'; +import { ColumnDef } from '@tanstack/react-table'; +import { Text } from '@chakra-ui/react'; +import EntityItem from '@/components/EntityItem'; +import StatusTag from '@/components/StatusTag'; +import dayjs from 'dayjs'; + +export const ConnectorsListColumns: ColumnDef[] = [ + { + accessorKey: 'attributes.name', + header: () => Name, + cell: (info) => ( + + {info.getValue() as string} + + ), + }, + { + accessorKey: 'attributes', + header: () => Type, + cell: (info) => ( + + ), + }, + { + accessorKey: 'attributes.updated_at', + header: () => Updated At, + cell: (info) => ( + + {dayjs(info.getValue() as number).format('DD/MM/YY')} + + ), + }, + { + accessorKey: 'attributes.status', + header: () => Status, + cell: () => , + }, +]; diff --git a/ui/src/views/Connectors/Destinations/DestinationsList/DestinationsList.tsx b/ui/src/views/Connectors/Destinations/DestinationsList/DestinationsList.tsx index 2b2442e7..f141296d 100644 --- a/ui/src/views/Connectors/Destinations/DestinationsList/DestinationsList.tsx +++ b/ui/src/views/Connectors/Destinations/DestinationsList/DestinationsList.tsx @@ -1,29 +1,53 @@ import { Box } from '@chakra-ui/react'; import { FiPlus } from 'react-icons/fi'; import TopBar from '@/components/TopBar'; -import { useNavigate } from 'react-router-dom'; import ContentContainer from '@/components/ContentContainer'; -import DestinationsTable from './DestinationsTable'; import { useQuery } from '@tanstack/react-query'; import { DESTINATIONS_LIST_QUERY_KEY } from '../../constant'; import { getUserConnectors } from '@/services/connectors'; import NoConnectors from '../../NoConnectors'; import Loader from '@/components/Loader'; +import { useStore } from '@/stores'; +import useCustomToast from '@/hooks/useCustomToast'; +import { CustomToastStatus } from '@/components/Toast/index'; +import titleCase from '@/utils/TitleCase'; +import DataTable from '@/components/DataTable'; +import { ConnectorsListColumns } from '@/views/Connectors/ConnectorsListColumns/ConnectorsListColumns'; +import { useNavigate } from 'react-router-dom'; const DestinationsList = (): JSX.Element | null => { + const showToast = useCustomToast(); + + const activeWorkspaceId = useStore((state) => state.workspaceId); + const navigate = useNavigate(); const { data, isLoading } = useQuery({ - queryKey: DESTINATIONS_LIST_QUERY_KEY, + queryKey: [...DESTINATIONS_LIST_QUERY_KEY, activeWorkspaceId], queryFn: () => getUserConnectors('destination'), refetchOnMount: true, refetchOnWindowFocus: false, + enabled: activeWorkspaceId > 0, }); if (isLoading && !data) return ; - if (data?.data.length === 0) return ; + if (data?.data?.length === 0 || !data) return ; + + if (data?.errors) { + data.errors?.forEach((error) => { + showToast({ + duration: 5000, + isClosable: true, + position: 'bottom-right', + colorScheme: 'red', + status: CustomToastStatus.Warning, + title: titleCase(error.detail), + }); + }); + return ; + } return ( @@ -36,17 +60,14 @@ const DestinationsList = (): JSX.Element | null => { ctaButtonVariant='solid' ctaButtonWidth='fit' ctaButtonHeight='40px' - isCtaVisible /> - {isLoading || !data ? ( - - ) : ( - navigate(`/setup/destinations/${row?.id}`)} - destinationData={data} - isLoading={isLoading} + + navigate(`/setup/destinations/${row?.original?.id}`)} /> - )} + ); diff --git a/ui/src/views/Connectors/Sources/SourcesList/SourcesList.tsx b/ui/src/views/Connectors/Sources/SourcesList/SourcesList.tsx index 07f1f512..6546bdfe 100644 --- a/ui/src/views/Connectors/Sources/SourcesList/SourcesList.tsx +++ b/ui/src/views/Connectors/Sources/SourcesList/SourcesList.tsx @@ -1,50 +1,21 @@ -import { useMemo } from 'react'; import { useQuery } from '@tanstack/react-query'; -import { Box, Text } from '@chakra-ui/react'; +import { Box } from '@chakra-ui/react'; import { FiPlus } from 'react-icons/fi'; import TopBar from '@/components/TopBar'; -import { useNavigate } from 'react-router-dom'; -import { SOURCES_LIST_QUERY_KEY, CONNECTOR_LIST_COLUMNS } from '@/views/Connectors/constant'; -import Table from '@/components/Table'; +import { SOURCES_LIST_QUERY_KEY } from '@/views/Connectors/constant'; import { getUserConnectors } from '@/services/connectors'; -import { ConnectorAttributes, ConnectorTableColumnFields } from '../../types'; -import moment from 'moment'; import ContentContainer from '@/components/ContentContainer'; -import EntityItem from '@/components/EntityItem'; import Loader from '@/components/Loader'; import NoConnectors from '@/views/Connectors/NoConnectors'; -import StatusTag from '@/components/StatusTag'; - -type TableItem = { - field: ConnectorTableColumnFields; - attributes: ConnectorAttributes; -}; - -const TableItem = ({ field, attributes }: TableItem): JSX.Element => { - switch (field) { - case 'icon': - return ; - - case 'updated_at': - return ( - - {moment(attributes?.updated_at).format('DD/MM/YY')} - - ); - - case 'status': - return ; - - default: - return ( - - {attributes?.[field]} - - ); - } -}; +import { CustomToastStatus } from '@/components/Toast/index'; +import titleCase from '@/utils/TitleCase'; +import { ConnectorsListColumns } from '@/views/Connectors/ConnectorsListColumns/ConnectorsListColumns'; +import DataTable from '@/components/DataTable'; +import { useNavigate } from 'react-router-dom'; +import useCustomToast from '@/hooks/useCustomToast'; const SourcesList = (): JSX.Element | null => { + const showToast = useCustomToast(); const navigate = useNavigate(); const { data, isLoading } = useQuery({ queryKey: SOURCES_LIST_QUERY_KEY, @@ -53,31 +24,23 @@ const SourcesList = (): JSX.Element | null => { refetchOnWindowFocus: false, }); - const connectors = data?.data; - - const tableData = useMemo(() => { - if (connectors && connectors?.length > 0) { - const rows = connectors.map(({ attributes, id }) => { - return CONNECTOR_LIST_COLUMNS.reduce( - (acc, { key }) => ({ - [key]: , - id, - ...acc, - }), - {}, - ); - }); - - return { - columns: CONNECTOR_LIST_COLUMNS, - data: rows, - }; - } - }, [data]); - if (isLoading) return ; - if (!isLoading && !tableData) return ; + if (data?.data?.length === 0 || !data) return ; + + if (data?.errors) { + data.errors?.forEach((error) => { + showToast({ + duration: 5000, + isClosable: true, + position: 'bottom-right', + colorScheme: 'red', + status: CustomToastStatus.Warning, + title: titleCase(error.detail), + }); + }); + return ; + } return ( @@ -90,9 +53,11 @@ const SourcesList = (): JSX.Element | null => { ctaButtonVariant='solid' isCtaVisible /> - {tableData ? ( - navigate(`/setup/sources/${row?.id}`)} /> - ) : null} + navigate(`/setup/sources/${row?.original?.id}`)} + /> ); diff --git a/ui/src/views/Connectors/types.ts b/ui/src/views/Connectors/types.ts index c0498c97..baf13571 100644 --- a/ui/src/views/Connectors/types.ts +++ b/ui/src/views/Connectors/types.ts @@ -1,3 +1,5 @@ +import { ErrorResponse } from '@/services/common'; + export type ConnectorTypes = 'source' | 'destination'; export type DatasourceType = { @@ -95,6 +97,7 @@ export type ConnectorInfoResponse = { export type ConnectorListResponse = { data: ConnectorItem[]; + errors?: ErrorResponse[]; }; export type ConnectorTableColumnFields = From 735eb43a75884fd0e8225fe8a15cd02489cf61c6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 09:53:29 +0530 Subject: [PATCH 14/31] fix(CE): request log redaction (#348) Co-authored-by: afthab vp --- .../app/middleware/multiwoven_server/request_response_logger.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app/middleware/multiwoven_server/request_response_logger.rb b/server/app/middleware/multiwoven_server/request_response_logger.rb index abf3d352..93b9b179 100644 --- a/server/app/middleware/multiwoven_server/request_response_logger.rb +++ b/server/app/middleware/multiwoven_server/request_response_logger.rb @@ -25,7 +25,7 @@ def log_request(env) Rails.logger.info({ request_method: request.request_method, request_url: request.url, - request_params: request.parameters, + request_params: request.filtered_parameters, request_headers: { "Workspace-Id": request.headers["Workspace-Id"] } }.to_s) end From 4d5d74041ee2adcbc79d1f0f2c32e61b98e29148 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 09:53:58 +0530 Subject: [PATCH 15/31] fix(CE): add more fields to log redaction (#349) Co-authored-by: datafloyd --- .../initializers/filter_parameter_logging.rb | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/server/config/initializers/filter_parameter_logging.rb b/server/config/initializers/filter_parameter_logging.rb index c2d89e28..240bab25 100644 --- a/server/config/initializers/filter_parameter_logging.rb +++ b/server/config/initializers/filter_parameter_logging.rb @@ -4,5 +4,27 @@ # Use this to limit dissemination of sensitive information. # See the ActiveSupport::ParameterFilter documentation for supported notations and behaviors. Rails.application.config.filter_parameters += [ - :passw, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn + :password, + :password_confirmation, + :secret, + :token, + :_key, + :crypt, + :salt, + :certificate, + :otp, + :ssn, + :credit_card_number, + :card_number, + :cvv, + :card_verification, + :expiration_date, + :authenticity_token, + :api_key, + :access_token, + :refresh_token, + :pin, + :current_password, + :new_password, + :ssn_last4, ] From b53ec8795d995fa021849a4a47d77d95e58ffc6b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 09:46:40 +0530 Subject: [PATCH 16/31] feat(CE): add aws sagemaker model source connector (#352) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- integrations/Gemfile | 4 + integrations/Gemfile.lock | 15 +- integrations/lib/multiwoven/integrations.rb | 3 + .../multiwoven/integrations/core/constants.rb | 3 - .../lib/multiwoven/integrations/rollout.rb | 3 +- .../source/aws_sagemaker_model/client.rb | 79 ++++++++++ .../aws_sagemaker_model/config/catalog.json | 6 + .../aws_sagemaker_model/config/meta.json | 15 ++ .../aws_sagemaker_model/config/spec.json | 50 ++++++ .../source/aws_sagemaker_model/icon.svg | 18 +++ integrations/multiwoven-integrations.gemspec | 2 + .../source/aws_sagemaker_model/client_spec.rb | 146 ++++++++++++++++++ 12 files changed, 339 insertions(+), 5 deletions(-) create mode 100644 integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/client.rb create mode 100644 integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/catalog.json create mode 100644 integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/meta.json create mode 100644 integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/spec.json create mode 100644 integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/icon.svg create mode 100644 integrations/spec/multiwoven/integrations/source/aws_sagemaker_model/client_spec.rb diff --git a/integrations/Gemfile b/integrations/Gemfile index 757d6502..970287a8 100644 --- a/integrations/Gemfile +++ b/integrations/Gemfile @@ -69,6 +69,10 @@ gem "aws-sdk-sts" gem "ruby-oci8", "~> 2.2.12" +gem "aws-sdk-sagemaker" + +gem "aws-sdk-sagemakerruntime" + group :development, :test do gem "simplecov", require: false gem "simplecov_json_formatter", require: false diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 0bcecf8b..5fc041b6 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,12 +7,14 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.9.2) + multiwoven-integrations (0.10.0) activesupport async-websocket aws-sdk-athena + aws-sdk-cloudwatchlogs aws-sdk-s3 aws-sdk-sts + aws-sigv4 csv dry-schema dry-struct @@ -67,6 +69,9 @@ GEM aws-sdk-athena (1.83.0) aws-sdk-core (~> 3, >= 3.193.0) aws-sigv4 (~> 1.1) + aws-sdk-cloudwatchlogs (1.82.0) + aws-sdk-core (~> 3, >= 3.193.0) + aws-sigv4 (~> 1.1) aws-sdk-core (3.196.1) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) @@ -79,6 +84,12 @@ GEM aws-sdk-core (~> 3, >= 3.194.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.8) + aws-sdk-sagemaker (1.229.0) + aws-sdk-core (~> 3, >= 3.188.0) + aws-sigv4 (~> 1.1) + aws-sdk-sagemakerruntime (1.63.0) + aws-sdk-core (~> 3, >= 3.193.0) + aws-sigv4 (~> 1.1) aws-sdk-sts (1.11.0) aws-sdk-core (~> 3, >= 3.110.0) aws-sigv4 (~> 1.1) @@ -338,6 +349,8 @@ DEPENDENCIES async-websocket (~> 0.8.0) aws-sdk-athena aws-sdk-s3 + aws-sdk-sagemaker + aws-sdk-sagemakerruntime aws-sdk-sts byebug csv diff --git a/integrations/lib/multiwoven/integrations.rb b/integrations/lib/multiwoven/integrations.rb index 02f2e262..bf214828 100644 --- a/integrations/lib/multiwoven/integrations.rb +++ b/integrations/lib/multiwoven/integrations.rb @@ -32,6 +32,8 @@ require "iterable-api-client" require "aws-sdk-sts" require "ruby-oci8" +require "aws-sdk-sagemaker" +require "aws-sdk-sagemakerruntime" # Service require_relative "integrations/config" @@ -63,6 +65,7 @@ require_relative "integrations/source/maria_db/client" require_relative "integrations/source/oracle_db/client" require_relative "integrations/source/databrics_model/client" +require_relative "integrations/source/aws_sagemaker_model/client" # Destination require_relative "integrations/destination/klaviyo/client" diff --git a/integrations/lib/multiwoven/integrations/core/constants.rb b/integrations/lib/multiwoven/integrations/core/constants.rb index 95e369dc..4d774a5f 100644 --- a/integrations/lib/multiwoven/integrations/core/constants.rb +++ b/integrations/lib/multiwoven/integrations/core/constants.rb @@ -50,9 +50,6 @@ module Constants DATABRICKS_HEALTH_URL = "https://%s/api/2.0/serving-endpoints/%s" DATABRICKS_SERVING_URL = "https://%s/serving-endpoints/%s/invocations" - AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"] - AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"] - # HTTP HTTP_GET = "GET" HTTP_POST = "POST" diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 1447eb3a..fbba1dfc 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.9.2" + VERSION = "0.10.0" ENABLED_SOURCES = %w[ Snowflake @@ -17,6 +17,7 @@ module Integrations MariaDB Oracle DatabricksModel + AwsSagemakerModel ].freeze ENABLED_DESTINATIONS = %w[ diff --git a/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/client.rb b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/client.rb new file mode 100644 index 00000000..f5ecce11 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/client.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Multiwoven::Integrations::Source + module AwsSagemakerModel + include Multiwoven::Integrations::Core + class Client < SourceConnector + def check_connection(connection_config) + connection_config = connection_config.with_indifferent_access + create_connection(connection_config) + response = @client.describe_endpoint(endpoint_name: connection_config[:endpoint_name]) + if response.endpoint_status == "InService" + success_status + else + failure_status + end + rescue StandardError => e + ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message + end + + def discover(_connection_config) + catalog_json = read_json(CATALOG_SPEC_PATH) + catalog = build_catalog(catalog_json) + catalog.to_multiwoven_message + rescue StandardError => e + handle_exception(e, { + context: "AWS:SAGEMAKER MODEL:DISCOVER:EXCEPTION", + type: "error" + }) + end + + def read(sync_config) + connection_config = sync_config.source.connection_specification + connection_config = connection_config.with_indifferent_access + payload = sync_config.model.query + create_connection(connection_config) + run_model(connection_config, payload) + rescue StandardError => e + handle_exception(e, { + context: "AWS:SAGEMAKER MODEL:READ:EXCEPTION", + type: "error", + sync_id: sync_config.sync_id, + sync_run_id: sync_config.sync_run_id + }) + end + + private + + def create_connection(connection_config) + @client = Aws::SageMaker::Client.new( + region: connection_config[:region], + access_key_id: connection_config[:access_key], + secret_access_key: connection_config[:secret_access_key] + ) + + @client_runtime = Aws::SageMakerRuntime::Client.new( + region: connection_config[:region], + access_key_id: connection_config[:access_key], + secret_access_key: connection_config[:secret_access_key] + ) + end + + def run_model(connection_config, payload) + response = @client_runtime.invoke_endpoint( + endpoint_name: connection_config[:endpoint_name], + content_type: "application/json", + body: payload + ) + process_response(response) + rescue StandardError => e + handle_exception(e, context: "AWS:SAGEMAKER MODEL:RUN_MODEL:EXCEPTION", type: "error") + end + + def process_response(response) + data = JSON.parse(response.body.read) + [RecordMessage.new(data: { response: data }, emitted_at: Time.now.to_i).to_multiwoven_message] + end + end + end +end diff --git a/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/catalog.json b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/catalog.json new file mode 100644 index 00000000..dacb788b --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/catalog.json @@ -0,0 +1,6 @@ +{ + "request_rate_limit": 600, + "request_rate_limit_unit": "minute", + "request_rate_concurrency": 10, + "streams": [] +} diff --git a/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/meta.json b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/meta.json new file mode 100644 index 00000000..9aeb49ba --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/meta.json @@ -0,0 +1,15 @@ +{ + "data": { + "name": "AwsSagemakerModel", + "title": "AWS Sagemaker Model", + "connector_type": "source", + "category": "AI Model", + "documentation_url": "https://docs.mutliwoven.com", + "github_issue_label": "source-aws-sagemaker-model", + "icon": "icon.svg", + "license": "MIT", + "release_stage": "alpha", + "support_level": "community", + "tags": ["language:ruby", "multiwoven"] + } +} diff --git a/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/spec.json b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/spec.json new file mode 100644 index 00000000..2253b4c1 --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/config/spec.json @@ -0,0 +1,50 @@ +{ + "documentation_url": "https://docs.multiwoven.com/integrations/sources/aws_sagemaker-model", + "stream_type": "user_defined", + "connector_query_type": "ai_ml", + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AWS Sagemaker Model", + "type": "object", + "required": ["access_key", "secret_access_key", "region", "endpoint_name", "request_format", "response_format"], + "properties": { + "access_key": { + "description": "The AWS Access Key ID to use for authentication", + "type": "string", + "title": "Personal Access Key", + "order": 0 + }, + "secret_access_key": { + "description": "The AWS Secret Access Key to use for authentication", + "type": "string", + "multiwoven_secret": true, + "title": "Secret Access Key", + "order": 1 + }, + "region": { + "description": "AWS region", + "type": "string", + "title": "Region", + "order": 2 + }, + "endpoint_name": { + "description": "Endpoint name for AWS Sagemaker", + "type": "string", + "title": "Endpoint name", + "order": 3 + }, + "request_format": { + "description": "Sample Request Format", + "type": "string", + "title": "Request Format", + "order": 4 + }, + "response_format": { + "description": "Sample Response Format", + "type": "string", + "title": "Response Format", + "order": 5 + } + } + } +} \ No newline at end of file diff --git a/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/icon.svg b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/icon.svg new file mode 100644 index 00000000..542ea19d --- /dev/null +++ b/integrations/lib/multiwoven/integrations/source/aws_sagemaker_model/icon.svg @@ -0,0 +1,18 @@ + + + + Icon-Architecture/64/Arch_AWS-SageMaker_64 + Created with Sketch. + + + + + + + + + + + + + \ No newline at end of file diff --git a/integrations/multiwoven-integrations.gemspec b/integrations/multiwoven-integrations.gemspec index 498a2d19..f312920d 100644 --- a/integrations/multiwoven-integrations.gemspec +++ b/integrations/multiwoven-integrations.gemspec @@ -36,8 +36,10 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "activesupport" spec.add_runtime_dependency "async-websocket" spec.add_runtime_dependency "aws-sdk-athena" + spec.add_runtime_dependency "aws-sdk-cloudwatchlogs" spec.add_runtime_dependency "aws-sdk-s3" spec.add_runtime_dependency "aws-sdk-sts" + spec.add_runtime_dependency "aws-sigv4" spec.add_runtime_dependency "csv" spec.add_runtime_dependency "dry-schema" spec.add_runtime_dependency "dry-struct" diff --git a/integrations/spec/multiwoven/integrations/source/aws_sagemaker_model/client_spec.rb b/integrations/spec/multiwoven/integrations/source/aws_sagemaker_model/client_spec.rb new file mode 100644 index 00000000..44b74eaf --- /dev/null +++ b/integrations/spec/multiwoven/integrations/source/aws_sagemaker_model/client_spec.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +RSpec.describe Multiwoven::Integrations::Source::AwsSagemakerModel::Client do + let(:client) { Multiwoven::Integrations::Source::AwsSagemakerModel::Client.new } + let(:payload) do + { + mode: "embedding", + text_inputs: [ + "Hi" + ] + } + end + let(:sync_config) do + { + "source": { + "name": "AWS Sagemaker Model", + "type": "source", + "connection_specification": { + "access_key": ENV["ATHENA_ACCESS"], + "secret_access_key": ENV["ATHENA_SECRET"], + "region": "us-east-2", + "endpoint_name": "Test-Endpoint", + "request_format": {}, + "response_format": {} + } + }, + "destination": { + "name": "Sample Destination Connector", + "type": "destination", + "connection_specification": { + "example_destination_key": "example_destination_value" + } + }, + "model": { + "name": "Anthena Account", + "query": payload.to_json, + "query_type": "raw_sql", + "primary_key": "id" + }, + "stream": { + "name": "example_stream", + "action": "create", + "json_schema": { "field1": "type1" }, + "supported_sync_modes": %w[full_refresh incremental], + "source_defined_cursor": true, + "default_cursor_field": ["field1"], + "source_defined_primary_key": [["field1"], ["field2"]], + "namespace": "exampleNamespace", + "url": "https://api.example.com/data", + "method": "GET" + }, + "sync_mode": "full_refresh", + "cursor_field": "timestamp", + "destination_sync_mode": "upsert", + "sync_id": "1" + } + end + + let(:runtime_client) { instance_double(Aws::SageMakerRuntime::Client) } + let(:sagemaker_client) { instance_double(Aws::SageMaker::Client) } + let(:response) { instance_double(Aws::SageMaker::Types::DescribeEndpointOutput, endpoint_status: "InService") } + let(:body_double) { instance_double(StringIO, read: "[1,2,3]") } + let(:endpoint_response) { instance_double(Aws::SageMakerRuntime::Types::InvokeEndpointOutput, body: body_double) } + + describe "#check_connection" do + before do + allow(Aws::SageMakerRuntime::Client).to receive(:new).and_return(runtime_client) + allow(Aws::SageMaker::Client).to receive(:new).and_return(sagemaker_client) + allow(sagemaker_client).to receive(:describe_endpoint).and_return(response) + end + context "when the connection is successful" do + it "returns a succeeded connection status" do + message = client.check_connection(sync_config[:source][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("succeeded") + expect(result.message).to be_nil + end + end + + context "when the connection fails" do + it "returns a failed connection status with an error message" do + allow_any_instance_of(Multiwoven::Integrations::Source::AwsSagemakerModel::Client).to receive(:create_connection).and_raise(StandardError, "Connection failed") + message = client.check_connection(sync_config[:source][:connection_specification]) + result = message.connection_status + expect(result.status).to eq("failed") + expect(result.message).to include("Connection failed") + end + end + end + + describe "#discover" do + it "successfully returns the catalog message" do + message = client.discover(nil) + catalog = message.catalog + expect(catalog).to be_a(Multiwoven::Integrations::Protocol::Catalog) + expect(catalog.request_rate_limit).to eql(600) + expect(catalog.request_rate_limit_unit).to eql("minute") + expect(catalog.request_rate_concurrency).to eql(10) + end + + it "handles exceptions during discovery" do + allow(client).to receive(:read_json).and_raise(StandardError.new("test error")) + expect(client).to receive(:handle_exception).with( + an_instance_of(StandardError), + hash_including(context: "AWS:SAGEMAKER MODEL:DISCOVER:EXCEPTION", type: "error") + ) + client.discover(nil) + end + end + + # read and #discover tests for AWS Athena + describe "#read" do + context "when the read is successful" do + it "reads records successfully" do + s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json) + allow(Aws::SageMakerRuntime::Client).to receive(:new).and_return(runtime_client) + allow(Aws::SageMaker::Client).to receive(:new).and_return(sagemaker_client) + allow(runtime_client).to receive(:invoke_endpoint).and_return(endpoint_response) + records = client.read(s_config) + expect(records).to be_an(Array) + expect(records.first.record).to be_a(Multiwoven::Integrations::Protocol::RecordMessage) + expect(records.first.record.data).to eq({ response: [1, 2, 3] }) + end + end + + context "when the read is failed" do + it "handles exceptions during reading" do + s_config = Multiwoven::Integrations::Protocol::SyncConfig.from_json(sync_config.to_json) + error_instance = StandardError.new("test error") + allow(Aws::SageMakerRuntime::Client).to receive(:new).and_return(runtime_client) + allow(Aws::SageMaker::Client).to receive(:new).and_return(sagemaker_client) + allow(client).to receive(:run_model).and_raise(error_instance) + expect(client).to receive(:handle_exception).with( + error_instance, + { + context: "AWS:SAGEMAKER MODEL:READ:EXCEPTION", + sync_id: "1", + sync_run_id: nil, + type: "error" + } + ) + client.read(s_config) + end + end + end +end From 274c1999d9890de7aeacfd5c13566010f47fb479 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 09:47:45 +0530 Subject: [PATCH 17/31] chore(CE): gem server update 0.10.0 (#351) Co-authored-by: TivonB-AI2 <124182151+TivonB-AI2@users.noreply.github.com> --- server/Gemfile | 2 +- server/Gemfile.lock | 37 +++++++++++++++++++++---------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/server/Gemfile b/server/Gemfile index c4e15073..77280cb9 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -13,7 +13,7 @@ gem "interactor", "~> 3.0" gem "ruby-odbc", git: "https://github.com/Multiwoven/ruby-odbc.git" -gem "multiwoven-integrations", "~> 0.9.1" +gem "multiwoven-integrations", "~> 0.10.0" gem "temporal-ruby", github: "coinbase/temporal-ruby" diff --git a/server/Gemfile.lock b/server/Gemfile.lock index 383d6078..afb6c195 100644 --- a/server/Gemfile.lock +++ b/server/Gemfile.lock @@ -108,21 +108,23 @@ GEM appsignal (3.7.5) rack ast (2.4.2) - async (2.15.3) + async (2.16.1) console (~> 1.26) fiber-annotation io-event (~> 1.6, >= 1.6.5) - async-http (0.70.0) + async-http (0.72.0) async (>= 2.10.2) async-pool (~> 0.7) io-endpoint (~> 0.11) io-stream (~> 0.4) - protocol-http (~> 0.28) - protocol-http1 (~> 0.19) + protocol-http (~> 0.29) + protocol-http1 (~> 0.20) protocol-http2 (~> 0.18) traces (>= 0.10) - async-pool (0.8.0) + async-pool (0.8.1) async (>= 1.25) + metrics + traces async-websocket (0.28.0) async-http (~> 0.54) protocol-http (>= 0.28.1) @@ -1773,7 +1775,7 @@ GEM fugit (1.10.1) et-orbi (~> 1, >= 1.2.7) raabro (~> 1.4) - git (2.1.1) + git (2.2.0) activesupport (>= 5.0) addressable (~> 2.8) process_executer (~> 1.1) @@ -1781,7 +1783,7 @@ GEM gli (2.21.5) globalid (1.2.1) activesupport (>= 6.1) - google-apis-bigquery_v2 (0.76.0) + google-apis-bigquery_v2 (0.77.0) google-apis-core (>= 0.15.0, < 2.a) google-apis-core (0.15.1) addressable (~> 2.5, >= 2.5.1) @@ -1804,7 +1806,7 @@ GEM google-cloud-core (1.7.1) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) - google-cloud-env (2.1.1) + google-cloud-env (2.2.0) faraday (>= 1.0, < 3.a) google-cloud-errors (1.4.0) google-protobuf (3.25.2-aarch64-linux) @@ -1847,7 +1849,7 @@ GEM inflection (1.0.0) interactor (3.1.2) io-console (0.7.1) - io-endpoint (0.13.0) + io-endpoint (0.13.1) io-event (1.6.5) io-stream (0.4.0) irb (1.11.1) @@ -1893,17 +1895,20 @@ GEM net-pop net-smtp marcel (1.0.2) + metrics (0.10.2) mini_mime (1.1.5) minitest (5.21.2) msgpack (1.7.2) multi_json (1.15.0) multipart-post (2.4.1) - multiwoven-integrations (0.9.1) + multiwoven-integrations (0.10.0) activesupport async-websocket aws-sdk-athena + aws-sdk-cloudwatchlogs aws-sdk-s3 aws-sdk-sts + aws-sigv4 csv dry-schema dry-struct @@ -1973,8 +1978,8 @@ GEM premailer (~> 1.7, >= 1.7.9) process_executer (1.1.0) protocol-hpack (1.5.0) - protocol-http (0.28.2) - protocol-http1 (0.19.1) + protocol-http (0.29.0) + protocol-http1 (0.20.0) protocol-http (~> 0.22) protocol-http2 (0.18.0) protocol-hpack (~> 1.4) @@ -2123,7 +2128,7 @@ GEM faraday-multipart gli hashie - sorbet-runtime (0.5.11532) + sorbet-runtime (0.5.11553) stringio (3.1.0) stripe (12.5.0) strong_migrations (1.8.0) @@ -2131,7 +2136,7 @@ GEM thor (1.3.0) timecop (0.9.8) timeout (0.4.1) - traces (0.11.1) + traces (0.13.1) trailblazer-option (0.1.2) typhoeus (1.4.1) ethon (>= 0.9.0) @@ -2143,7 +2148,7 @@ GEM unf_ext (0.0.9.1) unicode-display_width (2.5.0) uniform_notifier (1.16.0) - uri (0.13.0) + uri (0.13.1) uri_template (0.7.0) warden (1.2.9) rack (>= 2.0.9) @@ -2197,7 +2202,7 @@ DEPENDENCIES jwt kaminari liquid - multiwoven-integrations (~> 0.9.1) + multiwoven-integrations (~> 0.10.0) mysql2 newrelic_rpm parallel From 9e404651bd25129faa40f4d19dc8580b28cfadd1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 09:48:13 +0530 Subject: [PATCH 18/31] chore(CE): data app model migration (#350) Co-authored-by: afthab vp --- .../20240829110045_create_data_apps.rb | 13 ++++++++++ .../20240829131951_create_visual_component.rb | 15 ++++++++++++ server/db/schema.rb | 24 ++++++++++++++++++- 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 server/db/migrate/20240829110045_create_data_apps.rb create mode 100644 server/db/migrate/20240829131951_create_visual_component.rb diff --git a/server/db/migrate/20240829110045_create_data_apps.rb b/server/db/migrate/20240829110045_create_data_apps.rb new file mode 100644 index 00000000..7177b4e1 --- /dev/null +++ b/server/db/migrate/20240829110045_create_data_apps.rb @@ -0,0 +1,13 @@ +class CreateDataApps < ActiveRecord::Migration[7.1] + def change + create_table :data_apps do |t| + t.string :name, null: false + t.integer :status, null: false + t.integer :workspace_id, null: false + t.text :description + t.json :meta_data + + t.timestamps + end + end +end diff --git a/server/db/migrate/20240829131951_create_visual_component.rb b/server/db/migrate/20240829131951_create_visual_component.rb new file mode 100644 index 00000000..8abcc15b --- /dev/null +++ b/server/db/migrate/20240829131951_create_visual_component.rb @@ -0,0 +1,15 @@ +class CreateVisualComponent < ActiveRecord::Migration[7.1] + def change + create_table :visual_components do |t| + t.integer :component_type, null: false + t.string :name, null: false + t.integer :workspace_id, null: false + t.integer :data_app_id, null: false + t.integer :model_id, null: false + t.jsonb :properties + t.jsonb :feedback_config + + t.timestamps + end + end +end diff --git a/server/db/schema.rb b/server/db/schema.rb index c6672c4f..936958b6 100644 --- a/server/db/schema.rb +++ b/server/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_08_27_211003) do +ActiveRecord::Schema[7.1].define(version: 2024_08_29_131951) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -36,6 +36,16 @@ t.string "connector_category", default: "data", null: false end + create_table "data_apps", force: :cascade do |t| + t.string "name", null: false + t.integer "status", null: false + t.integer "workspace_id", null: false + t.text "description" + t.json "meta_data" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "data_migrations", primary_key: "version", id: :string, force: :cascade do |t| end @@ -175,6 +185,18 @@ t.index ["unlock_token"], name: "index_users_on_unlock_token", unique: true end + create_table "visual_components", force: :cascade do |t| + t.integer "component_type", null: false + t.string "name", null: false + t.integer "workspace_id", null: false + t.integer "data_app_id", null: false + t.integer "model_id", null: false + t.jsonb "properties" + t.jsonb "feedback_config" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "workspace_users", force: :cascade do |t| t.bigint "user_id", null: false t.bigint "workspace_id" From 529e56b54d8b25075725f4d46193b59518a49a60 Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Mon, 2 Sep 2024 21:41:59 -0400 Subject: [PATCH 19/31] Multiwoven release v0.23.0 (#354) Co-authored-by: github-actions --- release-notes.md | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/release-notes.md b/release-notes.md index 35a1828a..868d77d0 100644 --- a/release-notes.md +++ b/release-notes.md @@ -2,30 +2,34 @@ All notable changes to this project will be documented in this file. -## [0.22.0] - 2024-08-26 +## [0.23.0] - 2024-09-02 ### 🚀 Features -- *(CE)* Enable/disable sync (#321) -- *(CE)* Databricks AI Model connector (#325) -- *(CE)* Catalog creation via API (#329) -- *(CE)* Calalog update api (#333) -- *(CE)* Add AI/ML query type to model (#334) +- *(CE)* Filter connectors based on the category (#338) +- *(CE)* Filter connectors (#343) +- *(CE)* Filter model based on the query type (#347) +- *(CE)* Add aws sagemaker model source connector (#352) ### 🐛 Bug Fixes -- *(CE)* Correction in databricks model connection_spec (#331) +- *(CE)* Refresh catalog only when refresh is true (#336) +- *(CE)* Connection_spec mandatory changes (#342) +- *(CE)* Added tag label to auto-truncate text +- *(CE)* Request log redaction (#348) +- *(CE)* Add more fields to log redaction (#349) ### 🚜 Refactor -- *(CE)* Changed condition to render toast (#315) +- *(CE)* Created common connector lists component ### ⚙️ Miscellaneous Tasks -- *(CE)* Add request response log for Hubspot (#323) -- *(CE)* Add request response log for Stripe (#328) -- *(CE)* Add request response log for SalesforceCrm (#326) -- *(CE)* Upgrade version to 0.9.1 (#330) -- *(CE)* Refactor error message to agreed standard (#332) +- *(CE)* Validate catalog for query source (#340) +- *(CE)* Add catalog presence validation for models (#341) +- *(CE)* Refactor interactors (#344) +- *(CE)* Add filtering scope to connectors (#345) +- *(CE)* Gem server update 0.10.0 (#351) +- *(CE)* Data app model migration (#350) From 50d3e085495e33f2550ccea4786568e5495efb86 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 11:51:23 +0530 Subject: [PATCH 20/31] feat(CE): data app model changes (#353) Co-authored-by: afthab vp --- server/app/models/data_app.rb | 21 +++++++++++++++++++ server/app/models/model.rb | 1 + server/app/models/visual_component.rb | 14 +++++++++++++ server/app/models/workspace.rb | 1 + server/spec/models/data_app_spec.rb | 23 +++++++++++++++++++++ server/spec/models/model_spec.rb | 1 + server/spec/models/visual_component_spec.rb | 16 ++++++++++++++ server/spec/models/workspace_spec.rb | 1 + 8 files changed, 78 insertions(+) create mode 100644 server/app/models/data_app.rb create mode 100644 server/app/models/visual_component.rb create mode 100644 server/spec/models/data_app_spec.rb create mode 100644 server/spec/models/visual_component_spec.rb diff --git a/server/app/models/data_app.rb b/server/app/models/data_app.rb new file mode 100644 index 00000000..628561cc --- /dev/null +++ b/server/app/models/data_app.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class DataApp < ApplicationRecord + validates :workspace_id, presence: true + validates :status, presence: true + validates :name, presence: true + + enum :status, %i[inactive active draft] + + belongs_to :workspace + has_many :visual_components, dependent: :destroy + has_many :models, through: :visual_components + + after_initialize :set_default_status, if: :new_record? + + private + + def set_default_status + self.status ||= :draft + end +end diff --git a/server/app/models/model.rb b/server/app/models/model.rb index f7db2832..9a4f991c 100644 --- a/server/app/models/model.rb +++ b/server/app/models/model.rb @@ -29,6 +29,7 @@ class Model < ApplicationRecord belongs_to :connector has_many :syncs, dependent: :destroy + has_many :visual_components, dependent: :destroy scope :data, -> { where(query_type: %i[raw_sql dbt soql table_selector]) } scope :ai_ml, -> { where(query_type: :ai_ml) } diff --git a/server/app/models/visual_component.rb b/server/app/models/visual_component.rb new file mode 100644 index 00000000..e0737f2a --- /dev/null +++ b/server/app/models/visual_component.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class VisualComponent < ApplicationRecord + validates :workspace_id, presence: true + validates :component_type, presence: true + validates :model_id, presence: true + validates :data_app_id, presence: true + + enum :component_type, %i[pie bar data_table] + + belongs_to :workspace + belongs_to :data_app + belongs_to :model +end diff --git a/server/app/models/workspace.rb b/server/app/models/workspace.rb index b9bc40a2..97410b97 100644 --- a/server/app/models/workspace.rb +++ b/server/app/models/workspace.rb @@ -20,6 +20,7 @@ class Workspace < ApplicationRecord has_many :catalogs, dependent: :nullify has_many :syncs, dependent: :nullify has_many :sync_runs, dependent: :nullify + has_many :data_apps, dependent: :nullify belongs_to :organization STATUS_ACTIVE = "active" diff --git a/server/spec/models/data_app_spec.rb b/server/spec/models/data_app_spec.rb new file mode 100644 index 00000000..1877f18e --- /dev/null +++ b/server/spec/models/data_app_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe DataApp, type: :model do + it { should validate_presence_of(:workspace_id) } + it { should validate_presence_of(:status) } + it { should validate_presence_of(:name) } + + it { should define_enum_for(:status).with_values(inactive: 0, active: 1, draft: 2) } + + it { should belong_to(:workspace) } + it { should have_many(:visual_components).dependent(:destroy) } + it { should have_many(:models).through(:visual_components) } + + describe "#set_default_status" do + let(:data_app) { DataApp.new } + + it "sets default status" do + expect(data_app.status).to eq("draft") + end + end +end diff --git a/server/spec/models/model_spec.rb b/server/spec/models/model_spec.rb index 1d7d23a1..b28d7a76 100644 --- a/server/spec/models/model_spec.rb +++ b/server/spec/models/model_spec.rb @@ -22,6 +22,7 @@ it { should belong_to(:workspace) } it { should belong_to(:connector) } it { should have_many(:syncs).dependent(:destroy) } + it { should have_many(:visual_components).dependent(:destroy) } end describe "validations" do diff --git a/server/spec/models/visual_component_spec.rb b/server/spec/models/visual_component_spec.rb new file mode 100644 index 00000000..0ba909dd --- /dev/null +++ b/server/spec/models/visual_component_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe VisualComponent, type: :model do + it { should validate_presence_of(:workspace_id) } + it { should validate_presence_of(:component_type) } + it { should validate_presence_of(:model_id) } + it { should validate_presence_of(:data_app_id) } + + it { should define_enum_for(:component_type).with_values(pie: 0, bar: 1, data_table: 2) } + + it { should belong_to(:workspace) } + it { should belong_to(:data_app) } + it { should belong_to(:model) } +end diff --git a/server/spec/models/workspace_spec.rb b/server/spec/models/workspace_spec.rb index 96d1ada1..5e0e1ce3 100644 --- a/server/spec/models/workspace_spec.rb +++ b/server/spec/models/workspace_spec.rb @@ -29,6 +29,7 @@ it { should have_many(:models).dependent(:nullify) } it { should have_many(:catalogs).dependent(:nullify) } it { should have_many(:syncs).dependent(:nullify) } + it { should have_many(:data_apps).dependent(:nullify) } it { should belong_to(:organization) } end From e184972d3b7f6d69db916d1c6a927aedaa8814ee Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 01:21:16 +0530 Subject: [PATCH 21/31] refactor(CE): moved common elements of sign up and sign in to separate views * Resolve conflict in cherry-pick of 9f11d0258219f006606605557beece58912e373e and change the commit message * feat(CE): added common schema * feat(CE): added privacyPolicy and TOS * refactor(CE): resolved conflicts * feat(CE): updated AuthFooter * refactor(CE): updated types and auth views --------- Co-authored-by: Tushar Selvakumar <54372016+macintushar@users.noreply.github.com> Co-authored-by: Tushar Selvakumar --- ui/src/chakra-core.config.ts | 2 + ui/src/components/HiddenInput/HiddenInput.tsx | 16 +- ui/src/services/authentication.ts | 9 +- .../Authentication/AuthCard/AuthCard.tsx | 38 +++ ui/src/views/Authentication/AuthCard/index.ts | 1 + .../Authentication/AuthFooter/AuthFooter.tsx | 18 +- .../AuthViews/SignInAuthView.tsx | 118 +++++++ ui/src/views/Authentication/SignIn/SignIn.tsx | 233 +------------ ui/src/views/Authentication/SignUp/SignUp.tsx | 311 +++--------------- ui/src/views/Authentication/types.ts | 1 - 10 files changed, 243 insertions(+), 504 deletions(-) create mode 100644 ui/src/views/Authentication/AuthCard/AuthCard.tsx create mode 100644 ui/src/views/Authentication/AuthCard/index.ts create mode 100644 ui/src/views/Authentication/AuthViews/SignInAuthView.tsx diff --git a/ui/src/chakra-core.config.ts b/ui/src/chakra-core.config.ts index ebc49a4a..28ff9115 100644 --- a/ui/src/chakra-core.config.ts +++ b/ui/src/chakra-core.config.ts @@ -293,4 +293,6 @@ export const defaultExtension = { }, brandName: 'Multiwoven', logoUrl: '', + termsOfServiceUrl: 'https://multiwoven.com/terms-of-service', + privacyPolicyUrl: 'https://multiwoven.com/privacy-policy', }; diff --git a/ui/src/components/HiddenInput/HiddenInput.tsx b/ui/src/components/HiddenInput/HiddenInput.tsx index 2164b938..ea7c4fcd 100644 --- a/ui/src/components/HiddenInput/HiddenInput.tsx +++ b/ui/src/components/HiddenInput/HiddenInput.tsx @@ -17,6 +17,14 @@ const HiddenInput = (props: InputProps): JSX.Element => { return ( + { onClick={onClickReveal} /> - ); }; diff --git a/ui/src/services/authentication.ts b/ui/src/services/authentication.ts index 5b832b2f..657e9636 100644 --- a/ui/src/services/authentication.ts +++ b/ui/src/services/authentication.ts @@ -1,4 +1,4 @@ -import { multiwovenFetch } from './common'; +import { ErrorResponse, multiwovenFetch } from './common'; export type SignUpPayload = { email: string; @@ -28,6 +28,12 @@ export type SignInResponse = { errors?: SignInErrorResponse[]; }; +export type AuthErrorResponse = { + status: number; + title: string; + detail: string; +}; + export type AuthResponse = { type: string; id: string; @@ -44,6 +50,7 @@ export type AuthResponse = { export type ApiResponse = { data?: T; status: number; + errors?: ErrorResponse[]; }; export const signUp = async (payload: SignUpPayload) => diff --git a/ui/src/views/Authentication/AuthCard/AuthCard.tsx b/ui/src/views/Authentication/AuthCard/AuthCard.tsx new file mode 100644 index 00000000..2c7583cd --- /dev/null +++ b/ui/src/views/Authentication/AuthCard/AuthCard.tsx @@ -0,0 +1,38 @@ +import { Box, Container, Flex, Image, Stack } from '@chakra-ui/react'; +import { AuthCardProps } from '../types'; + +const AuthCard = ({ children, brandName, logoUrl }: AuthCardProps): JSX.Element => ( + <> + + + + + + + + + + {children} + + + + + +); + +export default AuthCard; diff --git a/ui/src/views/Authentication/AuthCard/index.ts b/ui/src/views/Authentication/AuthCard/index.ts new file mode 100644 index 00000000..37915af0 --- /dev/null +++ b/ui/src/views/Authentication/AuthCard/index.ts @@ -0,0 +1 @@ +export { default } from './AuthCard'; diff --git a/ui/src/views/Authentication/AuthFooter/AuthFooter.tsx b/ui/src/views/Authentication/AuthFooter/AuthFooter.tsx index 3f6e6e2d..52cbc369 100644 --- a/ui/src/views/Authentication/AuthFooter/AuthFooter.tsx +++ b/ui/src/views/Authentication/AuthFooter/AuthFooter.tsx @@ -1,7 +1,17 @@ import { Box, HStack, Text } from '@chakra-ui/layout'; import { Link } from 'react-router-dom'; -const AuthFooter = (): JSX.Element => { +type AuthFooterProps = { + brandName: string; + privacyPolicyUrl: string; + termsOfServiceUrl: string; +}; + +const AuthFooter = ({ + brandName, + privacyPolicyUrl, + termsOfServiceUrl, +}: AuthFooterProps): JSX.Element => { return ( { > - © Multiwoven Inc. All rights reserved. + {`© ${brandName} Inc. All rights reserved.`} - + Terms of use @@ -24,7 +34,7 @@ const AuthFooter = (): JSX.Element => { • - + Privacy Policy diff --git a/ui/src/views/Authentication/AuthViews/SignInAuthView.tsx b/ui/src/views/Authentication/AuthViews/SignInAuthView.tsx new file mode 100644 index 00000000..2b4544ab --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/SignInAuthView.tsx @@ -0,0 +1,118 @@ +import AuthCard from '../AuthCard'; +import { Form, Formik } from 'formik'; +import { Button, Checkbox, HStack, Heading, Stack, Text } from '@chakra-ui/react'; +import { FormField, PasswordField } from '@/components/Fields'; +import { Link } from 'react-router-dom'; +import { SignInAuthViewProps } from '../types'; +import { SignInSchema } from '@/constants/schemas'; +import { useEffect } from 'react'; +import { useStore } from '@/stores'; +import Cookies from 'js-cookie'; + +export const SignInAuthView = ({ + brandName, + logoUrl, + handleSubmit, + submitting, +}: SignInAuthViewProps) => { + // clears the state when the sign in auth loads + useEffect(() => { + Cookies?.remove('authToken'); + useStore.getState().clearState(); + }, []); + + return ( + <> + + handleSubmit(values)} + validationSchema={SignInSchema} + > + {({ getFieldProps, touched, errors }) => ( +
+ + + {"Let's activate your data"} + + + {`Sign In to your ${brandName} account`} + + + + + + + + + + Stay signed in + + + + + Forgot Password? + + + + + + + + + + {"Don't have an account?"}{' '} + + + + Sign Up + + + + + + )} + + + + ); +}; diff --git a/ui/src/views/Authentication/SignIn/SignIn.tsx b/ui/src/views/Authentication/SignIn/SignIn.tsx index 5e413c1b..82ca37f0 100644 --- a/ui/src/views/Authentication/SignIn/SignIn.tsx +++ b/ui/src/views/Authentication/SignIn/SignIn.tsx @@ -1,106 +1,14 @@ import { useState } from 'react'; -import { Formik, Form, ErrorMessage, FormikTouched, FormikErrors, FieldInputProps } from 'formik'; -import * as Yup from 'yup'; -import { Link, useNavigate } from 'react-router-dom'; -import { - Box, - Button, - FormControl, - Input, - Heading, - Text, - Container, - Stack, - Flex, - HStack, - Image, - Checkbox, -} from '@chakra-ui/react'; -import MultiwovenIcon from '@/assets/images/icon-white.svg'; -import { SignInErrorResponse, SignInPayload, signIn } from '@/services/authentication'; +import { useNavigate } from 'react-router-dom'; +import { AuthErrorResponse, SignInPayload, signIn } from '@/services/authentication'; import Cookies from 'js-cookie'; import titleCase from '@/utils/TitleCase'; import AuthFooter from '../AuthFooter'; -import HiddenInput from '@/components/HiddenInput'; import { CustomToastStatus } from '@/components/Toast/index'; import useCustomToast from '@/hooks/useCustomToast'; import mwTheme from '@/chakra.config'; import { useMutation } from '@tanstack/react-query'; - -const SignInSchema = Yup.object().shape({ - email: Yup.string().email('Please enter a valid email address').required('Email is required'), - password: Yup.string() - .min(8, 'Password must be at least 8 characters') - .required('Password is required'), -}); - -interface SignInFormProps { - name: string; - type: string; - placeholder?: string; - getFieldProps: ( - nameOrOptions: - | string - | { - name: string; - value?: any; - onChange?: (e: any) => void; - onBlur?: (e: any) => void; - }, - ) => FieldInputProps; - touched: FormikTouched; - errors: FormikErrors; -} - -const FormField = ({ - name, - type, - getFieldProps, - touched, - errors, - placeholder, -}: SignInFormProps) => ( - - - - - - -); - -const PasswordField = ({ - name, - type, - getFieldProps, - touched, - errors, - placeholder, -}: SignInFormProps) => ( - - - - - - -); +import { SignInAuthView } from '../AuthViews/SignInAuthView'; const SignIn = (): JSX.Element => { const [submitting, setSubmitting] = useState(false); @@ -130,7 +38,7 @@ const SignIn = (): JSX.Element => { }); navigate('/', { replace: true }); } else { - result.data?.errors?.forEach((error: SignInErrorResponse) => { + result?.errors?.forEach((error: AuthErrorResponse) => { showToast({ duration: 5000, isClosable: true, @@ -154,130 +62,21 @@ const SignIn = (): JSX.Element => { } }; - const { logoUrl, brandName } = mwTheme; + const { logoUrl, brandName, privacyPolicyUrl, termsOfServiceUrl } = mwTheme; return ( <> - - handleSubmit(values)} - validationSchema={SignInSchema} - > - {({ getFieldProps, touched, errors }) => ( -
- - - - - - - - - - - {"Let's activate your data"} - - - {`Sign In to your ${brandName} account`} - - - - - - - ={' '} - - - - Stay signed in - - - {/* - Forgot Password? - */} - - - - - - - - {"Don't have an account?"}{' '} - - - - Sign Up - - - - - - - - - )} - - - + + ); }; diff --git a/ui/src/views/Authentication/SignUp/SignUp.tsx b/ui/src/views/Authentication/SignUp/SignUp.tsx index c34ebe36..13fc6956 100644 --- a/ui/src/views/Authentication/SignUp/SignUp.tsx +++ b/ui/src/views/Authentication/SignUp/SignUp.tsx @@ -1,125 +1,27 @@ -import { useState } from 'react'; -import { Formik, Form, ErrorMessage, FormikTouched, FormikErrors, FieldInputProps } from 'formik'; -import * as Yup from 'yup'; -import { Link, useNavigate } from 'react-router-dom'; -import { - Box, - Button, - FormControl, - Input, - Heading, - Text, - Container, - Stack, - Flex, - HStack, - Image, -} from '@chakra-ui/react'; -import MultiwovenIcon from '@/assets/images/icon-white.svg'; -import { SignUpPayload, signUp } from '@/services/authentication'; -import Cookies from 'js-cookie'; -import titleCase from '@/utils/TitleCase'; +import { useNavigate } from 'react-router-dom'; +import { Stack, Heading } from '@chakra-ui/react'; +import mwTheme from '@/chakra.config'; import AuthFooter from '../AuthFooter'; -import HiddenInput from '@/components/HiddenInput'; -import { CustomToastStatus } from '@/components/Toast/index'; +import { SignUpAuthView } from '@/views/Authentication/AuthViews/SignUpAuthView'; +import AuthCard from '../AuthCard'; +import { CustomToastStatus } from '@/components/Toast'; +import { SignUpPayload, signUp } from '@/services/authentication'; +import { useState } from 'react'; import useCustomToast from '@/hooks/useCustomToast'; -import mwTheme from '@/chakra.config'; import { useMutation } from '@tanstack/react-query'; - -const SignUpSchema = Yup.object().shape({ - company_name: Yup.string().required('Company name is required'), - name: Yup.string().required('Name is required'), - email: Yup.string().email('Invalid email address').required('Email is required'), - password: Yup.string() - .min(8, 'Password must be at least 8 characters') - .required('Password is required'), - password_confirmation: Yup.string() - .oneOf([Yup.ref('password'), ''], 'Passwords must match') - .required('Confirm Password is required'), -}); - -interface SignUpFormProps { - name: string; - type: string; - placeholder?: string; - getFieldProps: ( - nameOrOptions: - | string - | { - name: string; - value?: any; - onChange?: (e: any) => void; - onBlur?: (e: any) => void; - }, - ) => FieldInputProps; - touched: FormikTouched; - errors: FormikErrors; -} - -const FormField = ({ - name, - type, - getFieldProps, - touched, - errors, - placeholder, -}: SignUpFormProps) => ( - - - - - - -); - -const PasswordField = ({ - name, - type, - getFieldProps, - touched, - errors, - placeholder, -}: SignUpFormProps) => ( - - - - - - -); - -type SignUpErrors = { - source: { - [key: string]: string; - }; -}; +import { useAPIErrorsToast, useErrorToast } from '@/hooks/useErrorToast'; +// import isValidEmailDomain from '@/utils/isValidEmailDomain'; const SignUp = (): JSX.Element => { const [submitting, setSubmitting] = useState(false); const navigate = useNavigate(); const showToast = useCustomToast(); + const apiErrorToast = useAPIErrorsToast(); + const errorToast = useErrorToast(); const { mutateAsync } = useMutation({ mutationFn: (values: SignUpPayload) => signUp(values), - mutationKey: ['signIn'], + mutationKey: ['signUp'], }); const handleSubmit = async (values: any) => { @@ -128,12 +30,6 @@ const SignUp = (): JSX.Element => { const result = await mutateAsync(values); if (result.data?.attributes) { - const token = result.data.attributes.token; - Cookies.set('authToken', token, { - secure: true, - sameSite: 'Lax', - }); - showToast({ title: 'Account created.', status: CustomToastStatus.Success, @@ -142,179 +38,48 @@ const SignUp = (): JSX.Element => { position: 'bottom-right', }); - navigate('/'); + navigate(`/sign-up/success?email=${values.email}`); } else { - result.data?.errors?.map((error: SignUpErrors) => { - Object.keys(error.source).map((error_key) => { - showToast({ - title: titleCase(error_key) + ' ' + error.source[error_key], - status: CustomToastStatus.Warning, - duration: 5000, - isClosable: true, - position: 'bottom-right', - colorScheme: 'red', - }); - }); - }); + apiErrorToast(result.errors || []); } } catch (error) { - showToast({ - title: 'An error occured. Please try again later.', - status: CustomToastStatus.Error, - duration: 5000, - isClosable: true, - position: 'bottom-right', - colorScheme: 'red', - }); + errorToast('An error occured. Please try again later.', true, null, true); } finally { setSubmitting(false); } }; - const { logoUrl, brandName } = mwTheme; + const { logoUrl, brandName, privacyPolicyUrl, termsOfServiceUrl } = mwTheme; return ( <> - - + + + Get started with {brandName} + + + handleSubmit(values)} - validationSchema={SignUpSchema} - > - {({ getFieldProps, touched, errors }) => ( -
- - - - - - - - - - - {`Get started with ${brandName}`} - - - Sign up and create your account - - - - - - - - - - - - By creating an account, I agree to the{' '} - - - - Terms - - - - and - - - - Privacy Policy - - - - - - - - - Do you have an account?{' '} - - - - Sign In - - - - - - - - - - )} - - - + /> + + ); }; diff --git a/ui/src/views/Authentication/types.ts b/ui/src/views/Authentication/types.ts index 3d2185d9..f0168859 100644 --- a/ui/src/views/Authentication/types.ts +++ b/ui/src/views/Authentication/types.ts @@ -1,5 +1,4 @@ import { ReactNode } from 'react'; - export type AuthCardProps = { children: ReactNode; brandName: string; From 5a2b57ca48f0a4faa7217d83590bf996bd35743d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:15:23 +0530 Subject: [PATCH 22/31] chore(CE): add data-app permissions to roles (#357) Co-authored-by: afthab vp --- ...055705_add_data_app_permission_to_roles.rb | 60 +++++++++++++++++++ server/db/data_schema.rb | 2 +- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 server/db/data/20240903055705_add_data_app_permission_to_roles.rb diff --git a/server/db/data/20240903055705_add_data_app_permission_to_roles.rb b/server/db/data/20240903055705_add_data_app_permission_to_roles.rb new file mode 100644 index 00000000..16bf2961 --- /dev/null +++ b/server/db/data/20240903055705_add_data_app_permission_to_roles.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +class AddDataAppPermissionToRoles < ActiveRecord::Migration[7.1] + def change # rubocop:disable Metrics/MethodLength + admin_role = Role.find_by(role_name: "Admin") + member_role = Role.find_by(role_name: "Member") + viewer_role = Role.find_by(role_name: "Viewer") + + admin_role&.update!( + policies: { + permissions: { + connector_definition: { create: true, read: true, update: true, delete: true }, + connector: { create: true, read: true, update: true, delete: true }, + model: { create: true, read: true, update: true, delete: true }, + report: { create: true, read: true, update: true, delete: true }, + sync_record: { create: true, read: true, update: true, delete: true }, + sync_run: { create: true, read: true, update: true, delete: true }, + sync: { create: true, read: true, update: true, delete: true }, + user: { create: true, read: true, update: true, delete: true }, + workspace: { create: true, read: true, update: true, delete: true }, + data_app: { create: true, read: true, update: true, delete: true } + } + } + ) + + member_role&.update!( + policies: { + permissions: { + connector_definition: { create: true, read: true, update: true, delete: true }, + connector: { create: true, read: true, update: true, delete: true }, + model: { create: true, read: true, update: true, delete: true }, + report: { create: true, read: true, update: true, delete: true }, + sync_record: { create: true, read: true, update: true, delete: true }, + sync_run: { create: true, read: true, update: true, delete: true }, + sync: { create: true, read: true, update: true, delete: true }, + user: { create: false, read: true, update: false, delete: false }, + workspace: { create: false, read: true, update: false, delete: false }, + data_app: { create: true, read: true, update: true, delete: true } + } + } + ) + + viewer_role&.update!( + policies: { + permissions: { + connector_definition: { create: false, read: true, update: false, delete: false }, + connector: { create: false, read: true, update: false, delete: false }, + model: { create: false, read: true, update: false, delete: false }, + report: { create: false, read: true, update: false, delete: false }, + sync_record: { create: false, read: true, update: false, delete: false }, + sync_run: { create: false, read: true, update: false, delete: false }, + sync: { create: false, read: true, update: false, delete: false }, + user: { create: false, read: true, update: false, delete: false }, + workspace: { create: false, read: true, update: false, delete: false }, + data_app: { create: false, read: true, update: false, delete: false } + } + } + ) + end +end diff --git a/server/db/data_schema.rb b/server/db/data_schema.rb index f54a3d06..ba8761bd 100644 --- a/server/db/data_schema.rb +++ b/server/db/data_schema.rb @@ -1,3 +1,3 @@ # frozen_string_literal: true -DataMigrate::Data.define(version: 20_240_827_212_112) +DataMigrate::Data.define(version: 20_240_903_055_705) From d11a7801fe61009cf6a02285bfc434276dbeb0cc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:16:38 +0530 Subject: [PATCH 23/31] fix(CE): validate_catalog disabled query_source (#356) Co-authored-by: afthab vp --- .../api/v1/connectors_controller.rb | 3 ++- .../api/v1/connectors_controller_spec.rb | 26 +++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/server/app/controllers/api/v1/connectors_controller.rb b/server/app/controllers/api/v1/connectors_controller.rb index 77bbcd08..821a99f7 100644 --- a/server/app/controllers/api/v1/connectors_controller.rb +++ b/server/app/controllers/api/v1/connectors_controller.rb @@ -8,7 +8,8 @@ class ConnectorsController < ApplicationController before_action :set_connector, only: %i[show update destroy discover query_source] # TODO: Enable this once we have query validation implemented for all the connectors # before_action :validate_query, only: %i[query_source] - before_action :validate_catalog, only: %i[query_source] + # TODO: Enable this for ai_ml sources + # before_action :validate_catalog, only: %i[query_source] after_action :event_logger def index diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb index e5f194b1..02f9da27 100644 --- a/server/spec/requests/api/v1/connectors_controller_spec.rb +++ b/server/spec/requests/api/v1/connectors_controller_spec.rb @@ -481,19 +481,19 @@ expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) end - it "returns an error message for missing catalog" do - catalog = connector.catalog - catalog.connector_id = connectors.second.id - catalog.save - - allow(Connectors::QuerySource).to receive(:call).and_return(double(:context, success?: true, - records: [record1, record2])) - post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: - { "Content-Type": "application/json" }.merge(auth_headers(user, workspace.id)) - expect(response).to have_http_status(:unprocessable_entity) - response_hash = JSON.parse(response.body).with_indifferent_access - expect(response_hash.dig(:errors, 0, :detail)).to eq("Catalog is not present for the connector") - end + # it "returns an error message for missing catalog" do + # catalog = connector.catalog + # catalog.connector_id = connectors.second.id + # catalog.save + + # allow(Connectors::QuerySource).to receive(:call).and_return(double(:context, success?: true, + # records: [record1, record2])) + # post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: + # { "Content-Type": "application/json" }.merge(auth_headers(user, workspace.id)) + # expect(response).to have_http_status(:unprocessable_entity) + # response_hash = JSON.parse(response.body).with_indifferent_access + # expect(response_hash.dig(:errors, 0, :detail)).to eq("Catalog is not present for the connector") + # end it "returns success status for a valid query for viewer role" do workspace.workspace_users.first.update(role: viewer_role) From fb86473751b382814ce55592c8ef126c1dba2761 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 12:50:56 +0400 Subject: [PATCH 24/31] fix(CE): enable catalog validation only for ai models (#355) * Resolve conflict in cherry-pick of 6de5e956cb5cd7d54544cfb096802d8e00e15b5f and change the commit message * fix(CE): resolve conflicts * fix(CE): remove duplicate * fix(CE): fix spec --------- Co-authored-by: datafloyd Co-authored-by: afthab vp --- .../api/v1/connectors_controller.rb | 3 +- server/app/models/connector.rb | 4 ++ server/spec/models/connector_spec.rb | 5 ++ .../api/v1/connectors_controller_spec.rb | 46 +++++++++++++------ 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/server/app/controllers/api/v1/connectors_controller.rb b/server/app/controllers/api/v1/connectors_controller.rb index 821a99f7..5d8e9e67 100644 --- a/server/app/controllers/api/v1/connectors_controller.rb +++ b/server/app/controllers/api/v1/connectors_controller.rb @@ -9,7 +9,7 @@ class ConnectorsController < ApplicationController # TODO: Enable this once we have query validation implemented for all the connectors # before_action :validate_query, only: %i[query_source] # TODO: Enable this for ai_ml sources - # before_action :validate_catalog, only: %i[query_source] + before_action :validate_catalog, only: %i[query_source] after_action :event_logger def index @@ -126,6 +126,7 @@ def set_connector end def validate_catalog + return unless @connector.ai_model? return if @connector.catalog.present? render_error( diff --git a/server/app/models/connector.rb b/server/app/models/connector.rb index c2ee3c36..d48bc955 100644 --- a/server/app/models/connector.rb +++ b/server/app/models/connector.rb @@ -133,4 +133,8 @@ def set_category rescue StandardError => e Rails.logger.error("Failed to set category for connector ##{id}: #{e.message}") end + + def ai_model? + connector_category == "AI Model" + end end diff --git a/server/spec/models/connector_spec.rb b/server/spec/models/connector_spec.rb index fd814aaa..8a1c1981 100644 --- a/server/spec/models/connector_spec.rb +++ b/server/spec/models/connector_spec.rb @@ -196,6 +196,11 @@ expect(result).to include(ai_ml_connector) expect(result).not_to include(non_ai_ml_connector) end + + it "check whether connector is ai model or not" do + expect(ai_ml_connector.ai_model?).to eq(true) + expect(non_ai_ml_connector.ai_model?).to eq(false) + end end describe ".data" do diff --git a/server/spec/requests/api/v1/connectors_controller_spec.rb b/server/spec/requests/api/v1/connectors_controller_spec.rb index 02f9da27..693988f0 100644 --- a/server/spec/requests/api/v1/connectors_controller_spec.rb +++ b/server/spec/requests/api/v1/connectors_controller_spec.rb @@ -481,19 +481,39 @@ expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) end - # it "returns an error message for missing catalog" do - # catalog = connector.catalog - # catalog.connector_id = connectors.second.id - # catalog.save - - # allow(Connectors::QuerySource).to receive(:call).and_return(double(:context, success?: true, - # records: [record1, record2])) - # post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: - # { "Content-Type": "application/json" }.merge(auth_headers(user, workspace.id)) - # expect(response).to have_http_status(:unprocessable_entity) - # response_hash = JSON.parse(response.body).with_indifferent_access - # expect(response_hash.dig(:errors, 0, :detail)).to eq("Catalog is not present for the connector") - # end + it "returns an error message for missing catalog for ai connectors" do + catalog = connector.catalog + catalog.connector_id = connectors.second.id + catalog.save + # rubocop:disable Rails/SkipsModelValidations + connector.update_column(:connector_category, "AI Model") + # rubocop:enable Rails/SkipsModelValidations + + allow(Connectors::QuerySource).to receive(:call).and_return(double(:context, success?: true, + records: [record1, record2])) + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: + { "Content-Type": "application/json" }.merge(auth_headers(user, workspace.id)) + expect(response).to have_http_status(:unprocessable_entity) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash.dig(:errors, 0, :detail)).to eq("Catalog is not present for the connector") + end + + it "should not return error message for missing catalog for data connectors" do + catalog = connector.catalog + catalog.connector_id = connectors.second.id + # rubocop:disable Rails/SkipsModelValidations + connector.update_column(:connector_category, "Data Warehouse") + # rubocop:enable Rails/SkipsModelValidations + catalog.save + + allow(Connectors::QuerySource).to receive(:call).and_return(double(:context, success?: true, + records: [record1, record2])) + post "/api/v1/connectors/#{connector.id}/query_source", params: request_body.to_json, headers: + { "Content-Type": "application/json" }.merge(auth_headers(user, workspace.id)) + expect(response).to have_http_status(:ok) + response_hash = JSON.parse(response.body).with_indifferent_access + expect(response_hash[:data]).to eq([record1.record.data, record2.record.data]) + end it "returns success status for a valid query for viewer role" do workspace.workspace_users.first.update(role: viewer_role) From b20ab3865ef8d2ef2594723ac22ad66a74e067f6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 12:45:32 +0530 Subject: [PATCH 25/31] fix(CE): disable catalog validation for data models (#358) Co-authored-by: datafloyd --- .../controllers/api/v1/models_controller.rb | 1 + .../requests/api/v1/models_controller_spec.rb | 27 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/server/app/controllers/api/v1/models_controller.rb b/server/app/controllers/api/v1/models_controller.rb index 421d9165..5c81e006 100644 --- a/server/app/controllers/api/v1/models_controller.rb +++ b/server/app/controllers/api/v1/models_controller.rb @@ -81,6 +81,7 @@ def set_model end def validate_catalog + return unless connector.ai_model? return if connector.catalog.present? render_error( diff --git a/server/spec/requests/api/v1/models_controller_spec.rb b/server/spec/requests/api/v1/models_controller_spec.rb index c05177a5..229d7144 100644 --- a/server/spec/requests/api/v1/models_controller_spec.rb +++ b/server/spec/requests/api/v1/models_controller_spec.rb @@ -189,11 +189,22 @@ workspace.workspace_users.first.update(role: member_role) # set connector without catalog request_body[:model][:connector_id] = connector_without_catalog.id + # rubocop:disable Rails/SkipsModelValidations + connector_without_catalog.update_column(:connector_category, "AI Model") + # rubocop:enable Rails/SkipsModelValidations post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" } .merge(auth_headers(user, workspace_id)) expect(response).to have_http_status(:unprocessable_entity) end + it "shouldn't fail model creation for connector without catalog for data sources" do + workspace.workspace_users.first.update(role: member_role) + request_body[:model][:connector_id] = connector_without_catalog.id + post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:created) + end + it " creates a new model and returns success" do workspace.workspace_users.first.update(role: member_role) post "/api/v1/models", params: request_body.to_json, headers: { "Content-Type": "application/json" } @@ -297,11 +308,14 @@ expect(response_hash.dig(:data, :attributes, :primary_key)).to eq(request_body.dig(:model, :primary_key)) end - it "fails model update for connector without catalog" do + it "fails model update for connector without catalog for ai model" do workspace.workspace_users.first.update(role: member_role) model = models.second model.connector_id = connector_without_catalog.id model.save! + # rubocop:disable Rails/SkipsModelValidations + connector_without_catalog.update_column(:connector_category, "AI Model") + # rubocop:enable Rails/SkipsModelValidations put "/api/v1/models/#{models.second.id}", params: request_body.to_json, headers: { "Content-Type": "application/json" } @@ -309,6 +323,17 @@ expect(response).to have_http_status(:unprocessable_entity) end + it "shouldn't fail model update for connector without catalog for data connector" do + workspace.workspace_users.first.update(role: member_role) + model = models.second + model.connector_id = connector_without_catalog.id + model.save! + put "/api/v1/models/#{models.second.id}", params: request_body.to_json, + headers: { "Content-Type": "application/json" } + .merge(auth_headers(user, workspace_id)) + expect(response).to have_http_status(:ok) + end + it "updates the model and returns success for member role" do workspace.workspace_users.first.update(role: member_role) put "/api/v1/models/#{models.second.id}", params: request_body.to_json, headers: From 0cf9c0976ddbaa8aa0b0cd047ed232fc36c9deed Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 15:05:28 +0530 Subject: [PATCH 26/31] feat(CE): verify user after signup --- ui/src/routes/main.tsx | 43 ++++++++ ui/src/services/authentication.ts | 74 ++++++++++--- .../Authentication/AuthCard/AuthCard.tsx | 7 +- .../AuthViews/ForgotPasswordAuthView.tsx | 71 ++++++++++++ .../ForgotPasswordSuccessAuthView.tsx | 45 ++++++++ .../AuthViews/ResetPasswordAuthView.tsx | 83 ++++++++++++++ .../ResetPasswordSuccessAuthView.tsx | 44 ++++++++ .../AuthViews/SignUpVerificationAuthView.tsx | 36 +++++++ .../AuthViews/VerifyUserAuthView.tsx | 36 +++++++ .../AuthViews/VerifyUserFailedAuthView.tsx | 33 ++++++ .../AuthViews/VerifyUserSuccessAuthView.tsx | 37 +++++++ .../ForgotPassword/ForgotPassword.tsx | 96 +++++++++++++++++ .../Authentication/ForgotPassword/index.ts | 1 + .../ResetPassword/ResetPassword.tsx | 102 ++++++++++++++++++ .../Authentication/ResetPassword/index.ts | 1 + ui/src/views/Authentication/SignUp/SignUp.tsx | 2 + .../SignUp/SignUpVerification.tsx | 72 +++++++++++++ .../Authentication/VerifyUser/VerifyUser.tsx | 88 +++++++++++++++ .../views/Authentication/VerifyUser/index.ts | 1 + ui/src/views/Authentication/types.ts | 56 ++++++---- ui/src/views/Sidebar/Sidebar.tsx | 9 +- 21 files changed, 894 insertions(+), 43 deletions(-) create mode 100644 ui/src/views/Authentication/AuthViews/ForgotPasswordAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/ForgotPasswordSuccessAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/ResetPasswordAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/ResetPasswordSuccessAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/SignUpVerificationAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/VerifyUserAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/VerifyUserFailedAuthView.tsx create mode 100644 ui/src/views/Authentication/AuthViews/VerifyUserSuccessAuthView.tsx create mode 100644 ui/src/views/Authentication/ForgotPassword/ForgotPassword.tsx create mode 100644 ui/src/views/Authentication/ForgotPassword/index.ts create mode 100644 ui/src/views/Authentication/ResetPassword/ResetPassword.tsx create mode 100644 ui/src/views/Authentication/ResetPassword/index.ts create mode 100644 ui/src/views/Authentication/SignUp/SignUpVerification.tsx create mode 100644 ui/src/views/Authentication/VerifyUser/VerifyUser.tsx create mode 100644 ui/src/views/Authentication/VerifyUser/index.ts diff --git a/ui/src/routes/main.tsx b/ui/src/routes/main.tsx index f1563519..2292bb12 100644 --- a/ui/src/routes/main.tsx +++ b/ui/src/routes/main.tsx @@ -3,6 +3,13 @@ const AboutUs = lazy(() => import('@/views/AboutUs')); const Dashboard = lazy(() => import('@/views/Dashboard')); const SignIn = lazy(() => import('@/views/Authentication/SignIn')); const SignUp = lazy(() => import('@/views/Authentication/SignUp')); + +const SignUpVerification = lazy(() => import('@/views/Authentication/SignUp/SignUpVerification')); +const VerifyUser = lazy(() => import('@/views/Authentication/VerifyUser')); + +const ForgotPassword = lazy(() => import('@/views/Authentication/ForgotPassword')); +const ResetPassword = lazy(() => import('@/views/Authentication/ResetPassword')); + const Models = lazy(() => import('@/views/Models')); const SetupConnectors = lazy(() => import('@/views/Connectors/SetupConnectors')); @@ -102,4 +109,40 @@ export const AUTH_ROUTES: MAIN_PAGE_ROUTES_ITEM[] = [ ), }, + { + name: 'Sign Up Success', + url: '/sign-up/success', + component: ( + + + + ), + }, + { + name: 'Verify User', + url: '/verify-user', + component: ( + + + + ), + }, + { + name: 'Forgot Password', + url: '/forgot-password', + component: ( + + + + ), + }, + { + name: 'Reset Password', + url: '/reset-password', + component: ( + + + + ), + }, ]; diff --git a/ui/src/services/authentication.ts b/ui/src/services/authentication.ts index 657e9636..bcc9311d 100644 --- a/ui/src/services/authentication.ts +++ b/ui/src/services/authentication.ts @@ -13,7 +13,7 @@ export type SignInPayload = { password: string; }; -export type SignInErrorResponse = { +export type AuthErrorResponse = { status: number; title: string; detail: string; @@ -25,13 +25,7 @@ export type SignInResponse = { attributes: { token: string; }; - errors?: SignInErrorResponse[]; -}; - -export type AuthErrorResponse = { - status: number; - title: string; - detail: string; + errors?: AuthErrorResponse[]; }; export type AuthResponse = { @@ -40,11 +34,7 @@ export type AuthResponse = { attributes: { token: string; }; - errors?: Array<{ - source: { - [key: string]: string; - }; - }>; + errors?: AuthErrorResponse[]; }; export type ApiResponse = { @@ -53,8 +43,37 @@ export type ApiResponse = { errors?: ErrorResponse[]; }; +export type ForgotPasswordPayload = { + email: string; +}; + +export type MessageResponse = { + type: string; + id: number; + attributes: { + message: string; + }; +}; + +export type ResetPasswordPayload = { + reset_password_token: string; + password: string; + password_confirmation: string; +}; + +export type SignUpResponse = { + type: string; + id: string; + attributes: { + created_at: string; + email: string; + name: string; + }; + errors?: AuthErrorResponse[]; +}; + export const signUp = async (payload: SignUpPayload) => - multiwovenFetch>({ + multiwovenFetch>({ method: 'post', url: '/signup', data: payload, @@ -66,3 +85,30 @@ export const signIn = async (payload: SignInPayload) => url: '/login', data: payload, }); + +export const forgotPassword = async (payload: ForgotPasswordPayload) => + multiwovenFetch>({ + method: 'post', + url: '/forgot_password', + data: payload, + }); + +export const resetPassword = async (payload: ResetPasswordPayload) => + multiwovenFetch>({ + method: 'post', + url: '/reset_password', + data: payload, + }); + +export const verifyUser = async (confirmation_token: string) => + multiwovenFetch>({ + method: 'get', + url: `/verify_user?confirmation_token=${confirmation_token}`, + }); + +export const resendUserVerification = async (payload: ForgotPasswordPayload) => + multiwovenFetch>({ + method: 'post', + url: `/resend_verification`, + data: payload, + }); diff --git a/ui/src/views/Authentication/AuthCard/AuthCard.tsx b/ui/src/views/Authentication/AuthCard/AuthCard.tsx index 2c7583cd..b51145ab 100644 --- a/ui/src/views/Authentication/AuthCard/AuthCard.tsx +++ b/ui/src/views/Authentication/AuthCard/AuthCard.tsx @@ -1,5 +1,6 @@ import { Box, Container, Flex, Image, Stack } from '@chakra-ui/react'; import { AuthCardProps } from '../types'; +import multiwovenLogo from '@/assets/images/icon-white.svg'; const AuthCard = ({ children, brandName, logoUrl }: AuthCardProps): JSX.Element => ( <> @@ -17,7 +18,11 @@ const AuthCard = ({ children, brandName, logoUrl }: AuthCardProps): JSX.Element borderRadius='11px' mx='auto' > - + { + return ( + <> + + + {({ getFieldProps, touched, errors }) => ( +
+ + + Reset your password + + + Enter your email address and we will send you instructions to reset your password. + + + + + + + + + + + + + Back to {brandName} + + + + + + )} + + + + ); +}; diff --git a/ui/src/views/Authentication/AuthViews/ForgotPasswordSuccessAuthView.tsx b/ui/src/views/Authentication/AuthViews/ForgotPasswordSuccessAuthView.tsx new file mode 100644 index 00000000..b666aa89 --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/ForgotPasswordSuccessAuthView.tsx @@ -0,0 +1,45 @@ +import AuthCard from '../AuthCard'; +import { Button, Heading, Stack, Text } from '@chakra-ui/react'; + +type ForgotPasswordSuccessAuthViewProps = { + brandName: string; + logoUrl: string; + email: string; + handleEmailResend: (values: any) => void; + submitting: boolean; +}; + +export const ForgotPasswordSuccessAuthView = ({ + brandName, + logoUrl, + email, + handleEmailResend, + submitting, +}: ForgotPasswordSuccessAuthViewProps) => { + return ( + <> + + + + Check your email + + + Please check the email address {email} for instructions to reset your password. + + + + + + + + ); +}; diff --git a/ui/src/views/Authentication/AuthViews/ResetPasswordAuthView.tsx b/ui/src/views/Authentication/AuthViews/ResetPasswordAuthView.tsx new file mode 100644 index 00000000..08912681 --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/ResetPasswordAuthView.tsx @@ -0,0 +1,83 @@ +import AuthCard from '../AuthCard'; +import { Form, Formik } from 'formik'; +import { Button, HStack, Heading, Stack, Text } from '@chakra-ui/react'; +import { PasswordField } from '@/components/Fields'; +import { Link } from 'react-router-dom'; +import { ResetPasswordAuthViewProps } from '../types'; +import { ResetPasswordSchema } from '@/constants/schemas'; + +export const ResetPasswordAuthView = ({ + brandName, + logoUrl, + handleSubmit, + submitting, +}: ResetPasswordAuthViewProps) => { + return ( + <> + + + {({ getFieldProps, touched, errors }) => ( +
+ + + {'Reset your password'} + + + {`Enter a new password below to change your password`} + + + + + + + + + + + + + + Back to {brandName} + + + + + + )} + + + + ); +}; diff --git a/ui/src/views/Authentication/AuthViews/ResetPasswordSuccessAuthView.tsx b/ui/src/views/Authentication/AuthViews/ResetPasswordSuccessAuthView.tsx new file mode 100644 index 00000000..5828e2c8 --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/ResetPasswordSuccessAuthView.tsx @@ -0,0 +1,44 @@ +import { useNavigate } from 'react-router-dom'; +import AuthCard from '../AuthCard'; +import { Button, Heading, Stack, Text } from '@chakra-ui/react'; + +type ResetPasswordSuccessAuthViewProps = { + brandName: string; + logoUrl: string; + submitting: boolean; +}; + +export const ResetPasswordSuccessAuthView = ({ + brandName, + logoUrl, + submitting, +}: ResetPasswordSuccessAuthViewProps) => { + const navigate = useNavigate(); + + return ( + <> + + + + Password Changed! + + + Your password has been changed successfully. + + + + + + + + ); +}; diff --git a/ui/src/views/Authentication/AuthViews/SignUpVerificationAuthView.tsx b/ui/src/views/Authentication/AuthViews/SignUpVerificationAuthView.tsx new file mode 100644 index 00000000..878f28df --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/SignUpVerificationAuthView.tsx @@ -0,0 +1,36 @@ +import AuthCard from '../AuthCard'; +import { Button, Heading, Stack, Text } from '@chakra-ui/react'; +import { VerificationAuthViewProps } from '../types'; + +export const SignUpVerificationAuthView = ({ + brandName, + logoUrl, + email, + handleEmailResend, + submitting, +}: VerificationAuthViewProps) => ( + <> + + + + Check your email + + + To complete sign up, click the verification button in the email we sent to {email}. + + + + + + + +); diff --git a/ui/src/views/Authentication/AuthViews/VerifyUserAuthView.tsx b/ui/src/views/Authentication/AuthViews/VerifyUserAuthView.tsx new file mode 100644 index 00000000..8f2b14da --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/VerifyUserAuthView.tsx @@ -0,0 +1,36 @@ +import AuthCard from '../AuthCard'; +import { Button, Heading, Stack, Text } from '@chakra-ui/react'; +import { VerificationAuthViewProps } from '../types'; + +export const VerifyUserAuthView = ({ + brandName, + logoUrl, + email, + handleEmailResend, + submitting, +}: VerificationAuthViewProps) => ( + <> + + + + Check your email + + + To complete sign up, click the verification button in the email we sent to {email}. + + + + + + + +); diff --git a/ui/src/views/Authentication/AuthViews/VerifyUserFailedAuthView.tsx b/ui/src/views/Authentication/AuthViews/VerifyUserFailedAuthView.tsx new file mode 100644 index 00000000..31a7a86c --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/VerifyUserFailedAuthView.tsx @@ -0,0 +1,33 @@ +import AuthCard from '../AuthCard'; +import { Button, Heading, Stack, Text } from '@chakra-ui/react'; +import { VerifyUserFailedAuthViewProps } from '../types'; + +export const VerifyUserFailedAuthView = ({ + brandName, + logoUrl, + resendEmail, +}: VerifyUserFailedAuthViewProps) => ( + <> + + + + Link expired{' '} + + + To verify your email and complete sign up, please resend the verification email. + + + + + + + +); diff --git a/ui/src/views/Authentication/AuthViews/VerifyUserSuccessAuthView.tsx b/ui/src/views/Authentication/AuthViews/VerifyUserSuccessAuthView.tsx new file mode 100644 index 00000000..69b64d23 --- /dev/null +++ b/ui/src/views/Authentication/AuthViews/VerifyUserSuccessAuthView.tsx @@ -0,0 +1,37 @@ +import { useNavigate } from 'react-router-dom'; +import AuthCard from '../AuthCard'; +import { Button, Heading, Stack, Text } from '@chakra-ui/react'; +import { VerifyUserSuccessAuthViewProps } from '../types'; + +export const VerifyUserSuccessAuthView = ({ + brandName, + logoUrl, +}: VerifyUserSuccessAuthViewProps) => { + const navigate = useNavigate(); + + return ( + <> + + + + Email verified! + + + Your account has been successfully verified. + + + + + + + + ); +}; diff --git a/ui/src/views/Authentication/ForgotPassword/ForgotPassword.tsx b/ui/src/views/Authentication/ForgotPassword/ForgotPassword.tsx new file mode 100644 index 00000000..d9563234 --- /dev/null +++ b/ui/src/views/Authentication/ForgotPassword/ForgotPassword.tsx @@ -0,0 +1,96 @@ +import { useState } from 'react'; +import { ForgotPasswordPayload, forgotPassword } from '@/services/authentication'; +import titleCase from '@/utils/TitleCase'; +import AuthFooter from '../AuthFooter'; +import { CustomToastStatus } from '@/components/Toast/index'; +import useCustomToast from '@/hooks/useCustomToast'; +import { useMutation } from '@tanstack/react-query'; + +import { ForgotPasswordAuthView } from '@/views/Authentication/AuthViews/ForgotPasswordAuthView'; + +import { ErrorResponse } from '@/services/common'; +import { ForgotPasswordSuccessAuthView } from '@/views/Authentication/AuthViews/ForgotPasswordSuccessAuthView'; +import mwTheme from '@/chakra.config'; + +const ForgotPassword = (): JSX.Element => { + const [submitting, setSubmitting] = useState(false); + const [success, setSuccess] = useState(false); + const [userEmail, setUserEmail] = useState(''); + + const showToast = useCustomToast(); + + const { mutateAsync } = useMutation({ + mutationFn: (values: ForgotPasswordPayload) => forgotPassword(values), + mutationKey: ['forgot-password'], + }); + + const handleSubmit = async (values: ForgotPasswordPayload) => { + setSubmitting(true); + try { + const result = await mutateAsync(values); + + if (result.data?.attributes.message) { + showToast({ + duration: 3000, + isClosable: true, + position: 'bottom-right', + title: result.data.attributes.message, + status: CustomToastStatus.Success, + }); + setUserEmail(values.email); + setSuccess(true); + } else { + result?.errors?.forEach((error: ErrorResponse) => { + showToast({ + duration: 5000, + isClosable: true, + position: 'bottom-right', + colorScheme: 'red', + status: CustomToastStatus.Warning, + title: titleCase(error.detail), + }); + }); + } + } catch (error) { + showToast({ + duration: 3000, + isClosable: true, + position: 'bottom-right', + title: 'There was an error connecting to the server. Please try again later.', + status: CustomToastStatus.Error, + }); + } finally { + setSubmitting(false); + } + }; + + const { logoUrl, brandName, privacyPolicyUrl, termsOfServiceUrl } = mwTheme; + + return ( + <> + {success ? ( + + ) : ( + + )} + + + ); +}; + +export default ForgotPassword; diff --git a/ui/src/views/Authentication/ForgotPassword/index.ts b/ui/src/views/Authentication/ForgotPassword/index.ts new file mode 100644 index 00000000..f7e2960d --- /dev/null +++ b/ui/src/views/Authentication/ForgotPassword/index.ts @@ -0,0 +1 @@ +export { default } from './ForgotPassword'; diff --git a/ui/src/views/Authentication/ResetPassword/ResetPassword.tsx b/ui/src/views/Authentication/ResetPassword/ResetPassword.tsx new file mode 100644 index 00000000..3b77f658 --- /dev/null +++ b/ui/src/views/Authentication/ResetPassword/ResetPassword.tsx @@ -0,0 +1,102 @@ +import { useState } from 'react'; +import { ResetPasswordPayload, resetPassword } from '@/services/authentication'; +import titleCase from '@/utils/TitleCase'; +import AuthFooter from '../AuthFooter'; +import { CustomToastStatus } from '@/components/Toast/index'; +import useCustomToast from '@/hooks/useCustomToast'; +import { useMutation } from '@tanstack/react-query'; + +import { ErrorResponse } from '@/services/common'; + +import { ResetPasswordAuthView } from '@/views/Authentication/AuthViews/ResetPasswordAuthView'; +import { ResetPasswordSuccessAuthView } from '@/views/Authentication/AuthViews/ResetPasswordSuccessAuthView'; +import { useSearchParams } from 'react-router-dom'; +import mwTheme from '@/chakra.config'; +import { ResetPasswordFormPayload } from '../types'; + +const ResetPassword = (): JSX.Element => { + const [submitting, setSubmitting] = useState(false); + const [success, setSuccess] = useState(false); + + const showToast = useCustomToast(); + + const [searchParams] = useSearchParams(); + const token = searchParams.get('reset_password_token'); + + const { mutateAsync } = useMutation({ + mutationFn: (values: ResetPasswordPayload) => resetPassword(values), + mutationKey: ['Reset-password'], + }); + + const handleSubmit = async (values: ResetPasswordFormPayload) => { + setSubmitting(true); + + if (token !== null) { + const payload = { ...values, reset_password_token: token }; + try { + const result = await mutateAsync(payload); + + if (result.data?.attributes.message) { + showToast({ + duration: 3000, + isClosable: true, + position: 'bottom-right', + title: result.data.attributes.message, + status: CustomToastStatus.Success, + }); + setSuccess(true); + } else { + result?.errors?.forEach((error: ErrorResponse) => { + showToast({ + duration: 5000, + isClosable: true, + position: 'bottom-right', + colorScheme: 'red', + status: CustomToastStatus.Warning, + title: titleCase(error.detail) + ' Please try resetting your password again.', + }); + }); + } + } catch (error) { + showToast({ + duration: 3000, + isClosable: true, + position: 'bottom-right', + title: 'There was an error connecting to the server. Please try again later.', + status: CustomToastStatus.Error, + }); + } finally { + setSubmitting(false); + } + } + }; + + const { logoUrl, brandName, privacyPolicyUrl, termsOfServiceUrl } = mwTheme; + + return ( + <> + {success ? ( + + ) : ( + + )} + + + + ); +}; + +export default ResetPassword; diff --git a/ui/src/views/Authentication/ResetPassword/index.ts b/ui/src/views/Authentication/ResetPassword/index.ts new file mode 100644 index 00000000..8e628c60 --- /dev/null +++ b/ui/src/views/Authentication/ResetPassword/index.ts @@ -0,0 +1 @@ +export { default } from './ResetPassword'; diff --git a/ui/src/views/Authentication/SignUp/SignUp.tsx b/ui/src/views/Authentication/SignUp/SignUp.tsx index 13fc6956..c336a940 100644 --- a/ui/src/views/Authentication/SignUp/SignUp.tsx +++ b/ui/src/views/Authentication/SignUp/SignUp.tsx @@ -1,5 +1,6 @@ import { useNavigate } from 'react-router-dom'; import { Stack, Heading } from '@chakra-ui/react'; + import mwTheme from '@/chakra.config'; import AuthFooter from '../AuthFooter'; import { SignUpAuthView } from '@/views/Authentication/AuthViews/SignUpAuthView'; @@ -9,6 +10,7 @@ import { SignUpPayload, signUp } from '@/services/authentication'; import { useState } from 'react'; import useCustomToast from '@/hooks/useCustomToast'; import { useMutation } from '@tanstack/react-query'; + import { useAPIErrorsToast, useErrorToast } from '@/hooks/useErrorToast'; // import isValidEmailDomain from '@/utils/isValidEmailDomain'; diff --git a/ui/src/views/Authentication/SignUp/SignUpVerification.tsx b/ui/src/views/Authentication/SignUp/SignUpVerification.tsx new file mode 100644 index 00000000..07d5e12d --- /dev/null +++ b/ui/src/views/Authentication/SignUp/SignUpVerification.tsx @@ -0,0 +1,72 @@ +import { useSearchParams } from 'react-router-dom'; +import mwTheme from '@/chakra.config'; +import AuthFooter from '../AuthFooter'; +import { CustomToastStatus } from '@/components/Toast'; +import { resendUserVerification } from '@/services/authentication'; +import { useState } from 'react'; +import useCustomToast from '@/hooks/useCustomToast'; +import { useMutation } from '@tanstack/react-query'; +import { SignUpVerificationAuthView } from '../AuthViews/SignUpVerificationAuthView'; +import { useAPIErrorsToast, useErrorToast } from '@/hooks/useErrorToast'; + +const SignUpVerification = (): JSX.Element => { + const [submitting, setSubmitting] = useState(false); + const showToast = useCustomToast(); + const apiErrorToast = useAPIErrorsToast(); + const errorToast = useErrorToast(); + + const [searchParams] = useSearchParams(); + const email = searchParams.get('email') || ''; + + const { mutateAsync } = useMutation({ + mutationFn: (email: string) => resendUserVerification({ email }), + mutationKey: ['verify-user'], + }); + + const handleEmailResend = async (email: string) => { + setSubmitting(true); + try { + const result = await mutateAsync(email); + + if (result.data?.attributes) { + showToast({ + title: result.data.attributes.message, + status: CustomToastStatus.Success, + duration: 3000, + isClosable: true, + position: 'bottom-right', + }); + + setSubmitting(false); + } else { + apiErrorToast(result.errors || []); + } + } catch (error) { + errorToast('An error occured. Please try again later.', true, null, true); + } finally { + setSubmitting(false); + } + }; + + const { logoUrl, brandName, privacyPolicyUrl, termsOfServiceUrl } = mwTheme; + + return ( + <> + handleEmailResend(email)} + submitting={submitting} + /> + + + + ); +}; + +export default SignUpVerification; diff --git a/ui/src/views/Authentication/VerifyUser/VerifyUser.tsx b/ui/src/views/Authentication/VerifyUser/VerifyUser.tsx new file mode 100644 index 00000000..efbc08ef --- /dev/null +++ b/ui/src/views/Authentication/VerifyUser/VerifyUser.tsx @@ -0,0 +1,88 @@ +import { useEffect, useState } from 'react'; +import { useSearchParams } from 'react-router-dom'; +import mwTheme from '@/chakra.config'; +import AuthFooter from '../AuthFooter'; +import { CustomToastStatus } from '@/components/Toast'; +import { resendUserVerification, verifyUser } from '@/services/authentication'; +import useCustomToast from '@/hooks/useCustomToast'; +import { useMutation } from '@tanstack/react-query'; +import { VerifyUserSuccessAuthView } from '../AuthViews/VerifyUserSuccessAuthView'; +import { VerifyUserFailedAuthView } from '../AuthViews/VerifyUserFailedAuthView'; +import Loader from '@/components/Loader'; +import { useAPIErrorsToast, useErrorToast } from '@/hooks/useErrorToast'; + +const VerifyUser = (): JSX.Element => { + const [success, setSuccess] = useState(false); + const [submitting, setSubmitting] = useState(true); // Add this to handle the final state + + const showToast = useCustomToast(); + const apiErrorToast = useAPIErrorsToast(); + const errorToast = useErrorToast(); + + const [searchParams] = useSearchParams(); + const confirmation_token = searchParams.get('confirmation_token'); + const email = searchParams.get('email') || ''; + + const { mutateAsync } = useMutation({ + mutationFn: (email: string) => resendUserVerification({ email }), + mutationKey: ['resend-verification'], + }); + + const { logoUrl, brandName, privacyPolicyUrl, termsOfServiceUrl } = mwTheme; + + const verifyUserToken = async () => { + if (!confirmation_token) { + setSubmitting(false); + return; + } + + try { + const result = await verifyUser(confirmation_token); + if (result.data?.attributes) { + showToast({ + title: 'Account verified.', + status: CustomToastStatus.Success, + duration: 3000, + isClosable: true, + position: 'bottom-right', + }); + setSuccess(true); + } else { + apiErrorToast(result.errors || []); + } + } catch (error) { + errorToast('An error occured. Please try again later.', true, null, true); + } finally { + setSubmitting(false); + } + }; + + useEffect(() => { + verifyUserToken(); + }, []); + + if (submitting) { + return ; + } + + return ( + <> + {success ? ( + + ) : ( + mutateAsync(email)} + /> + )} + + + ); +}; + +export default VerifyUser; diff --git a/ui/src/views/Authentication/VerifyUser/index.ts b/ui/src/views/Authentication/VerifyUser/index.ts new file mode 100644 index 00000000..dc1e1664 --- /dev/null +++ b/ui/src/views/Authentication/VerifyUser/index.ts @@ -0,0 +1 @@ +export { default } from './VerifyUser'; diff --git a/ui/src/views/Authentication/types.ts b/ui/src/views/Authentication/types.ts index f0168859..a4e4b673 100644 --- a/ui/src/views/Authentication/types.ts +++ b/ui/src/views/Authentication/types.ts @@ -1,28 +1,35 @@ import { ReactNode } from 'react'; -export type AuthCardProps = { - children: ReactNode; + +type DefaultAuthViewProps = { brandName: string; logoUrl: string; }; +type Email = { + email: string; +}; + +type Password = { + password: string; + password_confirmation: string; +}; + +export type AuthCardProps = { + children: ReactNode; +} & DefaultAuthViewProps; + export type SignInAuthViewProps = { - brandName: string; - logoUrl: string; handleSubmit: (values: any) => void; submitting: boolean; -}; +} & DefaultAuthViewProps; type InitialValues = { company_name: string; name: string; - email: string; - password: string; - password_confirmation: string; -}; +} & Email & + Password; export type SignUpAuthViewProps = { - brandName: string; - logoUrl: string; handleSubmit: (values: any) => void; submitting: boolean; initialValues?: InitialValues; @@ -30,23 +37,28 @@ export type SignUpAuthViewProps = { termsOfServiceUrl: string; isCompanyNameDisabled?: boolean; isEmailDisabled?: boolean; -}; +} & DefaultAuthViewProps; -export type ResetPasswordFormPayload = { - password: string; - password_confirmation: string; -}; +export type ResetPasswordFormPayload = Password; -export type ForgotPasswordFormPayload = { - email: string; -}; +export type ForgotPasswordFormPayload = Email; type ChangePasswordProps = { - brandName: string; - logoUrl: string; handleSubmit: (values: T) => void; submitting: boolean; -}; +} & DefaultAuthViewProps; export type ForgotPasswordAuthViewProps = ChangePasswordProps; export type ResetPasswordAuthViewProps = ChangePasswordProps; + +export type VerificationAuthViewProps = { + handleEmailResend: (email: string) => void; + submitting: boolean; +} & DefaultAuthViewProps & + Email; + +export type VerifyUserFailedAuthViewProps = { + resendEmail: () => void; +} & DefaultAuthViewProps; + +export type VerifyUserSuccessAuthViewProps = DefaultAuthViewProps; diff --git a/ui/src/views/Sidebar/Sidebar.tsx b/ui/src/views/Sidebar/Sidebar.tsx index 081168dd..5e798b52 100644 --- a/ui/src/views/Sidebar/Sidebar.tsx +++ b/ui/src/views/Sidebar/Sidebar.tsx @@ -1,6 +1,6 @@ import { Box, Flex, Stack, Text, Divider } from '@chakra-ui/react'; import { NavLink } from 'react-router-dom'; -import IconImage from '../../assets/images/multiwoven-logo.svg'; +import IconImage from '@/assets/images/multiwoven-logo.svg'; import { FiSettings, FiDatabase, @@ -15,8 +15,6 @@ import { NavButton } from './navButton'; import Profile from './Profile'; import Workspace from './Workspace/Workspace'; -import { useConfigStore } from '@/stores/useConfigStore'; - type MenuItem = { title: string; link: string; @@ -76,7 +74,7 @@ const renderMenuSection = (section: MenuSection, index: number) => ( )} {section.menu.map((menuItem, idx) => ( - + {({ isActive }) => ( ( ); const Sidebar = (): JSX.Element => { - const { logoUrl } = useConfigStore.getState().configs; return ( { - IconImage + IconImage From b67d824115a4dc53c5615d8320bf2c37ddcc81a6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:18:57 +0530 Subject: [PATCH 27/31] fix(CE): fixed sync runs on click function --- ui/src/hooks/syncs/useSyncRuns.tsx | 14 ++++++++++++++ .../Activate/Syncs/SyncRuns/SyncRuns.tsx | 19 +++++++++---------- 2 files changed, 23 insertions(+), 10 deletions(-) create mode 100644 ui/src/hooks/syncs/useSyncRuns.tsx diff --git a/ui/src/hooks/syncs/useSyncRuns.tsx b/ui/src/hooks/syncs/useSyncRuns.tsx new file mode 100644 index 00000000..34c72476 --- /dev/null +++ b/ui/src/hooks/syncs/useSyncRuns.tsx @@ -0,0 +1,14 @@ +import { useQuery } from '@tanstack/react-query'; +import { getSyncRunsBySyncId } from '@/services/syncs'; + +const useSyncRuns = (syncId: string, currentPage: number, activeWorkspaceId: number) => { + return useQuery({ + queryKey: ['activate', 'sync-runs', syncId, 'page-' + currentPage, activeWorkspaceId], + queryFn: () => getSyncRunsBySyncId(syncId, currentPage.toString()), + refetchOnMount: true, + refetchOnWindowFocus: false, + enabled: activeWorkspaceId > 0, + }); +}; + +export default useSyncRuns; diff --git a/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx b/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx index 9029ec05..cd57fa59 100644 --- a/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx +++ b/ui/src/views/Activate/Syncs/SyncRuns/SyncRuns.tsx @@ -1,15 +1,19 @@ -import { useQuery } from '@tanstack/react-query'; import { useNavigate, useParams, useSearchParams } from 'react-router-dom'; -import { getSyncRunsBySyncId } from '@/services/syncs'; import { useMemo, useState, useEffect } from 'react'; import { Box } from '@chakra-ui/react'; import Loader from '@/components/Loader'; import Pagination from '@/components/Pagination'; +import { useStore } from '@/stores'; +import useSyncRuns from '@/hooks/syncs/useSyncRuns'; import { SyncRunsColumns } from './SyncRunsColumns'; import DataTable from '@/components/DataTable'; +import { Row } from '@tanstack/react-table'; +import { SyncRunsResponse } from '../types'; import RowsNotFound from '@/components/DataTable/RowsNotFound'; const SyncRuns = () => { + const activeWorkspaceId = useStore((state) => state.workspaceId); + const { syncId } = useParams(); const [searchParams, setSearchParams] = useSearchParams(); const navigate = useNavigate(); @@ -21,15 +25,10 @@ const SyncRuns = () => { setSearchParams({ page: currentPage.toString() }); }, [currentPage, setSearchParams]); - const { data, isLoading } = useQuery({ - queryKey: ['activate', 'sync-runs', syncId, 'page-' + currentPage], - queryFn: () => getSyncRunsBySyncId(syncId as string, currentPage.toString()), - refetchOnMount: true, - refetchOnWindowFocus: false, - }); + const { data, isLoading } = useSyncRuns(syncId as string, currentPage, activeWorkspaceId); - const handleOnSyncClick = (row: Record<'id', string>) => { - navigate(`run/${row.id}`); + const handleOnSyncClick = (row: Row) => { + navigate(`run/${row.original.id}`); }; const syncList = data?.data; From a3587a97ccbb1e49b34d6efe59af06b8d08265ea Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Thu, 5 Sep 2024 09:57:54 -0400 Subject: [PATCH 28/31] fix(CE): added try catch to model query preview API Call --- ui/src/hooks/useAPIMutation.tsx | 56 +++++++++++++ ui/src/services/models.ts | 11 +-- ui/src/utils/ConvertToTableData.ts | 1 + ui/src/views/Authentication/SignUp/SignUp.tsx | 2 - .../DefineModel/DefineSQL/DefineSQL.tsx | 80 +++++++++++-------- .../TableSelector/TableSelector.tsx | 49 +++++------- 6 files changed, 125 insertions(+), 74 deletions(-) create mode 100644 ui/src/hooks/useAPIMutation.tsx diff --git a/ui/src/hooks/useAPIMutation.tsx b/ui/src/hooks/useAPIMutation.tsx new file mode 100644 index 00000000..6e1abd16 --- /dev/null +++ b/ui/src/hooks/useAPIMutation.tsx @@ -0,0 +1,56 @@ +import { useState } from 'react'; +import { useMutation } from '@tanstack/react-query'; +import { ApiResponse } from '@/services/common'; +import useCustomToast from '@/hooks/useCustomToast'; +import { CustomToastStatus } from '@/components/Toast/index'; +import { useAPIErrorsToast, useErrorToast } from './useErrorToast'; + +type UseApiMutationOptions = { + mutationFn: (variables: TVariables) => Promise>; + successMessage: string; + errorMessage: string; + onSuccessCallback?: (result: ApiResponse) => void; +}; + +const useApiMutation = ({ + mutationFn, + successMessage, + errorMessage, + onSuccessCallback, +}: UseApiMutationOptions) => { + const [isSubmitting, setIsSubmitting] = useState(false); + const showToast = useCustomToast(); + const apiErrorToast = useAPIErrorsToast(); + const errorToast = useErrorToast(); + + const mutation = useMutation, Error, TVariables>({ + mutationFn, + onSuccess: (result) => { + if (result?.errors && result?.errors.length > 0) { + apiErrorToast(result.errors); + } else { + showToast({ + status: CustomToastStatus.Success, + title: successMessage, + position: 'bottom-right', + isClosable: true, + }); + onSuccessCallback?.(result); + } + setIsSubmitting(false); + }, + onError: () => { + errorToast(errorMessage, true, null, true); + setIsSubmitting(false); + }, + }); + + const triggerMutation = async (variables: TVariables) => { + setIsSubmitting(true); + await mutation.mutateAsync(variables); + }; + + return { isSubmitting, triggerMutation }; +}; + +export default useApiMutation; diff --git a/ui/src/services/models.ts b/ui/src/services/models.ts index 2338e5a7..18cf4289 100644 --- a/ui/src/services/models.ts +++ b/ui/src/services/models.ts @@ -24,16 +24,7 @@ export type Field = { [key: string]: string | number | null; }; -export type ModelPreviewResponse = - | { errors?: ErrorResponse[] } - | Field[] - | { - errors?: { - detail: string; - status: number; - title: string; - }[]; - }; +export type ModelPreviewResponse = { errors?: ErrorResponse[] } | Field[]; export type ModelAttributes = { updated_at: string; diff --git a/ui/src/utils/ConvertToTableData.ts b/ui/src/utils/ConvertToTableData.ts index 514cb08a..b3d70303 100644 --- a/ui/src/utils/ConvertToTableData.ts +++ b/ui/src/utils/ConvertToTableData.ts @@ -39,6 +39,7 @@ type Field = { }; export function ConvertModelPreviewToTableData(apiData: Array): TableDataType { + if (apiData.length === 0) return { columns: [], data: [] }; const column_names = Object.keys(apiData[0]); const columns = column_names.map((column_name) => { diff --git a/ui/src/views/Authentication/SignUp/SignUp.tsx b/ui/src/views/Authentication/SignUp/SignUp.tsx index c336a940..13fc6956 100644 --- a/ui/src/views/Authentication/SignUp/SignUp.tsx +++ b/ui/src/views/Authentication/SignUp/SignUp.tsx @@ -1,6 +1,5 @@ import { useNavigate } from 'react-router-dom'; import { Stack, Heading } from '@chakra-ui/react'; - import mwTheme from '@/chakra.config'; import AuthFooter from '../AuthFooter'; import { SignUpAuthView } from '@/views/Authentication/AuthViews/SignUpAuthView'; @@ -10,7 +9,6 @@ import { SignUpPayload, signUp } from '@/services/authentication'; import { useState } from 'react'; import useCustomToast from '@/hooks/useCustomToast'; import { useMutation } from '@tanstack/react-query'; - import { useAPIErrorsToast, useErrorToast } from '@/hooks/useErrorToast'; // import isValidEmailDomain from '@/utils/isValidEmailDomain'; diff --git a/ui/src/views/Models/ModelsForm/DefineModel/DefineSQL/DefineSQL.tsx b/ui/src/views/Models/ModelsForm/DefineModel/DefineSQL/DefineSQL.tsx index 94f67554..07d2ef85 100644 --- a/ui/src/views/Models/ModelsForm/DefineModel/DefineSQL/DefineSQL.tsx +++ b/ui/src/views/Models/ModelsForm/DefineModel/DefineSQL/DefineSQL.tsx @@ -19,8 +19,8 @@ import { CustomToastStatus } from '@/components/Toast/index'; import useCustomToast from '@/hooks/useCustomToast'; import { format } from 'sql-formatter'; import { autocompleteEntries } from './autocomplete'; -import titleCase from '@/utils/TitleCase'; import ModelQueryResults from '../ModelQueryResults'; +import { useAPIErrorsToast, useErrorToast } from '@/hooks/useErrorToast'; const DefineSQL = ({ hasPrefilledValues = false, @@ -36,6 +36,9 @@ const DefineSQL = ({ const [userQuery, setUserQuery] = useState(prefillValues?.query || ''); const showToast = useCustomToast(); + const apiErrorsToast = useAPIErrorsToast(); + const errorToast = useErrorToast(); + const navigate = useNavigate(); const editorRef = useRef(null); const monaco = useMonaco(); @@ -78,22 +81,33 @@ const DefineSQL = ({ async function getPreview() { setLoading(true); const query = editorRef.current?.getValue() as string; - const response = await getModelPreviewById(query, connector_id?.toString()); - if ('errors' in response) { - response.errors?.forEach((error) => { - showToast({ - duration: 5000, - isClosable: true, - position: 'bottom-right', - colorScheme: 'red', - status: CustomToastStatus.Warning, - title: titleCase(error.detail), - }); - }); - setLoading(false); - } else { - setTableData(ConvertModelPreviewToTableData(response as Field[])); - canMoveForward(true); + try { + const response = await getModelPreviewById(query, connector_id?.toString()); + if ('errors' in response) { + if (response.errors) { + apiErrorsToast(response.errors); + } else { + errorToast('Error fetching preview data', true, null, true); + } + setLoading(false); + } else { + const previewData = response as Field[]; + if (previewData.length === 0) { + showToast({ + title: 'No data found', + status: CustomToastStatus.Success, + duration: 3000, + isClosable: true, + position: 'bottom-right', + }); + setLoading(false); + } else { + setTableData(ConvertModelPreviewToTableData(response as Field[])); + setLoading(false); + } + } + } catch (error) { + errorToast('Error fetching preview data', true, null, true); setLoading(false); } } @@ -111,27 +125,25 @@ const DefineSQL = ({ }, }; - const modelUpdateResponse = await putModelById(prefillValues?.model_id || '', updatePayload); - if (modelUpdateResponse.data) { - showToast({ - title: 'Model updated successfully', - status: CustomToastStatus.Success, - duration: 3000, - isClosable: true, - position: 'bottom-right', - }); - navigate('/define/models/' + prefillValues?.model_id || ''); - } else { - modelUpdateResponse.errors?.forEach((error) => { + try { + const modelUpdateResponse = await putModelById(prefillValues?.model_id || '', updatePayload); + if (modelUpdateResponse.errors) { + apiErrorsToast(modelUpdateResponse.errors); + setLoading(false); + } else { showToast({ - duration: 5000, + title: 'Model updated successfully', + status: CustomToastStatus.Success, + duration: 3000, isClosable: true, position: 'bottom-right', - colorScheme: 'red', - status: CustomToastStatus.Warning, - title: titleCase(error.detail), }); - }); + navigate('/define/models/' + prefillValues?.model_id || ''); + setLoading(false); + } + } catch (error) { + errorToast('Error fetching preview data', true, null, true); + setLoading(false); } } diff --git a/ui/src/views/Models/ModelsForm/DefineModel/TableSelector/TableSelector.tsx b/ui/src/views/Models/ModelsForm/DefineModel/TableSelector/TableSelector.tsx index 70754945..a33a80b2 100644 --- a/ui/src/views/Models/ModelsForm/DefineModel/TableSelector/TableSelector.tsx +++ b/ui/src/views/Models/ModelsForm/DefineModel/TableSelector/TableSelector.tsx @@ -14,7 +14,6 @@ import ListTables from './ListTables'; import { Field, getModelPreviewById, putModelById } from '@/services/models'; import useCustomToast from '@/hooks/useCustomToast'; import { CustomToastStatus } from '@/components/Toast/index'; -import titleCase from '@/utils/TitleCase'; import { TableDataType } from '@/components/Table/types'; import { ConvertModelPreviewToTableData } from '@/utils/ConvertToTableData'; import GenerateTable from '@/components/Table/Table'; @@ -23,6 +22,7 @@ import { useNavigate } from 'react-router-dom'; import { QueryType } from '@/views/Models/types'; import ViewSQLModal from './ViewSQLModal'; import SearchBar from '@/components/SearchBar/SearchBar'; +import { useAPIErrorsToast, useErrorToast } from '@/hooks/useErrorToast'; const generateQuery = (table: string) => `SELECT * FROM ${table}`; @@ -46,6 +46,8 @@ const TableSelector = ({ const navigate = useNavigate(); const showToast = useCustomToast(); + const apiErrorsToast = useAPIErrorsToast(); + const errorToast = useErrorToast(); const { state, stepInfo, handleMoveForward } = useContext(SteppedFormContext); @@ -99,7 +101,9 @@ const TableSelector = ({ }; const modelUpdateResponse = await putModelById(prefillValues?.model_id || '', updatePayload); - if (modelUpdateResponse.data) { + if (modelUpdateResponse.errors) { + apiErrorsToast(modelUpdateResponse.errors); + } else { showToast({ title: 'Model updated successfully', status: CustomToastStatus.Success, @@ -108,38 +112,27 @@ const TableSelector = ({ position: 'bottom-right', }); navigate('/define/models/' + prefillValues?.model_id || ''); - } else { - modelUpdateResponse.errors?.forEach((error) => { - showToast({ - duration: 5000, - isClosable: true, - position: 'bottom-right', - colorScheme: 'red', - status: CustomToastStatus.Warning, - title: titleCase(error.detail), - }); - }); } } const getPreview = async () => { setLoadingPreviewData(true); - const response = await getModelPreviewById(userQuery, connector_id?.toString()); - if ('errors' in response) { - response.errors?.forEach((error) => { - showToast({ - duration: 5000, - isClosable: true, - position: 'bottom-right', - colorScheme: 'red', - status: CustomToastStatus.Warning, - title: titleCase(error.detail), - }); - }); - setLoadingPreviewData(false); - } else { - setTableData(ConvertModelPreviewToTableData(response as Field[])); + try { + const response = await getModelPreviewById(userQuery, connector_id?.toString()); + if ('errors' in response) { + if (response.errors) { + apiErrorsToast(response.errors); + } else { + errorToast('Error fetching preview data', true, null, true); + } + setLoadingPreviewData(false); + } else { + setTableData(ConvertModelPreviewToTableData(response as Field[])); + setLoadingPreviewData(false); + } + } catch (error) { + errorToast('Error fetching preview data', true, null, true); setLoadingPreviewData(false); } }; From 4146f1bcec0f7e57732ad2e7ba4f7695a0ea315d Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:40:38 -0400 Subject: [PATCH 29/31] fix(CE): fixed sync mapping model column values --- ui/src/components/ModelTable/ModelTable.tsx | 2 +- ui/src/services/models.ts | 23 ++++---- .../ConfigureSyncs/MapCustomFields.tsx | 2 +- .../SyncForm/ConfigureSyncs/MapFields.tsx | 2 +- .../SyncForm/ConfigureSyncs/SelectStreams.tsx | 2 +- .../DefineModel/DefineSQL/DefineSQL.tsx | 16 ++--- .../TableSelector/TableSelector.tsx | 20 +++++-- ui/src/views/Models/ModelsList/ModelsList.tsx | 50 ++++++++-------- .../ModelsListTable/ModelsListTable.tsx | 59 +++++++++++++++++++ .../ModelsList/ModelsListTable/index.ts | 1 + 10 files changed, 127 insertions(+), 50 deletions(-) create mode 100644 ui/src/views/Models/ModelsList/ModelsListTable/ModelsListTable.tsx create mode 100644 ui/src/views/Models/ModelsList/ModelsListTable/index.ts diff --git a/ui/src/components/ModelTable/ModelTable.tsx b/ui/src/components/ModelTable/ModelTable.tsx index 6eac3176..a8984ff5 100644 --- a/ui/src/components/ModelTable/ModelTable.tsx +++ b/ui/src/components/ModelTable/ModelTable.tsx @@ -15,7 +15,7 @@ const ModelTable = ({ handleOnRowClick }: ModelTableProps): JSX.Element => { const { data } = useQueryWrapper( ['models', activeWorkspaceId], - () => getAllModels(), + () => getAllModels({ type: 'data' }), { refetchOnMount: true, refetchOnWindowFocus: false, diff --git a/ui/src/services/models.ts b/ui/src/services/models.ts index 18cf4289..d323104b 100644 --- a/ui/src/services/models.ts +++ b/ui/src/services/models.ts @@ -3,8 +3,9 @@ import { CreateModelResponse, GetModelByIdResponse, } from '@/views/Models/types'; -import { apiRequest, multiwovenFetch, ApiResponse, ErrorResponse } from './common'; +import { apiRequest, multiwovenFetch } from './common'; import { UpdateModelPayload } from '@/views/Models/ViewModel/types'; +import { ApiResponse } from './common'; export type APIData = { data?: Array; @@ -24,8 +25,6 @@ export type Field = { [key: string]: string | number | null; }; -export type ModelPreviewResponse = { errors?: ErrorResponse[] } | Field[]; - export type ModelAttributes = { updated_at: string; created_at: string; @@ -48,30 +47,32 @@ export type GetAllModelsResponse = { attributes: ModelAttributes; }; -// export const getAllModels = async (): Promise> => { -// return apiRequest("/models", null); -// }; +export type ModelQueryType = 'data' | 'ai_ml' | 'raw_sql' | 'dbt' | 'soql' | 'table_selector'; + +export type GetAllModelsProps = { + type: ModelQueryType; +}; export const getModelPreview = async (query: string, connector_id: string): Promise => { const url = '/connectors/' + connector_id + '/query_source'; return apiRequest(url, { query: query }); }; -export const getAllModels = async (): Promise => +export const getAllModels = async ({ type = 'data' }: GetAllModelsProps): Promise => multiwovenFetch({ method: 'get', - url: '/models', + url: type ? `/models?query_type=${type}` : '/models', }); export const getModelPreviewById = async (query: string, id: string) => - multiwovenFetch({ + multiwovenFetch>({ method: 'post', url: '/connectors/' + id + '/query_source', data: { query: query }, }); -export const createNewModel = async (payload: CreateModelPayload): Promise => - multiwovenFetch({ +export const createNewModel = async (payload: CreateModelPayload) => + multiwovenFetch>({ method: 'post', url: '/models', data: payload, diff --git a/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/MapCustomFields.tsx b/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/MapCustomFields.tsx index c34bfd13..5d924cfd 100644 --- a/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/MapCustomFields.tsx +++ b/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/MapCustomFields.tsx @@ -56,7 +56,7 @@ const MapCustomFields = ({ } }, [data]); - const firstRow = Array.isArray(previewModelData) && previewModelData[0]; + const firstRow = Array.isArray(previewModelData?.data) && previewModelData.data[0]; const modelColumns = Object.keys(firstRow ?? {}); diff --git a/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/MapFields.tsx b/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/MapFields.tsx index 3a0c670f..132c9fac 100644 --- a/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/MapFields.tsx +++ b/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/MapFields.tsx @@ -65,7 +65,7 @@ const MapFields = ({ } }, [data]); - const firstRow = Array.isArray(previewModelData) && previewModelData[0]; + const firstRow = Array.isArray(previewModelData?.data) && previewModelData.data[0]; const modelColumns = Object.keys(firstRow ?? {}); diff --git a/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/SelectStreams.tsx b/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/SelectStreams.tsx index 7603863e..1e13d841 100644 --- a/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/SelectStreams.tsx +++ b/ui/src/views/Activate/Syncs/SyncForm/ConfigureSyncs/SelectStreams.tsx @@ -56,7 +56,7 @@ const SelectStreams = ({ refetchOnWindowFocus: false, }); - const firstRow = Array.isArray(previewModelData) && previewModelData[0]; + const firstRow = Array.isArray(previewModelData?.data) && previewModelData.data[0]; const modelColumns = Object.keys(firstRow ?? {}); diff --git a/ui/src/views/Models/ModelsForm/DefineModel/DefineSQL/DefineSQL.tsx b/ui/src/views/Models/ModelsForm/DefineModel/DefineSQL/DefineSQL.tsx index 07d2ef85..dfd640ab 100644 --- a/ui/src/views/Models/ModelsForm/DefineModel/DefineSQL/DefineSQL.tsx +++ b/ui/src/views/Models/ModelsForm/DefineModel/DefineSQL/DefineSQL.tsx @@ -4,7 +4,7 @@ import StarsImage from '@/assets/images/stars.svg'; import Editor, { useMonaco } from '@monaco-editor/react'; import { useContext, useEffect, useRef, useState } from 'react'; -import { Field, getModelPreviewById, putModelById } from '@/services/models'; +import { getModelPreviewById, putModelById } from '@/services/models'; import { ConvertModelPreviewToTableData } from '@/utils/ConvertToTableData'; import GenerateTable from '@/components/Table/Table'; import { TableDataType } from '@/components/Table/types'; @@ -83,7 +83,7 @@ const DefineSQL = ({ const query = editorRef.current?.getValue() as string; try { const response = await getModelPreviewById(query, connector_id?.toString()); - if ('errors' in response) { + if (response.errors) { if (response.errors) { apiErrorsToast(response.errors); } else { @@ -91,8 +91,11 @@ const DefineSQL = ({ } setLoading(false); } else { - const previewData = response as Field[]; - if (previewData.length === 0) { + if (response.data && response.data.length > 0) { + setTableData(ConvertModelPreviewToTableData(response.data)); + setLoading(false); + canMoveForward(true); + } else { showToast({ title: 'No data found', status: CustomToastStatus.Success, @@ -100,10 +103,9 @@ const DefineSQL = ({ isClosable: true, position: 'bottom-right', }); + setTableData(null); setLoading(false); - } else { - setTableData(ConvertModelPreviewToTableData(response as Field[])); - setLoading(false); + canMoveForward(false); } } } catch (error) { diff --git a/ui/src/views/Models/ModelsForm/DefineModel/TableSelector/TableSelector.tsx b/ui/src/views/Models/ModelsForm/DefineModel/TableSelector/TableSelector.tsx index a33a80b2..4103977e 100644 --- a/ui/src/views/Models/ModelsForm/DefineModel/TableSelector/TableSelector.tsx +++ b/ui/src/views/Models/ModelsForm/DefineModel/TableSelector/TableSelector.tsx @@ -11,7 +11,7 @@ import { getCatalog } from '@/services/syncs'; import { useStore } from '@/stores'; import Loader from '@/components/Loader'; import ListTables from './ListTables'; -import { Field, getModelPreviewById, putModelById } from '@/services/models'; +import { getModelPreviewById, putModelById } from '@/services/models'; import useCustomToast from '@/hooks/useCustomToast'; import { CustomToastStatus } from '@/components/Toast/index'; import { TableDataType } from '@/components/Table/types'; @@ -120,7 +120,7 @@ const TableSelector = ({ try { const response = await getModelPreviewById(userQuery, connector_id?.toString()); - if ('errors' in response) { + if (response.errors) { if (response.errors) { apiErrorsToast(response.errors); } else { @@ -128,8 +128,20 @@ const TableSelector = ({ } setLoadingPreviewData(false); } else { - setTableData(ConvertModelPreviewToTableData(response as Field[])); - setLoadingPreviewData(false); + if (response.data && response.data.length > 0) { + setTableData(ConvertModelPreviewToTableData(response.data)); + setLoadingPreviewData(false); + } else { + showToast({ + title: 'No data found', + status: CustomToastStatus.Success, + duration: 3000, + isClosable: true, + position: 'bottom-right', + }); + setTableData(null); + setLoadingPreviewData(false); + } } } catch (error) { errorToast('Error fetching preview data', true, null, true); diff --git a/ui/src/views/Models/ModelsList/ModelsList.tsx b/ui/src/views/Models/ModelsList/ModelsList.tsx index 95605fc1..c214ddb3 100644 --- a/ui/src/views/Models/ModelsList/ModelsList.tsx +++ b/ui/src/views/Models/ModelsList/ModelsList.tsx @@ -2,32 +2,34 @@ import ContentContainer from '@/components/ContentContainer'; import TopBar from '@/components/TopBar'; import { Box } from '@chakra-ui/react'; import { FiPlus } from 'react-icons/fi'; -import { Outlet, useNavigate } from 'react-router-dom'; import { useQuery } from '@tanstack/react-query'; -import { getAllModels } from '@/services/models'; +import { getAllModels, GetAllModelsResponse } from '@/services/models'; import Loader from '@/components/Loader'; -import ModelTable from './ModelTable'; -import NoModels from '../NoModels'; +import NoModels from '@/views/Models/NoModels'; +import { useStore } from '@/stores'; +import DataTable from '@/components/DataTable'; +import { Row } from '@tanstack/react-table'; +import { useNavigate } from 'react-router-dom'; +import ModelsListTable from '@/views/Models/ModelsList/ModelsListTable'; const ModelsList = (): JSX.Element | null => { + const activeWorkspaceId = useStore((state) => state.workspaceId); const navigate = useNavigate(); - const handleOnRowClick = (row: any) => { - navigate(row?.id); + const handleOnRowClick = (row: Row) => { + navigate(`/define/models/${row.original.id}`); }; const { data, isLoading } = useQuery({ - queryKey: ['models'], - queryFn: () => getAllModels(), + queryKey: ['models', activeWorkspaceId, 'data'], + queryFn: () => getAllModels({ type: 'data' }), refetchOnMount: true, refetchOnWindowFocus: false, + enabled: activeWorkspaceId > 0, }); - if (isLoading && !data) return ; - if (data?.data?.length === 0) return ; - return ( - + { onCtaClicked={() => navigate('new')} isCtaVisible /> - - {isLoading || !data ? ( - - ) : ( - - )} - - + + {isLoading ? ( + + ) : data?.data && data.data.length > 0 ? ( + + + + ) : ( + + + + )} ); diff --git a/ui/src/views/Models/ModelsList/ModelsListTable/ModelsListTable.tsx b/ui/src/views/Models/ModelsList/ModelsListTable/ModelsListTable.tsx new file mode 100644 index 00000000..eebb22ce --- /dev/null +++ b/ui/src/views/Models/ModelsList/ModelsListTable/ModelsListTable.tsx @@ -0,0 +1,59 @@ +import EntityItem from '@/components/EntityItem'; +import { GetAllModelsResponse } from '@/services/models'; +import { Text } from '@chakra-ui/react'; +import { ColumnDef } from '@tanstack/react-table'; +import dayjs from 'dayjs'; + +const ModelsListTable: ColumnDef[] = [ + { + accessorKey: 'attributes.name', + header: 'Name', + cell: (cell) => { + return ( + + ); + }, + }, + { + accessorKey: 'attributes.query_type', + header: 'Method', + cell: (cell) => { + const queryType = cell.getValue() as string; + switch (queryType) { + case 'raw_sql': + return ( + + SQL Query + + ); + case 'table_selector': + return ( + + Table Selector + + ); + } + return ( + + {queryType} + + ); + }, + }, + { + accessorKey: 'updated_at', + header: 'Last Updated', + cell: (cell) => { + return ( + + {dayjs(cell.getValue() as number).format('D MMM YYYY')} + + ); + }, + }, +]; + +export default ModelsListTable; diff --git a/ui/src/views/Models/ModelsList/ModelsListTable/index.ts b/ui/src/views/Models/ModelsList/ModelsListTable/index.ts new file mode 100644 index 00000000..0263aa26 --- /dev/null +++ b/ui/src/views/Models/ModelsList/ModelsListTable/index.ts @@ -0,0 +1 @@ +export { default } from './ModelsListTable'; From 9aad406f4ff0a391319b5883ccc1bb357430bc5d Mon Sep 17 00:00:00 2001 From: "Rafael E. O'Neill" <106079170+RafaelOAiSquared@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:57:35 -0400 Subject: [PATCH 30/31] refactor(CE): changed Sync Records UX --- ui/src/assets/icons/FiBug.svg | 3 + ui/src/services/syncs.ts | 6 +- .../Syncs/SyncRecords/ErrorLogsModal.tsx | 5 +- .../Activate/Syncs/SyncRecords/FilterTabs.tsx | 24 ++------ .../Syncs/SyncRecords/SyncRecords.tsx | 61 ++++++++++++------- 5 files changed, 52 insertions(+), 47 deletions(-) create mode 100644 ui/src/assets/icons/FiBug.svg diff --git a/ui/src/assets/icons/FiBug.svg b/ui/src/assets/icons/FiBug.svg new file mode 100644 index 00000000..bd7b25a9 --- /dev/null +++ b/ui/src/assets/icons/FiBug.svg @@ -0,0 +1,3 @@ + + + diff --git a/ui/src/services/syncs.ts b/ui/src/services/syncs.ts index 3b6caf08..ad178b37 100644 --- a/ui/src/services/syncs.ts +++ b/ui/src/services/syncs.ts @@ -59,10 +59,14 @@ export const getSyncRecords = ( syncId: string, runId: string, page: string = '1', + isFiltered: boolean = false, + status: string = 'success', ): Promise>> => multiwovenFetch>>({ method: 'get', - url: `/syncs/${syncId}/sync_runs/${runId}/sync_records?page=${page}&per_page=10`, + url: isFiltered + ? `/syncs/${syncId}/sync_runs/${runId}/sync_records?page=${page}&per_page=10&status=${status}` + : `/syncs/${syncId}/sync_runs/${runId}/sync_records?page=${page}&per_page=10`, }); export const editSync = ( diff --git a/ui/src/views/Activate/Syncs/SyncRecords/ErrorLogsModal.tsx b/ui/src/views/Activate/Syncs/SyncRecords/ErrorLogsModal.tsx index addc0f2d..66d39c2f 100644 --- a/ui/src/views/Activate/Syncs/SyncRecords/ErrorLogsModal.tsx +++ b/ui/src/views/Activate/Syncs/SyncRecords/ErrorLogsModal.tsx @@ -12,10 +12,9 @@ import { useDisclosure, Image, ModalHeader, - Icon, } from '@chakra-ui/react'; import { FiCopy, FiArrowRight, FiAlertTriangle } from 'react-icons/fi'; -import { FaBug } from 'react-icons/fa'; +import FiBug from '@/assets/icons/FiBug.svg'; import { useSyncStore } from '@/stores/useSyncStore'; import { SyncRecordStatus } from '../types'; @@ -59,7 +58,7 @@ const ErrorLogsModal = ({ cursor='pointer' data-testid='logs-button' > - + diff --git a/ui/src/views/Activate/Syncs/SyncRecords/FilterTabs.tsx b/ui/src/views/Activate/Syncs/SyncRecords/FilterTabs.tsx index 477a91c7..961390e7 100644 --- a/ui/src/views/Activate/Syncs/SyncRecords/FilterTabs.tsx +++ b/ui/src/views/Activate/Syncs/SyncRecords/FilterTabs.tsx @@ -1,32 +1,16 @@ -import { ApiResponse } from '@/services/common'; -import { SyncRecordResponse, SyncRecordStatus } from '../types'; +import { SyncRecordStatus } from '../types'; import { TabList } from '@chakra-ui/react'; import TabItem from '@/components/TabItem'; type FilterTabsType = { setFilter: (filterValue: SyncRecordStatus) => void; - syncRunRecords: ApiResponse | undefined; }; -export const FilterTabs = ({ setFilter, syncRunRecords }: FilterTabsType) => { +export const FilterTabs = ({ setFilter }: FilterTabsType) => { return ( - setFilter(SyncRecordStatus.success)} - isBadgeVisible - badgeText={syncRunRecords?.data - ?.filter((record) => record.attributes.status === SyncRecordStatus.success) - .length.toString()} - /> - setFilter(SyncRecordStatus.failed)} - isBadgeVisible - badgeText={syncRunRecords?.data - ?.filter((record) => record.attributes.status === SyncRecordStatus.failed) - .length.toString()} - /> + setFilter(SyncRecordStatus.success)} /> + setFilter(SyncRecordStatus.failed)} /> ); }; diff --git a/ui/src/views/Activate/Syncs/SyncRecords/SyncRecords.tsx b/ui/src/views/Activate/Syncs/SyncRecords/SyncRecords.tsx index d04f5b26..5a228751 100644 --- a/ui/src/views/Activate/Syncs/SyncRecords/SyncRecords.tsx +++ b/ui/src/views/Activate/Syncs/SyncRecords/SyncRecords.tsx @@ -28,27 +28,35 @@ const SyncRecords = (): JSX.Element => { const toast = useCustomToast(); const pageId = searchParams.get('page'); - const [currentPage, setCurrentPage] = useState(Number(pageId) || 1); + const statusTab = searchParams.get('status'); - const [currentFilter, setCurrentFilter] = useState(SyncRecordStatus.success); + const [currentPage, setCurrentPage] = useState(Number(pageId) || 1); + const [currentStatusTab, setCurrentStatusTab] = useState( + statusTab === SyncRecordStatus.failed ? SyncRecordStatus.failed : SyncRecordStatus.success, + ); const { - data: syncRunRecords, - isLoading: isSyncRecordsLoading, - isError: isSyncRecordsError, + data: filteredSyncRunRecords, + isLoading: isFilteredSyncRecordsLoading, + isError: isFilteredSyncRecordsError, + refetch: refetchFilteredSyncRecords, } = useQueryWrapper>, Error>( - ['activate', 'sync-records', syncRunId, currentPage], - () => getSyncRecords(syncId as string, syncRunId as string, currentPage.toString()), + ['activate', 'sync-records', syncRunId, currentPage, statusTab], + () => + getSyncRecords( + syncId as string, + syncRunId as string, + currentPage.toString(), + true, + statusTab || 'success', + ), { - refetchOnMount: true, + refetchOnMount: false, refetchOnWindowFocus: false, }, ); - const data = useMemo( - () => syncRunRecords?.data?.filter?.((record) => record.attributes.status === currentFilter), - [syncRunRecords, currentFilter], - ); + const data = filteredSyncRunRecords?.data; const dynamicSyncColumns = useDynamicSyncColumns(data ? data : []); const allColumns = useMemo( @@ -57,11 +65,11 @@ const SyncRecords = (): JSX.Element => { ); useEffect(() => { - setSearchParams({ page: currentPage.toString() }); - }, [currentPage, setSearchParams]); + setSearchParams({ page: currentPage.toString(), status: currentStatusTab }); + }, [currentPage, currentStatusTab, setSearchParams]); useEffect(() => { - if (isSyncRecordsError) { + if (isFilteredSyncRecordsError) { toast({ title: 'Error', description: 'There was an issue fetching the sync records.', @@ -71,20 +79,26 @@ const SyncRecords = (): JSX.Element => { position: 'bottom-right', }); } - }, [isSyncRecordsError, toast]); + }, [isFilteredSyncRecordsError, toast]); const handleNextPage = () => { - if (syncRunRecords?.links?.next) { + if (filteredSyncRunRecords?.links?.next) { setCurrentPage((prevPage) => prevPage + 1); } }; const handlePrevPage = () => { - if (syncRunRecords?.links?.prev) { + if (filteredSyncRunRecords?.links?.prev) { setCurrentPage((prevPage) => Math.max(prevPage - 1, 1)); } }; + const handleStatusTabChange = (status: SyncRecordStatus) => { + setCurrentPage(1); + setCurrentStatusTab(status); + refetchFilteredSyncRecords; + }; + return ( {syncId && syncRunId ? : <>} @@ -92,8 +106,9 @@ const SyncRecords = (): JSX.Element => { size='md' variant='indicator' onChange={(index) => - setCurrentFilter(index === 0 ? SyncRecordStatus.success : SyncRecordStatus.failed) + handleStatusTabChange(index === 0 ? SyncRecordStatus.success : SyncRecordStatus.failed) } + index={currentStatusTab === SyncRecordStatus.success ? 0 : 1} background='gray.300' padding='4px' borderRadius='8px' @@ -103,9 +118,9 @@ const SyncRecords = (): JSX.Element => { width='fit-content' height='fit' > - + - {isSyncRecordsLoading ? ( + {isFilteredSyncRecordsLoading ? ( ) : ( @@ -131,8 +146,8 @@ const SyncRecords = (): JSX.Element => { From 3fba113330ce59c53e6f649c65520f5af6158624 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 00:52:06 +0530 Subject: [PATCH 31/31] refactor(CE): changed setup models to setup define --- ui/src/routes/main.tsx | 6 +++--- .../SetupDefine/SetupDefine.tsx} | 8 ++++---- ui/src/views/Define/SetupDefine/index.ts | 1 + ui/src/views/Models/index.ts | 1 - 4 files changed, 8 insertions(+), 8 deletions(-) rename ui/src/views/{Models/SetupModels.tsx => Define/SetupDefine/SetupDefine.tsx} (70%) create mode 100644 ui/src/views/Define/SetupDefine/index.ts delete mode 100644 ui/src/views/Models/index.ts diff --git a/ui/src/routes/main.tsx b/ui/src/routes/main.tsx index 2292bb12..405beb7b 100644 --- a/ui/src/routes/main.tsx +++ b/ui/src/routes/main.tsx @@ -10,10 +10,10 @@ const VerifyUser = lazy(() => import('@/views/Authentication/VerifyUser')); const ForgotPassword = lazy(() => import('@/views/Authentication/ForgotPassword')); const ResetPassword = lazy(() => import('@/views/Authentication/ResetPassword')); -const Models = lazy(() => import('@/views/Models')); const SetupConnectors = lazy(() => import('@/views/Connectors/SetupConnectors')); - +const SetupDefine = lazy(() => import('@/views/Define/SetupDefine')); const SetupActivate = lazy(() => import('@/views/Activate/SetupActivate')); + const Settings = lazy(() => import('@/views/Settings')); type MAIN_PAGE_ROUTES_ITEM = { @@ -53,7 +53,7 @@ export const MAIN_PAGE_ROUTES: MAIN_PAGE_ROUTES_ITEM[] = [ url: '/define/*', component: ( - + ), }, diff --git a/ui/src/views/Models/SetupModels.tsx b/ui/src/views/Define/SetupDefine/SetupDefine.tsx similarity index 70% rename from ui/src/views/Models/SetupModels.tsx rename to ui/src/views/Define/SetupDefine/SetupDefine.tsx index f9552d6c..67bbd0cf 100644 --- a/ui/src/views/Models/SetupModels.tsx +++ b/ui/src/views/Define/SetupDefine/SetupDefine.tsx @@ -1,8 +1,8 @@ import { Navigate, Route, Routes } from 'react-router-dom'; -import ModelsList from './ModelsList'; -import ModelsForm from './ModelsForm'; -import ViewModel from './ViewModel'; -import EditModel from './EditModel'; +import ModelsList from '@/views/Models/ModelsList'; +import ModelsForm from '@/views/Models/ModelsForm'; +import ViewModel from '@/views/Models/ViewModel'; +import EditModel from '@/views/Models/EditModel'; const SetupModels = (): JSX.Element => { return ( diff --git a/ui/src/views/Define/SetupDefine/index.ts b/ui/src/views/Define/SetupDefine/index.ts new file mode 100644 index 00000000..9aa040e4 --- /dev/null +++ b/ui/src/views/Define/SetupDefine/index.ts @@ -0,0 +1 @@ +export { default } from './SetupDefine'; diff --git a/ui/src/views/Models/index.ts b/ui/src/views/Models/index.ts deleted file mode 100644 index b2b677db..00000000 --- a/ui/src/views/Models/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { default } from './SetupModels';