From 695e01eaebf1c504df12203dabaa27194f0be9e7 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Mon, 27 Sep 2021 11:50:57 -0400 Subject: [PATCH 01/10] update snowflake spec for dyanmic form --- setup.cfg | 2 +- superset/databases/commands/validate.py | 2 +- superset/db_engine_specs/snowflake.py | 86 ++++++++++++++++++++++++- 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9a108f76a480e..20bf9082a28b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ combine_as_imports = true include_trailing_comma = true line_length = 88 known_first_party = superset -known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,graphlib,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,marshmallow_enum,msgpack,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml +known_third_party =alembic,apispec,babel,backoff,bleach,cachelib,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,graphlib,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,marshmallow_enum,msgpack,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml multi_line_output = 3 order_by_type = false diff --git a/superset/databases/commands/validate.py b/superset/databases/commands/validate.py index e2dcc581d7fbf..ac5816248838d 100644 --- a/superset/databases/commands/validate.py +++ b/superset/databases/commands/validate.py @@ -36,7 +36,7 @@ from superset.extensions import event_logger from superset.models.core import Database -BYPASS_VALIDATION_ENGINES = {"bigquery"} +BYPASS_VALIDATION_ENGINES = {"bigquery", "snowflake"} class ValidateDatabaseParametersCommand(BaseCommand): diff --git a/superset/db_engine_specs/snowflake.py b/superset/db_engine_specs/snowflake.py index 6dd85706562bd..fa31c2394ef39 100644 --- a/superset/db_engine_specs/snowflake.py +++ b/superset/db_engine_specs/snowflake.py @@ -17,14 +17,18 @@ import json import re from datetime import datetime -from typing import Any, Dict, Optional, Pattern, Tuple, TYPE_CHECKING +from typing import Any, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING, TypedDict from urllib import parse +from apispec import APISpec +from apispec.ext.marshmallow import MarshmallowPlugin +from babel.core import default_locale from flask_babel import gettext as __ +from marshmallow import fields, Schema from sqlalchemy.engine.url import URL from superset.db_engine_specs.postgres import PostgresBaseEngineSpec -from superset.errors import SupersetErrorType +from superset.errors import SupersetError, SupersetErrorType from superset.models.sql_lab import Query from superset.utils import core as utils @@ -42,12 +46,34 @@ ) +class SnowflakeParametersSchema(Schema): + username = fields.Str(required=True) + password = fields.Str(required=True) + account = fields.Str(required=True) + database = fields.Str(required=True) + role = fields.Str(required=True) + warehouse = fields.Str(required=True) + + +class SnowflakeParametersType(TypedDict): + username: str + password: str + account: str + database: str + role: str + warehouse: str + + class SnowflakeEngineSpec(PostgresBaseEngineSpec): engine = "snowflake" engine_name = "Snowflake" force_column_alias_quotes = True max_column_name_length = 256 + parameters_schema = SnowflakeParametersSchema() + default_driver = "snowflake" + sqlalchemy_uri_placeholder = "snowflake://" + _time_grain_expressions = { None: "{col}", "PT1S": "DATE_TRUNC('SECOND', {col})", @@ -160,3 +186,59 @@ def cancel_query(cls, cursor: Any, query: Query, cancel_query_id: str) -> bool: return False return True + + @classmethod + def build_sqlalchemy_uri( + cls, + parameters: SnowflakeParametersType, + encrypted_extra: Optional[Dict[str, Any]] = None, + ) -> str: + query = parameters.get("query", {}) + query_params = urllib.parse.urlencode(query) + + if not encrypted_extra: + raise ValidationError("Missing service credentials") + + project_id = encrypted_extra.get("credentials_info", {}).get("project_id") + + if project_id: + return f"{cls.default_driver}://{project_id}/?{query_params}" + + raise ValidationError("Invalid service credentials") + + @classmethod + def get_parameters_from_uri( + cls, uri: str, encrypted_extra: Optional[Dict[str, str]] = None + ) -> Any: + value = make_url(uri) + + # Building parameters from encrypted_extra and uri + if encrypted_extra: + return {**encrypted_extra, "query": value.query} + + raise ValidationError("Invalid service credentials") + + @classmethod + def validate_parameters( + cls, parameters: SnowflakeParametersType # pylint: disable=unused-argument + ) -> List[SupersetError]: + return [] + + @classmethod + def parameters_json_schema(cls) -> Any: + """ + Return configuration parameters as OpenAPI. + """ + if not cls.parameters_schema: + return None + + ma_plugin = MarshmallowPlugin() + spec = APISpec( + title="Database Parameters", + version="1.0.0", + openapi_version="3.0.0", + plugins=[ma_plugin], + ) + + spec.components.schema(cls.__name__, schema=cls.parameters_schema) + return spec.to_dict()["components"]["schemas"][cls.__name__] From b07188119a3737772caee2b9d1cc1868de105a0d Mon Sep 17 00:00:00 2001 From: AAfghahi <48933336+AAfghahi@users.noreply.github.com> Date: Mon, 27 Sep 2021 17:41:31 -0400 Subject: [PATCH 02/10] feat(snowflake): Snowflake dynamic form (#16861) * snowflake frontend * snowflake frontend --- .../DatabaseModal/DatabaseConnectionForm.tsx | 71 +++++++++++++++++++ .../src/views/CRUD/data/database/types.ts | 3 + superset/db_engine_specs/snowflake.py | 5 +- 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx index e2a3f0338daae..0fdcd833fd946 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx @@ -51,6 +51,9 @@ export const FormFieldOrder = [ 'catalog', 'query', 'encryption', + 'account', + 'warehouse', + 'role', ]; interface FieldPropTypes { @@ -432,6 +435,71 @@ const queryField = ({ /> ); +const warehouseField = ({ + required, + changeMethods, + getValidation, + validationErrors, + db, +}: FieldPropTypes) => ( + +); + +const roleField = ({ + required, + changeMethods, + getValidation, + validationErrors, + db, +}: FieldPropTypes) => ( + +); + +const accountField = ({ + required, + changeMethods, + getValidation, + validationErrors, + db, +}: FieldPropTypes) => ( + +); + const forceSSLField = ({ isEditMode, changeMethods, @@ -473,6 +541,9 @@ const FORM_FIELD_MAP = { encryption: forceSSLField, credentials_info: CredentialsInfo, catalog: TableCatalog, + warehouse: warehouseField, + role: roleField, + account: accountField, }; const DatabaseConnectionForm = ({ diff --git a/superset-frontend/src/views/CRUD/data/database/types.ts b/superset-frontend/src/views/CRUD/data/database/types.ts index d473ba50aac7c..4687b9e7050e9 100644 --- a/superset-frontend/src/views/CRUD/data/database/types.ts +++ b/superset-frontend/src/views/CRUD/data/database/types.ts @@ -47,6 +47,9 @@ export type DatabaseObject = { credentials_info?: string; query?: Record; catalog?: Record; + warehouse?: string; + role?: string; + account?: string; }; configuration_method: CONFIGURATION_METHOD; engine?: string; diff --git a/superset/db_engine_specs/snowflake.py b/superset/db_engine_specs/snowflake.py index fa31c2394ef39..b33aa6c6f32e3 100644 --- a/superset/db_engine_specs/snowflake.py +++ b/superset/db_engine_specs/snowflake.py @@ -16,6 +16,7 @@ # under the License. import json import re +import urllib from datetime import datetime from typing import Any, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING, TypedDict from urllib import parse @@ -25,7 +26,8 @@ from babel.core import default_locale from flask_babel import gettext as __ from marshmallow import fields, Schema -from sqlalchemy.engine.url import URL +from marshmallow.exceptions import ValidationError +from sqlalchemy.engine.url import make_url, URL from superset.db_engine_specs.postgres import PostgresBaseEngineSpec from superset.errors import SupersetError, SupersetErrorType @@ -211,7 +213,6 @@ def get_parameters_from_uri( cls, uri: str, encrypted_extra: Optional[Dict[str, str]] = None ) -> Any: value = make_url(uri) - # Building parameters from encrypted_extra and uri if encrypted_extra: return {**encrypted_extra, "query": value.query} From 62774d2028ab09ba228369300967293e573d4dc0 Mon Sep 17 00:00:00 2001 From: AAfghahi <48933336+AAfghahi@users.noreply.github.com> Date: Thu, 30 Sep 2021 12:14:31 -0400 Subject: [PATCH 03/10] refactor(DB Connections): Build Snowflake Dynamic Form (#16875) * moved all non-BasicParameters into own field * refactored DB Connection Form made ValidatedInput --- setup.cfg | 2 +- .../DatabaseModal/DatabaseConnectionForm.tsx | 668 ------------------ .../CommonParameters.tsx | 207 ++++++ .../DatabaseConnectionForm/EncryptedField.tsx | 198 ++++++ .../DatabaseConnectionForm/TableCatalog.tsx | 104 +++ .../ValidatedInputField.tsx | 62 ++ .../DatabaseConnectionForm/index.tsx | 172 +++++ superset/datasets/dao.py | 2 +- superset/db_engine_specs/snowflake.py | 56 +- superset/models/core.py | 2 +- 10 files changed, 779 insertions(+), 694 deletions(-) delete mode 100644 superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx create mode 100644 superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm/CommonParameters.tsx create mode 100644 superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm/EncryptedField.tsx create mode 100644 superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm/TableCatalog.tsx create mode 100644 superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm/ValidatedInputField.tsx create mode 100644 superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm/index.tsx diff --git a/setup.cfg b/setup.cfg index 20bf9082a28b7..9a108f76a480e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ combine_as_imports = true include_trailing_comma = true line_length = 88 known_first_party = superset -known_third_party =alembic,apispec,babel,backoff,bleach,cachelib,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,graphlib,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,marshmallow_enum,msgpack,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml +known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,graphlib,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,marshmallow_enum,msgpack,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml multi_line_output = 3 order_by_type = false diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx deleted file mode 100644 index a188126f4a50c..0000000000000 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx +++ /dev/null @@ -1,668 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -import React, { FormEvent, useState } from 'react'; -import { SupersetTheme, JsonObject, t } from '@superset-ui/core'; -import { InputProps } from 'antd/lib/input'; -import { Switch, Select, Button } from 'src/common/components'; -import InfoTooltip from 'src/components/InfoTooltip'; -import ValidatedInput from 'src/components/Form/LabeledErrorBoundInput'; -import FormLabel from 'src/components/Form/FormLabel'; -import { DeleteFilled, CloseOutlined } from '@ant-design/icons'; -import { - formScrollableStyles, - validatedFormStyles, - CredentialInfoForm, - toggleStyle, - infoTooltip, - StyledFooterButton, - StyledCatalogTable, - labelMarginBotton, -} from './styles'; -import { CatalogObject, DatabaseForm, DatabaseObject } from '../types'; - -// These are the columns that are going to be added to encrypted extra, they differ in name based -// on the engine, however we want to use the same component for each of them. Make sure to add the -// the engine specific name here. -export const encryptedCredentialsMap = { - gsheets: 'service_account_info', - bigquery: 'credentials_info', -}; - -enum CredentialInfoOptions { - jsonUpload, - copyPaste, -} - -const castStringToBoolean = (optionValue: string) => optionValue === 'true'; - -export const FormFieldOrder = [ - 'host', - 'port', - 'database', - 'username', - 'password', - 'database_name', - 'credentials_info', - 'service_account_info', - 'catalog', - 'query', - 'encryption', - 'account', - 'warehouse', - 'role', -]; - -interface FieldPropTypes { - required: boolean; - hasTooltip?: boolean; - tooltipText?: (value: any) => string; - onParametersChange: (value: any) => string; - onParametersUploadFileChange: (value: any) => string; - changeMethods: { onParametersChange: (value: any) => string } & { - onChange: (value: any) => string; - } & { - onQueryChange: (value: any) => string; - } & { onParametersUploadFileChange: (value: any) => string } & { - onAddTableCatalog: () => void; - onRemoveTableCatalog: (idx: number) => void; - }; - validationErrors: JsonObject | null; - getValidation: () => void; - db?: DatabaseObject; - isEditMode?: boolean; - sslForced?: boolean; - defaultDBName?: string; - editNewDb?: boolean; -} - -const CredentialsInfo = ({ - changeMethods, - isEditMode, - db, - editNewDb, -}: FieldPropTypes) => { - const [uploadOption, setUploadOption] = useState( - CredentialInfoOptions.jsonUpload.valueOf(), - ); - const [fileToUpload, setFileToUpload] = useState( - null, - ); - const [isPublic, setIsPublic] = useState(true); - const showCredentialsInfo = - db?.engine === 'gsheets' ? !isEditMode && !isPublic : !isEditMode; - // a database that has an optional encrypted field has an encrypted_extra that is an empty object, this checks for that. - const isEncrypted = isEditMode && db?.encrypted_extra !== '{}'; - const encryptedField = db?.engine && encryptedCredentialsMap[db.engine]; - const encryptedValue = - typeof db?.parameters?.[encryptedField] === 'object' - ? JSON.stringify(db?.parameters?.[encryptedField]) - : db?.parameters?.[encryptedField]; - return ( - - {db?.engine === 'gsheets' && ( -
- labelMarginBotton(theme)} - required - > - {t('Type of Google Sheets allowed')} - - -
- )} - {showCredentialsInfo && ( - <> - - {t('How∂ do you want to enter service account credentials?')} - - - - )} - {uploadOption === CredentialInfoOptions.copyPaste || - isEditMode || - editNewDb ? ( -
- {t('Service Account')} -