diff --git a/docs/static/resources/openapi.json b/docs/static/resources/openapi.json index 29d9d7e8a9543..580b87d153015 100644 --- a/docs/static/resources/openapi.json +++ b/docs/static/resources/openapi.json @@ -3557,6 +3557,9 @@ }, "name": { "type": "string" + }, + "engine_information": { + "type": "object" } }, "type": "object" @@ -3738,6 +3741,9 @@ "sqlalchemy_uri": { "maxLength": 1024, "type": "string" + }, + "engine_information": { + "readOnly": true } }, "required": [ @@ -3817,6 +3823,9 @@ "id": { "format": "int32", "type": "integer" + }, + "engine_information": { + "readOnly": true } }, "required": [ @@ -13634,6 +13643,10 @@ "sqlalchemy_uri_placeholder": { "description": "Example placeholder for the SQLAlchemy URI", "type": "string" + }, + "engine_information": { + "description": "Object with properties we want to expose from our DB engine", + "type": "object" } }, "type": "object" diff --git a/requirements/base.txt b/requirements/base.txt index 5984c38d3022c..905f4e1edaa3e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -255,7 +255,7 @@ sqlalchemy-utils==0.38.3 # via # apache-superset # flask-appbuilder -sqlparse==0.3.0 +sqlparse==0.4.3 # via apache-superset tabulate==0.8.9 # via apache-superset diff --git a/setup.py b/setup.py index caca3e3cf8019..2b21f0e292f6c 100644 --- a/setup.py +++ b/setup.py @@ -117,7 +117,7 @@ def get_git_sha() -> str: "slack_sdk>=3.1.1, <4", "sqlalchemy>=1.4, <2", "sqlalchemy-utils>=0.38.3, <0.39", - "sqlparse==0.3.0", # PINNED! see https://github.com/andialbrecht/sqlparse/issues/562 + "sqlparse>=0.4.3, <0.5", "tabulate>=0.8.9, <0.9", # needed to support Literal (3.8) and TypeGuard (3.10) "typing-extensions>=3.10, <4", diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/index.ts b/superset-frontend/packages/superset-ui-chart-controls/src/index.ts index 5ed1768e6983f..66ef14917ab23 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/index.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/index.ts @@ -37,3 +37,4 @@ export { legacySortBy } from './shared-controls/legacySortBy'; export * from './shared-controls/emitFilterControl'; export * from './shared-controls/components'; export * from './types'; +export { xAxisMixin, temporalColumnMixin } from './shared-controls/constants'; diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/constants.tsx b/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/constants.tsx index 91427e14612ec..8de12bfcf6a38 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/constants.tsx +++ b/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/constants.tsx @@ -20,10 +20,16 @@ import { FeatureFlag, isFeatureEnabled, QueryFormData, + QueryResponse, t, validateNonEmpty, } from '@superset-ui/core'; -import { ControlPanelState, ControlState } from '../types'; +import { + BaseControlConfig, + ControlPanelState, + ControlState, + Dataset, +} from '../types'; const getAxisLabel = ( formData: QueryFormData, @@ -32,7 +38,7 @@ const getAxisLabel = ( ? { label: t('Y-axis'), description: t('Dimension to use on y-axis.') } : { label: t('X-axis'), description: t('Dimension to use on x-axis.') }; -export const xAxisControlConfig = { +export const xAxisMixin = { label: (state: ControlPanelState) => getAxisLabel(state?.form_data).label, multi: false, description: (state: ControlPanelState) => @@ -51,3 +57,28 @@ export const xAxisControlConfig = { }, default: undefined, }; + +export const temporalColumnMixin: Pick = { + mapStateToProps: ({ datasource }) => { + if (datasource?.columns[0]?.hasOwnProperty('column_name')) { + const temporalColumns = + (datasource as Dataset)?.columns?.filter(c => c.is_dttm) ?? []; + return { + options: temporalColumns, + default: + (datasource as Dataset)?.main_dttm_col || + temporalColumns[0]?.column_name || + null, + isTemporal: true, + }; + } + const sortedQueryColumns = (datasource as QueryResponse)?.columns?.sort( + query => (query?.is_dttm ? -1 : 1), + ); + return { + options: sortedQueryColumns, + default: sortedQueryColumns[0]?.name || null, + isTemporal: true, + }; + }, +}; diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/dndControls.tsx b/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/dndControls.tsx index 679ac940a5bec..691dbca7c0c82 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/dndControls.tsx +++ b/superset-frontend/packages/superset-ui-chart-controls/src/shared-controls/dndControls.tsx @@ -22,7 +22,6 @@ import { FeatureFlag, isFeatureEnabled, QueryColumn, - QueryResponse, t, validateNonEmpty, } from '@superset-ui/core'; @@ -39,8 +38,9 @@ import { ColumnOption, ColumnMeta, FilterOption, + temporalColumnMixin, } from '..'; -import { xAxisControlConfig } from './constants'; +import { xAxisMixin } from './constants'; type Control = { savedMetrics?: Metric[] | null; @@ -231,6 +231,7 @@ export const dndSecondaryMetricControl: typeof dndAdhocMetricControl = { export const dndGranularitySqlaControl: typeof dndSeriesControl = { ...dndSeriesControl, + ...temporalColumnMixin, label: TIME_FILTER_LABELS.granularity_sqla, description: t( 'The time column for the visualization. Note that you ' + @@ -247,33 +248,11 @@ export const dndGranularitySqlaControl: typeof dndSeriesControl = { optionRenderer: (c: ColumnMeta) => , valueRenderer: (c: ColumnMeta) => , valueKey: 'column_name', - mapStateToProps: ({ datasource }) => { - if (datasource?.columns[0]?.hasOwnProperty('column_name')) { - const temporalColumns = - (datasource as Dataset)?.columns?.filter(c => c.is_dttm) ?? []; - return { - options: temporalColumns, - default: - (datasource as Dataset)?.main_dttm_col || - temporalColumns[0]?.column_name || - null, - isTemporal: true, - }; - } - const sortedQueryColumns = (datasource as QueryResponse)?.columns?.sort( - query => (query?.is_dttm ? -1 : 1), - ); - return { - options: sortedQueryColumns, - default: sortedQueryColumns[0]?.name || null, - isTemporal: true, - }; - }, }; export const dndXAxisControl: typeof dndGroupByControl = { ...dndGroupByControl, - ...xAxisControlConfig, + ...xAxisMixin, }; export function withDndFallback( diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/buildQuery.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/buildQuery.ts index de75b50838ad6..19ad713a26644 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/buildQuery.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/buildQuery.ts @@ -18,7 +18,9 @@ */ import { buildQueryContext, - DTTM_ALIAS, + ensureIsArray, + getXAxis, + isXAxisSet, QueryFormData, } from '@superset-ui/core'; import { @@ -29,25 +31,19 @@ import { } from '@superset-ui/chart-controls'; export default function buildQuery(formData: QueryFormData) { - return buildQueryContext(formData, baseQueryObject => { - const { x_axis } = formData; - const is_timeseries = x_axis === DTTM_ALIAS || !x_axis; - - return [ - { - ...baseQueryObject, - is_timeseries: true, - post_processing: [ - pivotOperator(formData, { - ...baseQueryObject, - index: x_axis, - is_timeseries, - }), - rollingWindowOperator(formData, baseQueryObject), - resampleOperator(formData, baseQueryObject), - flattenOperator(formData, baseQueryObject), - ], - }, - ]; - }); + return buildQueryContext(formData, baseQueryObject => [ + { + ...baseQueryObject, + columns: [ + ...(isXAxisSet(formData) ? ensureIsArray(getXAxis(formData)) : []), + ], + ...(isXAxisSet(formData) ? {} : { is_timeseries: true }), + post_processing: [ + pivotOperator(formData, baseQueryObject), + rollingWindowOperator(formData, baseQueryObject), + resampleOperator(formData, baseQueryObject), + flattenOperator(formData, baseQueryObject), + ], + }, + ]); } diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/controlPanel.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/controlPanel.tsx index d88d105f27eb2..7c46ca6a5f749 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/controlPanel.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/controlPanel.tsx @@ -16,7 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -import { smartDateFormatter, t } from '@superset-ui/core'; +import { + FeatureFlag, + isFeatureEnabled, + smartDateFormatter, + t, +} from '@superset-ui/core'; import { ControlPanelConfig, D3_FORMAT_DOCS, @@ -24,17 +29,27 @@ import { formatSelectOptions, getStandardizedControls, sections, + temporalColumnMixin, } from '@superset-ui/chart-controls'; import React from 'react'; import { headerFontSize, subheaderFontSize } from '../sharedControls'; const config: ControlPanelConfig = { controlPanelSections: [ - sections.legacyTimeseriesTime, + sections.genericTime, { label: t('Query'), expanded: true, - controlSetRows: [['metric'], ['adhoc_filters']], + controlSetRows: [ + [isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) ? 'x_axis' : null], + [ + isFeatureEnabled(FeatureFlag.GENERIC_CHART_AXES) + ? 'time_grain_sqla' + : null, + ], + ['metric'], + ['adhoc_filters'], + ], }, { label: t('Options'), @@ -270,6 +285,10 @@ const config: ControlPanelConfig = { y_axis_format: { label: t('Number format'), }, + x_axis: { + label: t('TEMPORAL X-AXIS'), + ...temporalColumnMixin, + }, }, formDataOverrides: formData => ({ ...formData, diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/transformProps.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/transformProps.ts index 98fe277486460..6a7ffdcbdce1b 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/transformProps.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberWithTrendline/transformProps.ts @@ -17,17 +17,16 @@ * under the License. */ import { - DTTM_ALIAS, extractTimegrain, getNumberFormatter, NumberFormats, - QueryFormData, GenericDataType, getMetricLabel, t, smartDateVerboseFormatter, NumberFormatter, TimeFormatter, + getXAxis, } from '@superset-ui/core'; import { EChartsCoreOption, graphic } from 'echarts'; import { @@ -88,7 +87,7 @@ export default function transformProps( yAxisFormat, timeRangeFixed, } = formData; - const granularity = extractTimegrain(rawFormData as QueryFormData); + const granularity = extractTimegrain(rawFormData); const { data = [], colnames = [], @@ -103,10 +102,11 @@ export default function transformProps( const { r, g, b } = colorPicker; const mainColor = `rgb(${r}, ${g}, ${b})`; + const timeColumn = getXAxis(rawFormData) as string; let trendLineData; let percentChange = 0; let bigNumber = data.length === 0 ? null : data[0][metricName]; - let timestamp = data.length === 0 ? null : data[0][DTTM_ALIAS]; + let timestamp = data.length === 0 ? null : data[0][timeColumn]; let bigNumberFallback; const metricColtypeIndex = colnames.findIndex(name => name === metricName); @@ -115,7 +115,7 @@ export default function transformProps( if (data.length > 0) { const sortedData = (data as BigNumberDatum[]) - .map(d => [d[DTTM_ALIAS], parseMetricValue(d[metricName])]) + .map(d => [d[timeColumn], parseMetricValue(d[metricName])]) // sort in time descending order .sort((a, b) => (a[0] !== null && b[0] !== null ? b[0] - a[0] : 0)); diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/types.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/types.ts index 49f5ea2bfd98f..60c43770e37c4 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/types.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/types.ts @@ -43,7 +43,7 @@ export type BigNumberWithTrendlineFormData = BigNumberTotalFormData & { compareLag?: string | number; }; -export type BigNumberTotalChartProps = ChartProps & { +export type BigNumberTotalChartProps = ChartProps & { formData: BigNumberTotalFormData; queriesData: (ChartDataResponseResult & { data?: BigNumberDatum[]; diff --git a/superset-frontend/plugins/plugin-chart-echarts/test/BigNumber/transformProps.test.ts b/superset-frontend/plugins/plugin-chart-echarts/test/BigNumber/transformProps.test.ts index 6529c2dafabf7..f138765987d5c 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/test/BigNumber/transformProps.test.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/test/BigNumber/transformProps.test.ts @@ -36,7 +36,8 @@ const formData = { a: 1, }, compareLag: 1, - timeGrainSqla: 'P3M' as TimeGranularity, + timeGrainSqla: TimeGranularity.QUARTER, + granularitySqla: 'ds', compareSuffix: 'over last quarter', viz_type: 'big_number', yAxisFormat: '.3s', @@ -44,6 +45,7 @@ const formData = { }; const rawFormData = { + datasource: '1__table', metric: 'value', color_picker: { r: 0, @@ -52,7 +54,8 @@ const rawFormData = { a: 1, }, compare_lag: 1, - time_grain_sqla: 'P3M' as TimeGranularity, + time_grain_sqla: TimeGranularity.QUARTER, + granularity_sqla: 'ds', compare_suffix: 'over last quarter', viz_type: 'big_number', y_axis_format: '.3s', diff --git a/superset-frontend/src/dashboard/components/PropertiesModal/PropertiesModal.test.tsx b/superset-frontend/src/dashboard/components/PropertiesModal/PropertiesModal.test.tsx index bffe9f3de6d6d..8b09a57d54370 100644 --- a/superset-frontend/src/dashboard/components/PropertiesModal/PropertiesModal.test.tsx +++ b/superset-frontend/src/dashboard/components/PropertiesModal/PropertiesModal.test.tsx @@ -38,7 +38,7 @@ spyColorSchemeControlWrapper.mockImplementation( ); fetchMock.get( - 'http://localhost/api/v1/dashboard/related/roles?q=(filter:%27%27)', + 'http://localhost/api/v1/dashboard/related/roles?q=(filter:%27%27,page:0,page_size:100)', { body: { count: 6, @@ -46,26 +46,32 @@ fetchMock.get( { text: 'Admin', value: 1, + extra: {}, }, { text: 'Alpha', value: 3, + extra: {}, }, { text: 'Gamma', value: 4, + extra: {}, }, { text: 'granter', value: 5, + extra: {}, }, { text: 'Public', value: 2, + extra: {}, }, { text: 'sql_lab', value: 6, + extra: {}, }, ], }, @@ -73,7 +79,7 @@ fetchMock.get( ); fetchMock.get( - 'http://localhost/api/v1/dashboard/related/owners?q=(filter:%27%27)', + 'http://localhost/api/v1/dashboard/related/owners?q=(filter:%27%27,page:0,page_size:100)', { body: { count: 1, @@ -81,45 +87,53 @@ fetchMock.get( { text: 'Superset Admin', value: 1, + extra: { active: true }, + }, + { + text: 'Inactive Admin', + value: 2, + extra: { active: false }, }, ], }, }, ); +const dashboardInfo = { + certified_by: 'John Doe', + certification_details: 'Sample certification', + changed_by: null, + changed_by_name: '', + changed_by_url: '', + changed_on: '2021-03-30T19:30:14.020942', + charts: [ + 'Vaccine Candidates per Country & Stage', + 'Vaccine Candidates per Country', + 'Vaccine Candidates per Country', + 'Vaccine Candidates per Approach & Stage', + 'Vaccine Candidates per Phase', + 'Vaccine Candidates per Phase', + 'Vaccine Candidates per Country & Stage', + 'Filtering Vaccines', + ], + css: '', + dashboard_title: 'COVID Vaccine Dashboard', + id: 26, + metadata: mockedJsonMetadata, + owners: [], + position_json: + '{"CHART-63bEuxjDMJ": {"children": [], "id": "CHART-63bEuxjDMJ", "meta": {"chartId": 369, "height": 76, "sliceName": "Vaccine Candidates per Country", "sliceNameOverride": "Map of Vaccine Candidates", "uuid": "ddc91df6-fb40-4826-bdca-16b85af1c024", "width": 7}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-zvw7luvEL"], "type": "CHART"}, "CHART-F-fkth0Dnv": {"children": [], "id": "CHART-F-fkth0Dnv", "meta": {"chartId": 314, "height": 76, "sliceName": "Vaccine Candidates per Country", "sliceNameOverride": "Treemap of Vaccine Candidates per Country", "uuid": "e2f5a8a7-feb0-4f79-bc6b-01fe55b98b3c", "width": 5}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-zvw7luvEL"], "type": "CHART"}, "CHART-RjD_ygqtwH": {"children": [], "id": "CHART-RjD_ygqtwH", "meta": {"chartId": 351, "height": 59, "sliceName": "Vaccine Candidates per Phase", "sliceNameOverride": "Vaccine Candidates per Phase", "uuid": "30b73c65-85e7-455f-bb24-801bb0cdc670", "width": 2}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-xSeNAspgw"], "type": "CHART"}, "CHART-aGfmWtliqA": {"children": [], "id": "CHART-aGfmWtliqA", "meta": {"chartId": 312, "height": 59, "sliceName": "Vaccine Candidates per Phase", "uuid": "392f293e-0892-4316-bd41-c927b65606a4", "width": 4}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-xSeNAspgw"], "type": "CHART"}, "CHART-dCUpAcPsji": {"children": [], "id": "CHART-dCUpAcPsji", "meta": {"chartId": 325, "height": 82, "sliceName": "Vaccine Candidates per Country & Stage", "sliceNameOverride": "Heatmap of Countries & Clinical Stages", "uuid": "cd111331-d286-4258-9020-c7949a109ed2", "width": 4}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-zhOlQLQnB"], "type": "CHART"}, "CHART-eirDduqb1A": {"children": [], "id": "CHART-eirDduqb1A", "meta": {"chartId": 358, "height": 59, "sliceName": "Filtering Vaccines", "sliceNameOverride": "Filter Box of Vaccines", "uuid": "c29381ce-0e99-4cf3-bf0f-5f55d6b94176", "width": 3}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-xSeNAspgw"], "type": "CHART"}, "CHART-fYo7IyvKZQ": {"children": [], "id": "CHART-fYo7IyvKZQ", "meta": {"chartId": 371, "height": 82, "sliceName": "Vaccine Candidates per Country & Stage", "sliceNameOverride": "Sunburst of Country & Clinical Stages", "uuid": "f69c556f-15fe-4a82-a8bb-69d5b6954123", "width": 5}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-zhOlQLQnB"], "type": "CHART"}, "CHART-j4hUvP5dDD": {"children": [], "id": "CHART-j4hUvP5dDD", "meta": {"chartId": 364, "height": 82, "sliceName": "Vaccine Candidates per Approach & Stage", "sliceNameOverride": "Heatmap of Aproaches & Clinical Stages", "uuid": "0c953c84-0c9a-418d-be9f-2894d2a2cee0", "width": 3}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-zhOlQLQnB"], "type": "CHART"}, "DASHBOARD_VERSION_KEY": "v2", "GRID_ID": {"children": [], "id": "GRID_ID", "parents": ["ROOT_ID"], "type": "GRID"}, "HEADER_ID": {"id": "HEADER_ID", "meta": {"text": "COVID Vaccine Dashboard"}, "type": "HEADER"}, "MARKDOWN-VjQQ5SFj5v": {"children": [], "id": "MARKDOWN-VjQQ5SFj5v", "meta": {"code": "# COVID-19 Vaccine Dashboard\\n\\nEverywhere you look, you see negative news about COVID-19. This is to be expected; it\'s been a brutal year and this disease is no joke. This dashboard hopes to use visualization to inject some optimism about the coming return to normalcy we enjoyed before 2020! There\'s lots to be optimistic about:\\n\\n- the sheer volume of attempts to fund the R&D needed to produce and bring an effective vaccine to market\\n- the large number of countries involved in at least one vaccine candidate (and the diversity of economic status of these countries)\\n- the diversity of vaccine approaches taken\\n- the fact that 2 vaccines have already been approved (and a hundreds of thousands of patients have already been vaccinated)\\n\\n### The Dataset\\n\\nThis dashboard is powered by data maintained by the Millken Institute ([link to dataset](https://airtable.com/shrSAi6t5WFwqo3GM/tblEzPQS5fnc0FHYR/viwDBH7b6FjmIBX5x?blocks=bipZFzhJ7wHPv7x9z)). We researched each vaccine candidate and added our own best guesses for the country responsible for each vaccine effort.\\n\\n_Note that this dataset was last updated on 12/23/2020_.\\n\\n", "height": 59, "width": 3}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-xSeNAspgw"], "type": "MARKDOWN"}, "ROOT_ID": {"children": ["TABS-wUKya7eQ0Z"], "id": "ROOT_ID", "type": "ROOT"}, "ROW-xSeNAspgw": {"children": ["MARKDOWN-VjQQ5SFj5v", "CHART-aGfmWtliqA", "CHART-RjD_ygqtwH", "CHART-eirDduqb1A"], "id": "ROW-xSeNAspgw", "meta": {"0": "ROOT_ID", "background": "BACKGROUND_TRANSPARENT"}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ"], "type": "ROW"}, "ROW-zhOlQLQnB": {"children": ["CHART-j4hUvP5dDD", "CHART-dCUpAcPsji", "CHART-fYo7IyvKZQ"], "id": "ROW-zhOlQLQnB", "meta": {"0": "ROOT_ID", "background": "BACKGROUND_TRANSPARENT"}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ"], "type": "ROW"}, "ROW-zvw7luvEL": {"children": ["CHART-63bEuxjDMJ", "CHART-F-fkth0Dnv"], "id": "ROW-zvw7luvEL", "meta": {"0": "ROOT_ID", "background": "BACKGROUND_TRANSPARENT"}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ"], "type": "ROW"}, "TAB-BCIJF4NvgQ": {"children": ["ROW-xSeNAspgw", "ROW-zvw7luvEL", "ROW-zhOlQLQnB"], "id": "TAB-BCIJF4NvgQ", "meta": {"text": "Overview"}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z"], "type": "TAB"}, "TABS-wUKya7eQ0Z": {"children": ["TAB-BCIJF4NvgQ"], "id": "TABS-wUKya7eQ0Z", "meta": {}, "parents": ["ROOT_ID"], "type": "TABS"}}', + published: false, + roles: [], + slug: null, + thumbnail_url: + '/api/v1/dashboard/26/thumbnail/b24805e98d90116da8c0974d24f5c533/', + url: '/superset/dashboard/26/', +}; + fetchMock.get('glob:*/api/v1/dashboard/26', { body: { - result: { - certified_by: 'John Doe', - certification_details: 'Sample certification', - changed_by: null, - changed_by_name: '', - changed_by_url: '', - changed_on: '2021-03-30T19:30:14.020942', - charts: [ - 'Vaccine Candidates per Country & Stage', - 'Vaccine Candidates per Country', - 'Vaccine Candidates per Country', - 'Vaccine Candidates per Approach & Stage', - 'Vaccine Candidates per Phase', - 'Vaccine Candidates per Phase', - 'Vaccine Candidates per Country & Stage', - 'Filtering Vaccines', - ], - css: '', - dashboard_title: 'COVID Vaccine Dashboard', - id: 26, - json_metadata: mockedJsonMetadata, - owners: [], - position_json: - '{"CHART-63bEuxjDMJ": {"children": [], "id": "CHART-63bEuxjDMJ", "meta": {"chartId": 369, "height": 76, "sliceName": "Vaccine Candidates per Country", "sliceNameOverride": "Map of Vaccine Candidates", "uuid": "ddc91df6-fb40-4826-bdca-16b85af1c024", "width": 7}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-zvw7luvEL"], "type": "CHART"}, "CHART-F-fkth0Dnv": {"children": [], "id": "CHART-F-fkth0Dnv", "meta": {"chartId": 314, "height": 76, "sliceName": "Vaccine Candidates per Country", "sliceNameOverride": "Treemap of Vaccine Candidates per Country", "uuid": "e2f5a8a7-feb0-4f79-bc6b-01fe55b98b3c", "width": 5}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-zvw7luvEL"], "type": "CHART"}, "CHART-RjD_ygqtwH": {"children": [], "id": "CHART-RjD_ygqtwH", "meta": {"chartId": 351, "height": 59, "sliceName": "Vaccine Candidates per Phase", "sliceNameOverride": "Vaccine Candidates per Phase", "uuid": "30b73c65-85e7-455f-bb24-801bb0cdc670", "width": 2}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-xSeNAspgw"], "type": "CHART"}, "CHART-aGfmWtliqA": {"children": [], "id": "CHART-aGfmWtliqA", "meta": {"chartId": 312, "height": 59, "sliceName": "Vaccine Candidates per Phase", "uuid": "392f293e-0892-4316-bd41-c927b65606a4", "width": 4}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-xSeNAspgw"], "type": "CHART"}, "CHART-dCUpAcPsji": {"children": [], "id": "CHART-dCUpAcPsji", "meta": {"chartId": 325, "height": 82, "sliceName": "Vaccine Candidates per Country & Stage", "sliceNameOverride": "Heatmap of Countries & Clinical Stages", "uuid": "cd111331-d286-4258-9020-c7949a109ed2", "width": 4}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-zhOlQLQnB"], "type": "CHART"}, "CHART-eirDduqb1A": {"children": [], "id": "CHART-eirDduqb1A", "meta": {"chartId": 358, "height": 59, "sliceName": "Filtering Vaccines", "sliceNameOverride": "Filter Box of Vaccines", "uuid": "c29381ce-0e99-4cf3-bf0f-5f55d6b94176", "width": 3}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-xSeNAspgw"], "type": "CHART"}, "CHART-fYo7IyvKZQ": {"children": [], "id": "CHART-fYo7IyvKZQ", "meta": {"chartId": 371, "height": 82, "sliceName": "Vaccine Candidates per Country & Stage", "sliceNameOverride": "Sunburst of Country & Clinical Stages", "uuid": "f69c556f-15fe-4a82-a8bb-69d5b6954123", "width": 5}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-zhOlQLQnB"], "type": "CHART"}, "CHART-j4hUvP5dDD": {"children": [], "id": "CHART-j4hUvP5dDD", "meta": {"chartId": 364, "height": 82, "sliceName": "Vaccine Candidates per Approach & Stage", "sliceNameOverride": "Heatmap of Aproaches & Clinical Stages", "uuid": "0c953c84-0c9a-418d-be9f-2894d2a2cee0", "width": 3}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-zhOlQLQnB"], "type": "CHART"}, "DASHBOARD_VERSION_KEY": "v2", "GRID_ID": {"children": [], "id": "GRID_ID", "parents": ["ROOT_ID"], "type": "GRID"}, "HEADER_ID": {"id": "HEADER_ID", "meta": {"text": "COVID Vaccine Dashboard"}, "type": "HEADER"}, "MARKDOWN-VjQQ5SFj5v": {"children": [], "id": "MARKDOWN-VjQQ5SFj5v", "meta": {"code": "# COVID-19 Vaccine Dashboard\\n\\nEverywhere you look, you see negative news about COVID-19. This is to be expected; it\'s been a brutal year and this disease is no joke. This dashboard hopes to use visualization to inject some optimism about the coming return to normalcy we enjoyed before 2020! There\'s lots to be optimistic about:\\n\\n- the sheer volume of attempts to fund the R&D needed to produce and bring an effective vaccine to market\\n- the large number of countries involved in at least one vaccine candidate (and the diversity of economic status of these countries)\\n- the diversity of vaccine approaches taken\\n- the fact that 2 vaccines have already been approved (and a hundreds of thousands of patients have already been vaccinated)\\n\\n### The Dataset\\n\\nThis dashboard is powered by data maintained by the Millken Institute ([link to dataset](https://airtable.com/shrSAi6t5WFwqo3GM/tblEzPQS5fnc0FHYR/viwDBH7b6FjmIBX5x?blocks=bipZFzhJ7wHPv7x9z)). We researched each vaccine candidate and added our own best guesses for the country responsible for each vaccine effort.\\n\\n_Note that this dataset was last updated on 12/23/2020_.\\n\\n", "height": 59, "width": 3}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ", "ROW-xSeNAspgw"], "type": "MARKDOWN"}, "ROOT_ID": {"children": ["TABS-wUKya7eQ0Z"], "id": "ROOT_ID", "type": "ROOT"}, "ROW-xSeNAspgw": {"children": ["MARKDOWN-VjQQ5SFj5v", "CHART-aGfmWtliqA", "CHART-RjD_ygqtwH", "CHART-eirDduqb1A"], "id": "ROW-xSeNAspgw", "meta": {"0": "ROOT_ID", "background": "BACKGROUND_TRANSPARENT"}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ"], "type": "ROW"}, "ROW-zhOlQLQnB": {"children": ["CHART-j4hUvP5dDD", "CHART-dCUpAcPsji", "CHART-fYo7IyvKZQ"], "id": "ROW-zhOlQLQnB", "meta": {"0": "ROOT_ID", "background": "BACKGROUND_TRANSPARENT"}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ"], "type": "ROW"}, "ROW-zvw7luvEL": {"children": ["CHART-63bEuxjDMJ", "CHART-F-fkth0Dnv"], "id": "ROW-zvw7luvEL", "meta": {"0": "ROOT_ID", "background": "BACKGROUND_TRANSPARENT"}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z", "TAB-BCIJF4NvgQ"], "type": "ROW"}, "TAB-BCIJF4NvgQ": {"children": ["ROW-xSeNAspgw", "ROW-zvw7luvEL", "ROW-zhOlQLQnB"], "id": "TAB-BCIJF4NvgQ", "meta": {"text": "Overview"}, "parents": ["ROOT_ID", "TABS-wUKya7eQ0Z"], "type": "TAB"}, "TABS-wUKya7eQ0Z": {"children": ["TAB-BCIJF4NvgQ"], "id": "TABS-wUKya7eQ0Z", "meta": {}, "parents": ["ROOT_ID"], "type": "TABS"}}', - published: false, - roles: [], - slug: null, - thumbnail_url: - '/api/v1/dashboard/26/thumbnail/b24805e98d90116da8c0974d24f5c533/', - url: '/superset/dashboard/26/', - }, + result: { ...dashboardInfo, json_metadata: mockedJsonMetadata }, }, }); @@ -347,3 +361,102 @@ test('Empty "Certified by" should clear "Certification details"', async () => { screen.getByRole('textbox', { name: 'Certification details' }), ).toHaveValue(''); }); + +test('should show all roles', async () => { + spyIsFeatureEnabled.mockReturnValue(true); + + const props = createProps(); + const propsWithDashboardIndo = { ...props, dashboardInfo }; + + const open = () => waitFor(() => userEvent.click(getSelect())); + const getSelect = () => + screen.getByRole('combobox', { name: SupersetCore.t('Roles') }); + + const getElementsByClassName = (className: string) => + document.querySelectorAll(className)! as NodeListOf; + + const findAllSelectOptions = () => + waitFor(() => getElementsByClassName('.ant-select-item-option-content')); + + render(, { + useRedux: true, + }); + + expect(screen.getAllByRole('combobox')).toHaveLength(2); + expect( + screen.getByRole('combobox', { name: SupersetCore.t('Roles') }), + ).toBeInTheDocument(); + + await open(); + + const options = await findAllSelectOptions(); + + expect(options).toHaveLength(6); + expect(options[0]).toHaveTextContent('Admin'); +}); + +test('should show active owners with dashboard rbac', async () => { + spyIsFeatureEnabled.mockReturnValue(true); + + const props = createProps(); + const propsWithDashboardIndo = { ...props, dashboardInfo }; + + const open = () => waitFor(() => userEvent.click(getSelect())); + const getSelect = () => + screen.getByRole('combobox', { name: SupersetCore.t('Owners') }); + + const getElementsByClassName = (className: string) => + document.querySelectorAll(className)! as NodeListOf; + + const findAllSelectOptions = () => + waitFor(() => getElementsByClassName('.ant-select-item-option-content')); + + render(, { + useRedux: true, + }); + + expect(screen.getAllByRole('combobox')).toHaveLength(2); + expect( + screen.getByRole('combobox', { name: SupersetCore.t('Owners') }), + ).toBeInTheDocument(); + + await open(); + + const options = await findAllSelectOptions(); + + expect(options).toHaveLength(1); + expect(options[0]).toHaveTextContent('Superset Admin'); +}); + +test('should show active owners without dashboard rbac', async () => { + spyIsFeatureEnabled.mockReturnValue(false); + + const props = createProps(); + const propsWithDashboardIndo = { ...props, dashboardInfo }; + + const open = () => waitFor(() => userEvent.click(getSelect())); + const getSelect = () => + screen.getByRole('combobox', { name: SupersetCore.t('Owners') }); + + const getElementsByClassName = (className: string) => + document.querySelectorAll(className)! as NodeListOf; + + const findAllSelectOptions = () => + waitFor(() => getElementsByClassName('.ant-select-item-option-content')); + + render(, { + useRedux: true, + }); + + expect(screen.getByRole('combobox')).toBeInTheDocument(); + expect( + screen.getByRole('combobox', { name: SupersetCore.t('Owners') }), + ).toBeInTheDocument(); + + await open(); + + const options = await findAllSelectOptions(); + + expect(options).toHaveLength(1); + expect(options[0]).toHaveTextContent('Superset Admin'); +}); diff --git a/superset-frontend/src/dashboard/components/PropertiesModal/index.tsx b/superset-frontend/src/dashboard/components/PropertiesModal/index.tsx index f015612d2aeed..302931a38c8b7 100644 --- a/superset-frontend/src/dashboard/components/PropertiesModal/index.tsx +++ b/superset-frontend/src/dashboard/components/PropertiesModal/index.tsx @@ -133,7 +133,9 @@ const PropertiesModal = ({ endpoint: `/api/v1/dashboard/related/${accessType}?q=${query}`, }).then(response => ({ data: response.json.result - .filter((item: { extra: { active: boolean } }) => item.extra.active) + .filter((item: { extra: { active: boolean } }) => + item.extra.active !== undefined ? item.extra.active : true, + ) .map((item: { value: number; text: string }) => ({ value: item.value, label: item.text, diff --git a/superset-frontend/src/explore/controlUtils/getControlValuesCompatibleWithDatasource.ts b/superset-frontend/src/explore/controlUtils/getControlValuesCompatibleWithDatasource.ts index 346768557c9dd..98590e63ee49b 100644 --- a/superset-frontend/src/explore/controlUtils/getControlValuesCompatibleWithDatasource.ts +++ b/superset-frontend/src/explore/controlUtils/getControlValuesCompatibleWithDatasource.ts @@ -36,11 +36,12 @@ const isControlValueCompatibleWithDatasource = ( ) => { if (controlState.options && typeof value === 'string') { if ( - (Array.isArray(controlState.options) && - controlState.options.some( - (option: [string | number, string]) => option[0] === value, - )) || - value in controlState.options + controlState.options.some( + (option: [string | number, string] | { column_name: string }) => + Array.isArray(option) + ? option[0] === value + : option.column_name === value, + ) ) { return datasource.columns.some(column => column.column_name === value); } diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseList.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseList.tsx index f217a60ec450d..666c6e24101dd 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseList.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseList.tsx @@ -229,7 +229,13 @@ function DatabaseList({ addDangerToast, addSuccessToast }: DatabaseListProps) { SupersetClient.get({ endpoint: `/api/v1/database/?q=${rison.encode(payload)}`, }).then(({ json }: Record) => { - setAllowUploads(json.count >= 1); + // There might be some existings Gsheets and Clickhouse DBs + // with allow_file_upload set as True which is not possible from now on + const allowedDatabasesWithFileUpload = + json?.result?.filter( + (database: any) => database?.engine_information?.supports_file_upload, + ) || []; + setAllowUploads(allowedDatabasesWithFileUpload?.length >= 1); }); }; diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/ExtraOptions.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/ExtraOptions.tsx index 45504b3d5ee71..349264d6819fd 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/ExtraOptions.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/ExtraOptions.tsx @@ -48,6 +48,8 @@ const ExtraOptions = ({ }) => { const expandableModalIsOpen = !!db?.expose_in_sqllab; const createAsOpen = !!(db?.allow_ctas || db?.allow_cvas); + const isFileUploadSupportedByEngine = + db?.engine_information?.supports_file_upload; return ( - -
- {t('Schemas allowed for CSV upload')} -
-
- -
-
- {t( - 'A comma-separated list of schemas that CSVs are allowed to upload to.', - )} -
- - +
- -
- - +
+ +
+ + )} + {isFileUploadSupportedByEngine && !!db?.allow_file_upload && ( + +
+ {t('Schemas allowed for File upload')} +
+
+ +
+
+ {t( + 'A comma-separated list of schemas that files are allowed to upload to.', )} - /> -
- +
+ + )} { const securityTab = screen.getByRole('tab', { name: /right security add extra connection information\./i, }); + const allowFileUploadCheckbox = screen.getByRole('checkbox', { + name: /Allow file uploads to database/i, + }); + const allowFileUploadText = screen.getByText( + /Allow file uploads to database/i, + ); + + const schemasForFileUploadText = screen.queryByText( + /Schemas allowed for File upload/i, + ); + + const visibleComponents = [ + closeButton, + advancedHeader, + basicHelper, + basicHeaderTitle, + basicHeaderSubtitle, + basicHeaderLink, + basicTab, + advancedTab, + sqlLabTab, + performanceTab, + securityTab, + allowFileUploadText, + ]; + // These components exist in the DOM but are not visible + const invisibleComponents = [allowFileUploadCheckbox]; // ---------- Assertions ---------- + visibleComponents.forEach(component => { + expect(component).toBeVisible(); + }); + invisibleComponents.forEach(component => { + expect(component).not.toBeVisible(); + }); + expect(schemasForFileUploadText).not.toBeInTheDocument(); + }); + + it('renders the "Advanced" - SECURITY tab correctly after selecting Allow file uploads', async () => { + // ---------- Components ---------- + // On step 1, click dbButton to access step 2 + userEvent.click( + screen.getByRole('button', { + name: /sqlite/i, + }), + ); + // Click the "Advanced" tab + userEvent.click(screen.getByRole('tab', { name: /advanced/i })); + // Click the "Security" tab + userEvent.click( + screen.getByRole('tab', { + name: /right security add extra connection information\./i, + }), + ); + // Click the "Allow file uploads" tab + + const allowFileUploadCheckbox = screen.getByRole('checkbox', { + name: /Allow file uploads to database/i, + }); + userEvent.click(allowFileUploadCheckbox); + + // ----- BEGIN STEP 2 (ADVANCED - SECURITY) + // - AntD header + const closeButton = screen.getByRole('button', { name: /close/i }); + const advancedHeader = screen.getByRole('heading', { + name: /connect a database/i, + }); + // - Connection header + const basicHelper = screen.getByText(/step 2 of 2/i); + const basicHeaderTitle = screen.getByText(/enter primary credentials/i); + const basicHeaderSubtitle = screen.getByText( + /need help\? learn how to connect your database \./i, + ); + const basicHeaderLink = within(basicHeaderSubtitle).getByRole('link', { + name: /here/i, + }); + // - Basic/Advanced tabs + const basicTab = screen.getByRole('tab', { name: /basic/i }); + const advancedTab = screen.getByRole('tab', { name: /advanced/i }); + // - Advanced tabs + const sqlLabTab = screen.getByRole('tab', { + name: /right sql lab adjust how this database will interact with sql lab\./i, + }); + const performanceTab = screen.getByRole('tab', { + name: /right performance adjust performance settings of this database\./i, + }); + const securityTab = screen.getByRole('tab', { + name: /right security add extra connection information\./i, + }); + const allowFileUploadText = screen.getByText( + /Allow file uploads to database/i, + ); + + const schemasForFileUploadText = screen.queryByText( + /Schemas allowed for File upload/i, + ); + const visibleComponents = [ closeButton, advancedHeader, @@ -775,11 +898,19 @@ describe('DatabaseModal', () => { sqlLabTab, performanceTab, securityTab, + allowFileUploadText, ]; + // These components exist in the DOM but are not visible + const invisibleComponents = [allowFileUploadCheckbox]; + // ---------- Assertions ---------- visibleComponents.forEach(component => { expect(component).toBeVisible(); }); + invisibleComponents.forEach(component => { + expect(component).not.toBeVisible(); + }); + expect(schemasForFileUploadText).toBeInTheDocument(); }); test('renders the "Advanced" - OTHER tab correctly', async () => { @@ -1072,4 +1203,70 @@ describe('DatabaseModal', () => { expect(step2of3text).toBeVisible(); }); }); + + describe('DatabaseModal w/ GSheet Engine', () => { + const renderAndWait = async () => { + const dbProps = { + show: true, + database_name: 'my database', + sqlalchemy_uri: 'gsheets://', + }; + const mounted = act(async () => { + render(, { + useRedux: true, + }); + }); + + return mounted; + }; + + beforeEach(async () => { + await renderAndWait(); + }); + + it('enters step 2 of 2 when proper database is selected', () => { + const step2of2text = screen.getByText(/step 2 of 2/i); + expect(step2of2text).toBeVisible(); + }); + + it('renders the "Advanced" - SECURITY tab without Allow File Upload Checkbox', async () => { + // Click the "Advanced" tab + userEvent.click(screen.getByRole('tab', { name: /advanced/i })); + // Click the "Security" tab + userEvent.click( + screen.getByRole('tab', { + name: /right security add extra connection information\./i, + }), + ); + + // ----- BEGIN STEP 2 (ADVANCED - SECURITY) + // - Advanced tabs + const impersonateLoggerUserCheckbox = screen.getByRole('checkbox', { + name: /impersonate logged in/i, + }); + const impersonateLoggerUserText = screen.getByText( + /impersonate logged in/i, + ); + const allowFileUploadText = screen.queryByText( + /Allow file uploads to database/i, + ); + const schemasForFileUploadText = screen.queryByText( + /Schemas allowed for File upload/i, + ); + + const visibleComponents = [impersonateLoggerUserText]; + // These components exist in the DOM but are not visible + const invisibleComponents = [impersonateLoggerUserCheckbox]; + + // ---------- Assertions ---------- + visibleComponents.forEach(component => { + expect(component).toBeVisible(); + }); + invisibleComponents.forEach(component => { + expect(component).not.toBeVisible(); + }); + expect(allowFileUploadText).not.toBeInTheDocument(); + expect(schemasForFileUploadText).not.toBeInTheDocument(); + }); + }); }); diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx index d33a5cb521f9a..9fb55246ce510 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx @@ -175,6 +175,7 @@ type DBReducerActionType = database_name?: string; engine?: string; configuration_method: CONFIGURATION_METHOD; + engine_information?: {}; }; } | { @@ -718,7 +719,7 @@ const DatabaseModal: FunctionComponent = ({ const selectedDbModel = availableDbs?.databases.filter( (db: DatabaseObject) => db.name === database_name, )[0]; - const { engine, parameters } = selectedDbModel; + const { engine, parameters, engine_information } = selectedDbModel; const isDynamic = parameters !== undefined; setDB({ type: ActionType.dbSelected, @@ -728,6 +729,7 @@ const DatabaseModal: FunctionComponent = ({ configuration_method: isDynamic ? CONFIGURATION_METHOD.DYNAMIC_FORM : CONFIGURATION_METHOD.SQLALCHEMY_URI, + engine_information, }, }); } diff --git a/superset-frontend/src/views/CRUD/data/database/types.ts b/superset-frontend/src/views/CRUD/data/database/types.ts index 92dd8e187851b..9a9386035eebc 100644 --- a/superset-frontend/src/views/CRUD/data/database/types.ts +++ b/superset-frontend/src/views/CRUD/data/database/types.ts @@ -101,6 +101,11 @@ export type DatabaseObject = { catalog?: Array; query_input?: string; extra?: string; + + // DB Engine Spec information + engine_information?: { + supports_file_upload?: boolean; + }; }; export type DatabaseForm = { diff --git a/superset-frontend/src/views/components/RightMenu.test.tsx b/superset-frontend/src/views/components/RightMenu.test.tsx new file mode 100644 index 0000000000000..f5ca82845d3c5 --- /dev/null +++ b/superset-frontend/src/views/components/RightMenu.test.tsx @@ -0,0 +1,268 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import * as reactRedux from 'react-redux'; +import fetchMock from 'fetch-mock'; +import waitForComponentToPaint from 'spec/helpers/waitForComponentToPaint'; +import { styledMount as mount } from 'spec/helpers/theming'; +import RightMenu from './RightMenu'; +import { RightMenuProps } from './types'; + +jest.mock('react-redux', () => ({ + ...jest.requireActual('react-redux'), + useSelector: jest.fn(), +})); + +const createProps = (): RightMenuProps => ({ + align: 'flex-end', + navbarRight: { + show_watermark: false, + bug_report_url: '/report/', + documentation_url: '/docs/', + languages: { + en: { + flag: 'us', + name: 'English', + url: '/lang/en', + }, + it: { + flag: 'it', + name: 'Italian', + url: '/lang/it', + }, + }, + show_language_picker: true, + user_is_anonymous: true, + user_info_url: '/users/userinfo/', + user_logout_url: '/logout/', + user_login_url: '/login/', + user_profile_url: '/profile/', + locale: 'en', + version_string: '1.0.0', + version_sha: 'randomSHA', + build_number: 'randomBuildNumber', + }, + settings: [ + { + name: 'Security', + icon: 'fa-cogs', + label: 'Security', + index: 1, + childs: [ + { + name: 'List Users', + icon: 'fa-user', + label: 'List Users', + url: '/users/list/', + index: 1, + }, + ], + }, + ], + isFrontendRoute: () => true, + environmentTag: { + color: 'error.base', + text: 'Development', + }, +}); + +const useSelectorMock = jest.spyOn(reactRedux, 'useSelector'); +const useStateMock = jest.spyOn(React, 'useState'); + +let setShowModal: any; +let setEngine: any; +let setAllowUploads: any; + +const mockNonGSheetsDBs = [...new Array(2)].map((_, i) => ({ + changed_by: { + first_name: `user`, + last_name: `${i}`, + }, + database_name: `db ${i}`, + backend: 'postgresql', + allow_run_async: true, + allow_dml: false, + allow_file_upload: true, + expose_in_sqllab: false, + changed_on_delta_humanized: `${i} day(s) ago`, + changed_on: new Date().toISOString, + id: i, + engine_information: { + supports_file_upload: true, + }, +})); + +const mockGsheetsDbs = [...new Array(2)].map((_, i) => ({ + changed_by: { + first_name: `user`, + last_name: `${i}`, + }, + database_name: `db ${i}`, + backend: 'gsheets', + allow_run_async: true, + allow_dml: false, + allow_file_upload: true, + expose_in_sqllab: false, + changed_on_delta_humanized: `${i} day(s) ago`, + changed_on: new Date().toISOString, + id: i, + engine_information: { + supports_file_upload: false, + }, +})); + +describe('RightMenu', () => { + const mockedProps = createProps(); + + beforeEach(async () => { + useSelectorMock.mockReset(); + useStateMock.mockReset(); + fetchMock.get( + 'glob:*api/v1/database/?q=(filters:!((col:allow_file_upload,opr:upload_is_enabled,value:!t)))', + { result: [], database_count: 0 }, + ); + // By default we get file extensions to be uploaded + useSelectorMock.mockReturnValue({ + CSV_EXTENSIONS: ['csv'], + EXCEL_EXTENSIONS: ['xls', 'xlsx'], + COLUMNAR_EXTENSIONS: ['parquet', 'zip'], + ALLOWED_EXTENSIONS: ['parquet', 'zip', 'xls', 'xlsx', 'csv'], + }); + setShowModal = jest.fn(); + setEngine = jest.fn(); + setAllowUploads = jest.fn(); + const mockSetStateModal: any = (x: any) => [x, setShowModal]; + const mockSetStateEngine: any = (x: any) => [x, setEngine]; + const mockSetStateAllow: any = (x: any) => [x, setAllowUploads]; + useStateMock.mockImplementationOnce(mockSetStateModal); + useStateMock.mockImplementationOnce(mockSetStateEngine); + useStateMock.mockImplementationOnce(mockSetStateAllow); + }); + afterEach(fetchMock.restore); + it('renders', async () => { + const wrapper = mount(); + await waitForComponentToPaint(wrapper); + expect(wrapper.find(RightMenu)).toExist(); + }); + it('If user has permission to upload files we query the existing DBs that has allow_file_upload as True', async () => { + useSelectorMock.mockReturnValueOnce({ + createdOn: '2021-04-27T18:12:38.952304', + email: 'admin', + firstName: 'admin', + isActive: true, + lastName: 'admin', + permissions: {}, + roles: { + Admin: [ + ['can_this_form_get', 'CsvToDatabaseView'], // So we can upload CSV + ], + }, + userId: 1, + username: 'admin', + }); + // Second call we get the dashboardId + useSelectorMock.mockReturnValueOnce('1'); + const wrapper = mount(); + await waitForComponentToPaint(wrapper); + const callsD = fetchMock.calls(/database\/\?q/); + expect(callsD).toHaveLength(1); + expect(callsD[0][0]).toMatchInlineSnapshot( + `"http://localhost/api/v1/database/?q=(filters:!((col:allow_file_upload,opr:upload_is_enabled,value:!t)))"`, + ); + }); + it('If user has no permission to upload files the query API should not be called', async () => { + useSelectorMock.mockReturnValueOnce({ + createdOn: '2021-04-27T18:12:38.952304', + email: 'admin', + firstName: 'admin', + isActive: true, + lastName: 'admin', + permissions: {}, + roles: { + Admin: [['can_write', 'Chart']], // no file permissions + }, + userId: 1, + username: 'admin', + }); + // Second call we get the dashboardId + useSelectorMock.mockReturnValueOnce('1'); + const wrapper = mount(); + await waitForComponentToPaint(wrapper); + const callsD = fetchMock.calls(/database\/\?q/); + expect(callsD).toHaveLength(0); + }); + it('If user has permission to upload files but there are only gsheets and clickhouse DBs', async () => { + fetchMock.get( + 'glob:*api/v1/database/?q=(filters:!((col:allow_file_upload,opr:upload_is_enabled,value:!t)))', + { result: [...mockGsheetsDbs], database_count: 2 }, + { overwriteRoutes: true }, + ); + useSelectorMock.mockReturnValueOnce({ + createdOn: '2021-04-27T18:12:38.952304', + email: 'admin', + firstName: 'admin', + isActive: true, + lastName: 'admin', + permissions: {}, + roles: { + Admin: [ + ['can_this_form_get', 'CsvToDatabaseView'], // So we can upload CSV + ], + }, + userId: 1, + username: 'admin', + }); + // Second call we get the dashboardId + useSelectorMock.mockReturnValueOnce('1'); + const wrapper = mount(); + await waitForComponentToPaint(wrapper); + const callsD = fetchMock.calls(/database\/\?q/); + expect(callsD).toHaveLength(1); + expect(setAllowUploads).toHaveBeenCalledWith(false); + }); + it('If user has permission to upload files and some DBs with allow_file_upload are not gsheets nor clickhouse', async () => { + fetchMock.get( + 'glob:*api/v1/database/?q=(filters:!((col:allow_file_upload,opr:upload_is_enabled,value:!t)))', + { result: [...mockNonGSheetsDBs, ...mockGsheetsDbs], database_count: 2 }, + { overwriteRoutes: true }, + ); + useSelectorMock.mockReturnValueOnce({ + createdOn: '2021-04-27T18:12:38.952304', + email: 'admin', + firstName: 'admin', + isActive: true, + lastName: 'admin', + permissions: {}, + roles: { + Admin: [ + ['can_this_form_get', 'CsvToDatabaseView'], // So we can upload CSV + ], + }, + userId: 1, + username: 'admin', + }); + // Second call we get the dashboardId + useSelectorMock.mockReturnValueOnce('1'); + const wrapper = mount(); + await waitForComponentToPaint(wrapper); + const callsD = fetchMock.calls(/database\/\?q/); + expect(callsD).toHaveLength(1); + expect(setAllowUploads).toHaveBeenCalledWith(true); + }); +}); diff --git a/superset-frontend/src/views/components/RightMenu.tsx b/superset-frontend/src/views/components/RightMenu.tsx index 27f84a0be697d..cacd4089cb092 100644 --- a/superset-frontend/src/views/components/RightMenu.tsx +++ b/superset-frontend/src/views/components/RightMenu.tsx @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import React, { Fragment, useState, useEffect } from 'react'; +import React, { Fragment, useEffect } from 'react'; import rison from 'rison'; import { useSelector } from 'react-redux'; import { Link } from 'react-router-dom'; @@ -118,8 +118,8 @@ const RightMenu = ({ ALLOWED_EXTENSIONS, HAS_GSHEETS_INSTALLED, } = useSelector(state => state.common.conf); - const [showModal, setShowModal] = useState(false); - const [engine, setEngine] = useState(''); + const [showModal, setShowModal] = React.useState(false); + const [engine, setEngine] = React.useState(''); const canSql = findPermission('can_sqllab', 'Superset', roles); const canDashboard = findPermission('can_write', 'Dashboard', roles); const canChart = findPermission('can_write', 'Chart', roles); @@ -135,7 +135,7 @@ const RightMenu = ({ ); const showActionDropdown = canSql || canChart || canDashboard; - const [allowUploads, setAllowUploads] = useState(false); + const [allowUploads, setAllowUploads] = React.useState(false); const isAdmin = isUserAdmin(user); const showUploads = allowUploads || isAdmin; const dropdownItems: MenuObjectProps[] = [ @@ -207,7 +207,13 @@ const RightMenu = ({ SupersetClient.get({ endpoint: `/api/v1/database/?q=${rison.encode(payload)}`, }).then(({ json }: Record) => { - setAllowUploads(json.count >= 1); + // There might be some existings Gsheets and Clickhouse DBs + // with allow_file_upload set as True which is not possible from now on + const allowedDatabasesWithFileUpload = + json?.result?.filter( + (database: any) => database?.engine_information?.supports_file_upload, + ) || []; + setAllowUploads(allowedDatabasesWithFileUpload?.length >= 1); }); }; @@ -241,7 +247,7 @@ const RightMenu = ({ const isDisabled = isAdmin && !allowUploads; const tooltipText = t( - "Enable 'Allow data upload' in any database's settings", + "Enable 'Allow file uploads to database' in any database's settings", ); const buildMenuItem = (item: Record) => { diff --git a/superset-frontend/src/views/components/SubMenu.tsx b/superset-frontend/src/views/components/SubMenu.tsx index e51c1e8eca8ce..6ace899bf6e48 100644 --- a/superset-frontend/src/views/components/SubMenu.tsx +++ b/superset-frontend/src/views/components/SubMenu.tsx @@ -302,7 +302,7 @@ const SubMenuComponent: React.FunctionComponent = props => { {item.label} diff --git a/superset/databases/api.py b/superset/databases/api.py index e61bce68db481..edf63392ee6f6 100644 --- a/superset/databases/api.py +++ b/superset/databases/api.py @@ -129,6 +129,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi): "server_cert", "sqlalchemy_uri", "is_managed_externally", + "engine_information", ] list_columns = [ "allow_file_upload", @@ -151,6 +152,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi): "force_ctas_schema", "id", "disable_data_preview", + "engine_information", ] add_columns = [ "database_name", @@ -1062,6 +1064,13 @@ def available(self) -> Response: parameters: description: JSON schema defining the needed parameters type: object + engine_information: + description: Dict with public properties form the DB Engine + type: object + properties: + supports_file_upload: + description: Whether the engine supports file uploads + type: boolean 400: $ref: '#/components/responses/400' 500: @@ -1078,6 +1087,7 @@ def available(self) -> Response: "engine": engine_spec.engine, "available_drivers": sorted(drivers), "preferred": engine_spec.engine_name in preferred_databases, + "engine_information": engine_spec.get_public_information(), } if engine_spec.default_driver: diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index 28a6e2f5c3899..5a8354fd78dc7 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -361,6 +361,10 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods Pattern[str], Tuple[str, SupersetErrorType, Dict[str, Any]] ] = {} + # Whether the engine supports file uploads + # if True, database will be listed as option in the upload file form + supports_file_upload = True + @classmethod def supports_url(cls, url: URL) -> bool: """ @@ -1637,6 +1641,17 @@ def unmask_encrypted_extra(cls, old: str, new: str) -> str: """ return new + @classmethod + def get_public_information(cls) -> Dict[str, Any]: + """ + Construct a Dict with properties we want to expose. + + :returns: Dict with properties of our class like supports_file_upload + """ + return { + "supports_file_upload": cls.supports_file_upload, + } + # schema for adding a database by providing parameters instead of the # full SQLAlchemy URI diff --git a/superset/db_engine_specs/clickhouse.py b/superset/db_engine_specs/clickhouse.py index 4f34d2a5543c2..4531dca69860e 100644 --- a/superset/db_engine_specs/clickhouse.py +++ b/superset/db_engine_specs/clickhouse.py @@ -58,6 +58,8 @@ class ClickHouseEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method _show_functions_column = "name" + supports_file_upload = False + @classmethod def get_dbapi_exception_mapping(cls) -> Dict[Type[Exception], Type[Exception]]: return {NewConnectionError: SupersetDBAPIDatabaseError} diff --git a/superset/db_engine_specs/gsheets.py b/superset/db_engine_specs/gsheets.py index acde55b480c2b..3ee284788c245 100644 --- a/superset/db_engine_specs/gsheets.py +++ b/superset/db_engine_specs/gsheets.py @@ -81,6 +81,8 @@ class GSheetsEngineSpec(SqliteEngineSpec): ), } + supports_file_upload = False + @classmethod def get_url_for_impersonation( cls, diff --git a/superset/models/core.py b/superset/models/core.py index a8ab4df6b02cf..8c1b58171d470 100755 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -231,6 +231,7 @@ def data(self) -> Dict[str, Any]: "parameters": self.parameters, "disable_data_preview": self.disable_data_preview, "parameters_schema": self.parameters_schema, + "engine_information": self.engine_information, } @property @@ -312,6 +313,14 @@ def default_schemas(self) -> List[str]: def connect_args(self) -> Dict[str, Any]: return self.get_extra().get("engine_params", {}).get("connect_args", {}) + @property + def engine_information(self) -> Dict[str, Any]: + try: + engine_information = self.db_engine_spec.get_public_information() + except Exception: # pylint: disable=broad-except + engine_information = {} + return engine_information + @classmethod def get_password_masked_url_from_uri( # pylint: disable=invalid-name cls, uri: str diff --git a/superset/views/database/forms.py b/superset/views/database/forms.py index 4cf594c48705e..a44a412b483b0 100644 --- a/superset/views/database/forms.py +++ b/superset/views/database/forms.py @@ -52,6 +52,7 @@ def file_allowed_dbs() -> List[Database]: # type: ignore file_enabled_db for file_enabled_db in file_enabled_dbs if UploadToDatabaseForm.at_least_one_schema_is_allowed(file_enabled_db) + and UploadToDatabaseForm.is_engine_allowed_to_file_upl(file_enabled_db) ] @staticmethod @@ -89,6 +90,19 @@ def at_least_one_schema_is_allowed(database: Database) -> bool: return True return False + @staticmethod + def is_engine_allowed_to_file_upl(database: Database) -> bool: + """ + This method is mainly used for existing Gsheets and Clickhouse DBs + that have allow_file_upload set as True but they are no longer valid + DBs for file uploading. + New GSheets and Clickhouse DBs won't have the option to set + allow_file_upload set as True. + """ + if database.db_engine_spec.supports_file_upload: + return True + return False + class CsvToDatabaseForm(UploadToDatabaseForm): name = StringField( diff --git a/tests/integration_tests/databases/api_tests.py b/tests/integration_tests/databases/api_tests.py index fab3708c9968d..4bd80279a0274 100644 --- a/tests/integration_tests/databases/api_tests.py +++ b/tests/integration_tests/databases/api_tests.py @@ -195,6 +195,7 @@ def test_get_items(self): "created_by", "database_name", "disable_data_preview", + "engine_information", "explore_database_id", "expose_in_sqllab", "extra", @@ -1941,6 +1942,9 @@ def test_available(self, app, get_available_engine_specs): }, "preferred": True, "sqlalchemy_uri_placeholder": "postgresql://user:password@host:port/dbname[?key=value&key=value...]", + "engine_information": { + "supports_file_upload": True, + }, }, { "available_drivers": ["bigquery"], @@ -1960,6 +1964,9 @@ def test_available(self, app, get_available_engine_specs): }, "preferred": True, "sqlalchemy_uri_placeholder": "bigquery://{project_id}", + "engine_information": { + "supports_file_upload": True, + }, }, { "available_drivers": ["psycopg2"], @@ -2008,6 +2015,9 @@ def test_available(self, app, get_available_engine_specs): }, "preferred": False, "sqlalchemy_uri_placeholder": "redshift+psycopg2://user:password@host:port/dbname[?key=value&key=value...]", + "engine_information": { + "supports_file_upload": True, + }, }, { "available_drivers": ["apsw"], @@ -2027,6 +2037,9 @@ def test_available(self, app, get_available_engine_specs): }, "preferred": False, "sqlalchemy_uri_placeholder": "gsheets://", + "engine_information": { + "supports_file_upload": False, + }, }, { "available_drivers": ["mysqlconnector", "mysqldb"], @@ -2075,12 +2088,18 @@ def test_available(self, app, get_available_engine_specs): }, "preferred": False, "sqlalchemy_uri_placeholder": "mysql://user:password@host:port/dbname[?key=value&key=value...]", + "engine_information": { + "supports_file_upload": True, + }, }, { "available_drivers": [""], "engine": "hana", "name": "SAP HANA", "preferred": False, + "engine_information": { + "supports_file_upload": True, + }, }, ] } @@ -2108,12 +2127,18 @@ def test_available_no_default(self, app, get_available_engine_specs): "engine": "mysql", "name": "MySQL", "preferred": True, + "engine_information": { + "supports_file_upload": True, + }, }, { "available_drivers": [""], "engine": "hana", "name": "SAP HANA", "preferred": False, + "engine_information": { + "supports_file_upload": True, + }, }, ] } diff --git a/tests/unit_tests/sql_parse_tests.py b/tests/unit_tests/sql_parse_tests.py index 2f168d205cdaf..70e5d4d3b9c56 100644 --- a/tests/unit_tests/sql_parse_tests.py +++ b/tests/unit_tests/sql_parse_tests.py @@ -266,13 +266,9 @@ def test_extract_tables_illdefined() -> None: assert extract_tables("SELECT * FROM catalogname..tbname") == set() -@unittest.skip("Requires sqlparse>=3.1") def test_extract_tables_show_tables_from() -> None: """ Test ``SHOW TABLES FROM``. - - This is currently broken in the pinned version of sqlparse, and fixed in - ``sqlparse>=3.1``. However, ``sqlparse==3.1`` breaks some sql formatting. """ assert extract_tables("SHOW TABLES FROM s1 like '%order%'") == set() @@ -1017,15 +1013,15 @@ def test_unknown_select() -> None: Test that `is_select` works when sqlparse fails to identify the type. """ sql = "WITH foo AS(SELECT 1) SELECT 1" - assert sqlparse.parse(sql)[0].get_type() == "UNKNOWN" + assert sqlparse.parse(sql)[0].get_type() == "SELECT" assert ParsedQuery(sql).is_select() sql = "WITH foo AS(SELECT 1) INSERT INTO my_table (a) VALUES (1)" - assert sqlparse.parse(sql)[0].get_type() == "UNKNOWN" + assert sqlparse.parse(sql)[0].get_type() == "INSERT" assert not ParsedQuery(sql).is_select() sql = "WITH foo AS(SELECT 1) DELETE FROM my_table" - assert sqlparse.parse(sql)[0].get_type() == "UNKNOWN" + assert sqlparse.parse(sql)[0].get_type() == "DELETE" assert not ParsedQuery(sql).is_select() @@ -1108,15 +1104,6 @@ def test_messy_breakdown_statements() -> None: def test_sqlparse_formatting(): """ Test that ``from_unixtime`` is formatted correctly. - - ``sqlparse==0.3.1`` has a bug and removes space between ``from`` and - ``from_unixtime``, resulting in:: - - SELECT extract(HOUR - fromfrom_unixtime(hour_ts) - AT TIME ZONE 'America/Los_Angeles') - from table - """ assert sqlparse.format( "SELECT extract(HOUR from from_unixtime(hour_ts) "