From 53f9b8cac28463a0e1b80dc69b5382ff43d7e485 Mon Sep 17 00:00:00 2001 From: Antonio Rivero Date: Tue, 1 Nov 2022 10:43:21 -0300 Subject: [PATCH 1/3] CSV Upload: - Add expand/collapse to sections uisng custom macros - Add other delimiter text input using custom macros - Add custom jquery scripts to support our new features --- .../superset/form_view/csv_macros.html | 75 +++++++++ .../superset/form_view/csv_scripts.html | 37 +++++ .../form_view/csv_to_database_view/edit.html | 148 +++++++++++++++--- superset/views/database/forms.py | 12 +- superset/views/database/views.py | 53 ++++++- 5 files changed, 302 insertions(+), 23 deletions(-) create mode 100644 superset/templates/superset/form_view/csv_macros.html create mode 100644 superset/templates/superset/form_view/csv_scripts.html diff --git a/superset/templates/superset/form_view/csv_macros.html b/superset/templates/superset/form_view/csv_macros.html new file mode 100644 index 0000000000000..40c7bf54a0b27 --- /dev/null +++ b/superset/templates/superset/form_view/csv_macros.html @@ -0,0 +1,75 @@ +{# +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +#} +{% macro render_delimiter_field(field, begin_sep_label='', end_sep_label='', begin_sep_field='', end_sep_field='') %} + {% if field.id != 'csrf_token' %} + {% if field.type == 'HiddenField' %} + {{ field}} + {% else %} + {{begin_sep_label|safe}} + + {{end_sep_label|safe}} + {{begin_sep_field|safe}} + {{ field(**kwargs)|safe }} + + {{ field.description }} + {% endif %} + {% if field.errors %} +
+ {% for error in field.errors %} + {{ _(error) }} + {% endfor %} +
+ {% endif %} + {{end_sep_field|safe}} + {% endif %} +{% endmacro %} + +{% macro render_collapsable_form_group(id, section_title='') %} +
+
+ + + + + + + + + + +
+ + {{section_title}} +
+
+ + + {{ caller() }} + +
+
+
+
+
+{% endmacro %} diff --git a/superset/templates/superset/form_view/csv_scripts.html b/superset/templates/superset/form_view/csv_scripts.html new file mode 100644 index 0000000000000..bb7b94b1a37f7 --- /dev/null +++ b/superset/templates/superset/form_view/csv_scripts.html @@ -0,0 +1,37 @@ +{# +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +#} + diff --git a/superset/templates/superset/form_view/csv_to_database_view/edit.html b/superset/templates/superset/form_view/csv_to_database_view/edit.html index 2bec3aa12abb1..535271184c7f3 100644 --- a/superset/templates/superset/form_view/csv_to_database_view/edit.html +++ b/superset/templates/superset/form_view/csv_to_database_view/edit.html @@ -1,25 +1,137 @@ {# - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. #} +{% extends "appbuilder/base.html" %} +{% import 'appbuilder/general/lib.html' as lib %} +{% set begin_sep_label = '' %} + {% set end_sep_label = '' %} +{% set begin_sep_field = '' %} + {% set end_sep_field = '' %} {% import 'superset/form_view/database_schemas_selector.html' as schemas_selector %} -{% extends 'appbuilder/general/model/edit.html' %} - +{% import 'superset/form_view/csv_scripts.html' as csv_scripts %} +{% import 'superset/form_view/csv_macros.html' as csv_macros %} +{% block content %} +{{ lib.panel_begin(title, "edit") }} +
+
+ {{form.hidden_tag()}} +
+
+ + + + {{ lib.render_field(form.csv_file, begin_sep_label, end_sep_label, begin_sep_field, end_sep_field) }} + + + {{ lib.render_field(form.table_name, begin_sep_label, end_sep_label, begin_sep_field, end_sep_field) }} + + + {{ lib.render_field(form.database, begin_sep_label, end_sep_label, begin_sep_field, end_sep_field) }} + + + {{ lib.render_field(form.schema, begin_sep_label, end_sep_label, begin_sep_field, end_sep_field) }} + + + {{ csv_macros.render_delimiter_field(form.delimiter, begin_sep_label, end_sep_label, begin_sep_field, end_sep_field) }} + + +
+
+
+ {% call csv_macros.render_collapsable_form_group("accordion1", "File Settings") %} + + {{ lib.render_field(form.if_exists, begin_sep_label, end_sep_label, begin_sep_field, + end_sep_field) }} + + + {{ lib.render_field(form.skip_initial_space, begin_sep_label, end_sep_label, begin_sep_field, + end_sep_field) }} + + + {{ lib.render_field(form.skip_blank_lines, begin_sep_label, end_sep_label, begin_sep_field, + end_sep_field) }} + + + {{ lib.render_field(form.parse_dates, begin_sep_label, end_sep_label, begin_sep_field, + end_sep_field) }} + + + {{ lib.render_field(form.infer_datetime_format, begin_sep_label, end_sep_label, begin_sep_field, + end_sep_field) }} + + + {{ lib.render_field(form.decimal, begin_sep_label, end_sep_label, begin_sep_field, + end_sep_field) }} + + + {{ lib.render_field(form.null_values, begin_sep_label, end_sep_label, begin_sep_field, + end_sep_field) }} + + {% endcall %} + {% call csv_macros.render_collapsable_form_group("accordion2", "Columns") %} + + {{ lib.render_field(form.index_col, begin_sep_label, end_sep_label, begin_sep_field, + end_sep_field) }} + + + {{ lib.render_field(form.dataframe_index, begin_sep_label, end_sep_label, begin_sep_field, + end_sep_field) }} + + + {{ lib.render_field(form.index_label, begin_sep_label, end_sep_label, begin_sep_field, + end_sep_field) }} + + + {{ lib.render_field(form.use_cols, begin_sep_label, end_sep_label, begin_sep_field, + end_sep_field) }} + + + {{ lib.render_field(form.overwrite_duplicate, begin_sep_label, end_sep_label, begin_sep_field, + end_sep_field) }} + + {% endcall %} + {% call csv_macros.render_collapsable_form_group("accordion3", "Rows") %} + + {{ lib.render_field(form.header, begin_sep_label, end_sep_label, begin_sep_field, end_sep_field) + }} + + + {{ lib.render_field(form.nrows, begin_sep_label, end_sep_label, begin_sep_field, end_sep_field) + }} + + + {{ lib.render_field(form.skiprows, begin_sep_label, end_sep_label, begin_sep_field, + end_sep_field) }} + + {% endcall %} +
+
+ {{ lib.render_form_controls() }} +
+
+
+
+{% endblock %} +{% block add_tail_js %} + +{% endblock %} {% block tail_js %} - {{ super() }} - {{ schemas_selector }} +{{ super() }} +{{ schemas_selector }} +{{ csv_scripts }} {% endblock %} diff --git a/superset/views/database/forms.py b/superset/views/database/forms.py index 29ad9cdf0a3c7..425e3320c3596 100644 --- a/superset/views/database/forms.py +++ b/superset/views/database/forms.py @@ -146,11 +146,19 @@ class CsvToDatabaseForm(UploadToDatabaseForm): validators=[Optional()], widget=BS3TextFieldWidget(), ) - delimiter = StringField( + delimiter = SelectField( _("Delimiter"), description=_("Enter a delimiter for this data"), + choices=[ + (",", _(",")), + (".", _(".")), + ("other", _("Other")), + ], validators=[DataRequired()], - widget=BS3TextFieldWidget(), + default=[","], + ) + otherInput = StringField( + _("Other"), ) if_exists = SelectField( _("If Table Already Exists"), diff --git a/superset/views/database/views.py b/superset/views/database/views.py index 112328c1de7f1..7651b0eaf4076 100644 --- a/superset/views/database/views.py +++ b/superset/views/database/views.py @@ -18,7 +18,7 @@ import os import tempfile import zipfile -from typing import TYPE_CHECKING +from typing import Any, TYPE_CHECKING import pandas as pd from flask import flash, g, redirect @@ -109,7 +109,50 @@ def list(self) -> FlaskResponse: return super().render_app_template() -class CsvToDatabaseView(SimpleFormView): +class CustomFormView(SimpleFormView): + """ + View for presenting your own forms + Inherit from this view to provide some base + processing for your customized form views. + + Notice that this class inherits from BaseView + so all properties from the parent class can be overridden also. + + Implement form_get and form_post to implement + your form pre-processing and post-processing + """ + + @expose("/form", methods=["GET"]) + def this_form_get(self) -> Any: + self._init_vars() + form = self.form.refresh() + self.form_get(form) + self.update_redirect() + return self.render_template( + self.form_template, + title=self.form_title, + form=form, + appbuilder=self.appbuilder, + ) + + @expose("/form", methods=["POST"]) + def this_form_post(self) -> Any: + self._init_vars() + form = self.form.refresh() + if form.validate_on_submit(): + response = self.form_post(form) # pylint: disable=assignment-from-no-return + if not response: + return redirect(self.get_redirect()) + return response + return self.render_template( + self.form_template, + title=self.form_title, + form=form, + appbuilder=self.appbuilder, + ) + + +class CsvToDatabaseView(CustomFormView): form = CsvToDatabaseForm form_template = "superset/form_view/csv_to_database_view/edit.html" form_title = _("CSV to Database configuration") @@ -128,6 +171,7 @@ def form_get(self, form: CsvToDatabaseForm) -> None: def form_post(self, form: CsvToDatabaseForm) -> Response: database = form.database.data csv_table = Table(table=form.table_name.data, schema=form.schema.data) + delimiter_input = form.delimiter.data if not schema_allows_file_upload(database, csv_table.schema): message = __( @@ -139,6 +183,9 @@ def form_post(self, form: CsvToDatabaseForm) -> Response: flash(message, "danger") return redirect("/csvtodatabaseview/form") + if form.delimiter.data == "other": + delimiter_input = form.otherInput.data + try: df = pd.concat( pd.read_csv( @@ -155,7 +202,7 @@ def form_post(self, form: CsvToDatabaseForm) -> Response: na_values=form.null_values.data if form.null_values.data else None, nrows=form.nrows.data, parse_dates=form.parse_dates.data, - sep=form.delimiter.data, + sep=delimiter_input, skip_blank_lines=form.skip_blank_lines.data, skipinitialspace=form.skip_initial_space.data, skiprows=form.skiprows.data, From 236d0c102b55b2f775e5bb83455c44ae0c71377c Mon Sep 17 00:00:00 2001 From: Antonio Rivero Date: Tue, 1 Nov 2022 11:37:28 -0300 Subject: [PATCH 2/3] CSV Upload: - Add has_access decorator to new custom view so it is secure --- superset/views/database/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/superset/views/database/views.py b/superset/views/database/views.py index 7651b0eaf4076..94c1d6ff12c1f 100644 --- a/superset/views/database/views.py +++ b/superset/views/database/views.py @@ -123,6 +123,7 @@ class CustomFormView(SimpleFormView): """ @expose("/form", methods=["GET"]) + @has_access def this_form_get(self) -> Any: self._init_vars() form = self.form.refresh() @@ -136,6 +137,7 @@ def this_form_get(self) -> Any: ) @expose("/form", methods=["POST"]) + @has_access def this_form_post(self) -> Any: self._init_vars() form = self.form.refresh() From f266d22d1d568025812b8178342516cdb3f54eb3 Mon Sep 17 00:00:00 2001 From: Antonio Rivero Date: Sun, 27 Nov 2022 12:21:28 -0300 Subject: [PATCH 3/3] CSV Upload: - Fix formatting --- .../form_view/csv_to_database_view/edit.html | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/superset/templates/superset/form_view/csv_to_database_view/edit.html b/superset/templates/superset/form_view/csv_to_database_view/edit.html index 535271184c7f3..b09f9bd3838b1 100644 --- a/superset/templates/superset/form_view/csv_to_database_view/edit.html +++ b/superset/templates/superset/form_view/csv_to_database_view/edit.html @@ -1,20 +1,20 @@ {# -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at -http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. #} {% extends "appbuilder/base.html" %} {% import 'appbuilder/general/lib.html' as lib %} @@ -131,7 +131,7 @@ {% endblock %} {% block tail_js %} -{{ super() }} -{{ schemas_selector }} -{{ csv_scripts }} + {{ super() }} + {{ schemas_selector }} + {{ csv_scripts }} {% endblock %}