From ba96984048487ee368392c3b81a8b221b6c161ff Mon Sep 17 00:00:00 2001 From: John Bodley <4567245+john-bodley@users.noreply.github.com> Date: Fri, 10 Nov 2017 17:52:34 -0800 Subject: [PATCH 01/13] [flake8] Resolving E3?? errors (#3814) --- setup.py | 1 + superset/connectors/druid/views.py | 5 +++++ superset/connectors/sqla/models.py | 1 + superset/connectors/sqla/views.py | 4 ++++ superset/db_engine_specs.py | 1 + superset/db_engines/hive.py | 1 - superset/views/core.py | 18 ++++++++++++++++++ superset/views/sql_lab.py | 2 ++ tests/access_tests.py | 1 - tests/base_tests.py | 2 -- tests/celery_tests.py | 2 ++ tests/core_tests.py | 1 - tests/druid_tests.py | 3 +++ tests/email_tests.py | 1 + tests/superset_test_config.py | 2 ++ tests/utils_tests.py | 1 + tox.ini | 4 ---- 17 files changed, 41 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 7b887edeca7f4..00df4b4c3198b 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ def get_git_sha(): except Exception: return "" + GIT_SHA = get_git_sha() version_info = { 'GIT_SHA': GIT_SHA, diff --git a/superset/connectors/druid/views.py b/superset/connectors/druid/views.py index ade28c754aefb..5293166c77fc5 100644 --- a/superset/connectors/druid/views.py +++ b/superset/connectors/druid/views.py @@ -68,6 +68,7 @@ def post_update(self, col): def post_add(self, col): self.post_update(col) + appbuilder.add_view_no_menu(DruidColumnInlineView) @@ -117,6 +118,7 @@ def post_update(self, metric): if metric.is_restricted: security.merge_perm(sm, 'metric_access', metric.get_perm()) + appbuilder.add_view_no_menu(DruidMetricInlineView) @@ -155,6 +157,7 @@ def pre_update(self, cluster): def _delete(self, pk): DeleteMixin._delete(self, pk) + appbuilder.add_view( DruidClusterModelView, name="Druid Clusters", @@ -257,6 +260,7 @@ def post_update(self, datasource): def _delete(self, pk): DeleteMixin._delete(self, pk) + appbuilder.add_view( DruidDatasourceModelView, "Druid Datasources", @@ -303,6 +307,7 @@ def scan_new_datasources(self): """ return self.refresh_datasources(refreshAll=False) + appbuilder.add_view_no_menu(Druid) appbuilder.add_link( diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py index 03fac8eb2c1bb..8c70db31e606a 100644 --- a/superset/connectors/sqla/models.py +++ b/superset/connectors/sqla/models.py @@ -707,5 +707,6 @@ def query_datasources_by_name( query = query.filter_by(schema=schema) return query.all() + sa.event.listen(SqlaTable, 'after_insert', set_perm) sa.event.listen(SqlaTable, 'after_update', set_perm) diff --git a/superset/connectors/sqla/views.py b/superset/connectors/sqla/views.py index 513dd5a4f2fff..586c776368a10 100644 --- a/superset/connectors/sqla/views.py +++ b/superset/connectors/sqla/views.py @@ -89,6 +89,8 @@ class TableColumnInlineView(CompactCRUDMixin, SupersetModelView): # noqa 'database_expression': _("Database Expression"), 'type': _('Type'), } + + appbuilder.add_view_no_menu(TableColumnInlineView) @@ -142,6 +144,7 @@ def post_update(self, metric): if metric.is_restricted: security.merge_perm(sm, 'metric_access', metric.get_perm()) + appbuilder.add_view_no_menu(SqlMetricInlineView) @@ -284,6 +287,7 @@ def refresh(self, tables): flash(msg, 'info') return redirect('/tablemodelview/list/') + appbuilder.add_view( TableModelView, "Tables", diff --git a/superset/db_engine_specs.py b/superset/db_engine_specs.py index 421ca03b624f4..136b6594a2d4d 100644 --- a/superset/db_engine_specs.py +++ b/superset/db_engine_specs.py @@ -886,6 +886,7 @@ def get_configuration_for_impersonation(cls, uri, impersonate_user, username): configuration["hive.server2.proxy.user"] = username return configuration + class MssqlEngineSpec(BaseEngineSpec): engine = 'mssql' epoch_to_dttm = "dateadd(S, {col}, '1970-01-01')" diff --git a/superset/db_engines/hive.py b/superset/db_engines/hive.py index 1a1f51350b9e5..635a73bdfc245 100644 --- a/superset/db_engines/hive.py +++ b/superset/db_engines/hive.py @@ -3,7 +3,6 @@ from thrift import Thrift - # TODO: contribute back to pyhive. def fetch_logs(self, max_rows=1024, orientation=ttypes.TFetchOrientation.FETCH_NEXT): diff --git a/superset/views/core.py b/superset/views/core.py index 375a9ed38ab0a..3cbe7a62ecc87 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -272,6 +272,7 @@ def pre_update(self, db): def _delete(self, pk): DeleteMixin._delete(self, pk) + appbuilder.add_link( 'Import Dashboards', label=__("Import Dashboards"), @@ -299,12 +300,14 @@ class DatabaseAsync(DatabaseView): 'allow_run_async', 'allow_run_sync', 'allow_dml', ] + appbuilder.add_view_no_menu(DatabaseAsync) class DatabaseTablesAsync(DatabaseView): list_columns = ['id', 'all_table_names', 'all_schema_names'] + appbuilder.add_view_no_menu(DatabaseTablesAsync) @@ -324,6 +327,7 @@ class AccessRequestsModelView(SupersetModelView, DeleteMixin): 'created_on': _("Created On"), } + appbuilder.add_view( AccessRequestsModelView, "Access requests", @@ -411,6 +415,7 @@ def add(self): }), ) + appbuilder.add_view( SliceModelView, "Slices", @@ -429,6 +434,7 @@ class SliceAsync(SliceModelView): # noqa 'slice_link': _('Slice'), } + appbuilder.add_view_no_menu(SliceAsync) @@ -437,6 +443,7 @@ class SliceAddView(SliceModelView): # noqa 'id', 'slice_name', 'slice_link', 'viz_type', 'owners', 'modified', 'changed_on'] + appbuilder.add_view_no_menu(SliceAddView) @@ -554,6 +561,7 @@ class DashboardModelViewAsync(DashboardModelView): # noqa 'modified': _('Modified'), } + appbuilder.add_view_no_menu(DashboardModelViewAsync) @@ -569,6 +577,7 @@ class LogModelView(SupersetModelView): 'json': _("JSON"), } + appbuilder.add_view( LogModelView, "Action Log", @@ -582,10 +591,12 @@ class LogModelView(SupersetModelView): def health(): return "OK" + @app.route('/healthcheck') def healthcheck(): return "OK" + @app.route('/ping') def ping(): return "OK" @@ -619,6 +630,7 @@ def get_value(self, key_id): return json_error_response(e) return Response(kv.value, status=200) + appbuilder.add_view_no_menu(KV) @@ -652,6 +664,7 @@ def msg(self): flash(Markup(request.args.get("msg")), "info") return redirect(request.args.get("url")) + appbuilder.add_view_no_menu(R) @@ -2388,6 +2401,7 @@ def sliceQuery(self, slice_id): return json_error_response(DATASOURCE_ACCESS_ERR, status=401) return self.get_query_string_response(viz_obj) + appbuilder.add_view_no_menu(Superset) @@ -2404,6 +2418,7 @@ class CssTemplateModelView(SupersetModelView, DeleteMixin): class CssTemplateAsyncModelView(CssTemplateModelView): list_columns = ['template_name', 'css'] + appbuilder.add_separator("Sources") appbuilder.add_view( CssTemplateModelView, @@ -2426,6 +2441,7 @@ class CssTemplateAsyncModelView(CssTemplateModelView): category='SQL Lab', category_label=__("SQL Lab"), ) + appbuilder.add_link( 'Query Search', label=_("Query Search"), @@ -2451,6 +2467,8 @@ class RegexConverter(BaseConverter): def __init__(self, url_map, *items): super(RegexConverter, self).__init__(url_map) self.regex = items[0] + + app.url_map.converters['regex'] = RegexConverter diff --git a/superset/views/sql_lab.py b/superset/views/sql_lab.py index 7748b8732df94..06afb4e302425 100644 --- a/superset/views/sql_lab.py +++ b/superset/views/sql_lab.py @@ -20,6 +20,7 @@ class QueryView(SupersetModelView): 'end_time': _('End Time'), } + appbuilder.add_view( QueryView, "Queries", @@ -70,6 +71,7 @@ class SavedQueryViewApi(SavedQueryView): add_columns = show_columns edit_columns = add_columns + appbuilder.add_view_no_menu(SavedQueryViewApi) appbuilder.add_view_no_menu(SavedQueryView) diff --git a/tests/access_tests.py b/tests/access_tests.py index 5a8e8ee3b318d..437a75e1628b0 100644 --- a/tests/access_tests.py +++ b/tests/access_tests.py @@ -299,7 +299,6 @@ def test_clean_requests_after_schema_grant(self): ds = session.query(SqlaTable).filter_by( table_name='wb_health_population').first() - ds.schema = 'temp_schema' security.merge_perm( sm, 'schema_access', ds.schema_perm) diff --git a/tests/base_tests.py b/tests/base_tests.py index 4ba5f81344b02..f18dfe2fd8e71 100644 --- a/tests/base_tests.py +++ b/tests/base_tests.py @@ -112,8 +112,6 @@ def __init__(self, *args, **kwargs): session.add(druid_datasource2) session.commit() - - def get_table(self, table_id): return db.session.query(SqlaTable).filter_by( id=table_id).first() diff --git a/tests/celery_tests.py b/tests/celery_tests.py index b2681f27a5567..a825b4d345f6f 100644 --- a/tests/celery_tests.py +++ b/tests/celery_tests.py @@ -30,6 +30,8 @@ class CeleryConfig(object): CELERY_RESULT_BACKEND = 'db+sqlite:///' + app.config.get('SQL_CELERY_RESULTS_DB_FILE_PATH') CELERY_ANNOTATIONS = {'sql_lab.add': {'rate_limit': '10/s'}} CONCURRENCY = 1 + + app.config['CELERY_CONFIG'] = CeleryConfig diff --git a/tests/core_tests.py b/tests/core_tests.py index 6c3bae11cc34e..04f4746c6cf00 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -181,7 +181,6 @@ def test_save_slice(self): assert slc.slice_name == new_slice_name db.session.delete(slc) - def test_filter_endpoint(self): self.login(username='admin') slice_name = "Energy Sankey" diff --git a/tests/druid_tests.py b/tests/druid_tests.py index 09ecc8754f568..b8250f3b6193c 100644 --- a/tests/druid_tests.py +++ b/tests/druid_tests.py @@ -16,10 +16,12 @@ ) from .base_tests import SupersetTestCase + class PickableMock(Mock): def __reduce__(self): return (Mock, ()) + SEGMENT_METADATA = [{ "id": "some_id", "intervals": ["2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z"], @@ -199,6 +201,7 @@ def test_druid_sync_from_config(self): }, }, } + def check(): resp = self.client.post('/superset/sync_druid/', data=json.dumps(cfg)) druid_ds = ( diff --git a/tests/email_tests.py b/tests/email_tests.py index 8213a6f302c8c..f7b33c9c5f5f8 100644 --- a/tests/email_tests.py +++ b/tests/email_tests.py @@ -117,5 +117,6 @@ def test_send_mime_dryrun(self, mock_smtp, mock_smtp_ssl): assert not mock_smtp.called assert not mock_smtp_ssl.called + if __name__ == '__main__': unittest.main() diff --git a/tests/superset_test_config.py b/tests/superset_test_config.py index d65c85ff917f1..4f8c32c448163 100644 --- a/tests/superset_test_config.py +++ b/tests/superset_test_config.py @@ -29,4 +29,6 @@ class CeleryConfig(object): CELERY_RESULT_BACKEND = 'db+sqlite:///' + SQL_CELERY_RESULTS_DB_FILE_PATH CELERY_ANNOTATIONS = {'sql_lab.add': {'rate_limit': '10/s'}} CONCURRENCY = 1 + + CELERY_CONFIG = CeleryConfig diff --git a/tests/utils_tests.py b/tests/utils_tests.py index 22623aac6be0a..5096f80ad3525 100644 --- a/tests/utils_tests.py +++ b/tests/utils_tests.py @@ -12,6 +12,7 @@ SupersetException, validate_json, zlib_compress, zlib_decompress_to_string, ) + class UtilsTestCase(unittest.TestCase): def test_json_int_dttm_ser(self): dttm = datetime(2020, 1, 1) diff --git a/tox.ini b/tox.ini index ffdce842ad73a..fc926b97e270f 100644 --- a/tox.ini +++ b/tox.ini @@ -17,10 +17,6 @@ exclude = superset/migrations superset/templates ignore = - E302 - E303 - E305 - E306 E501 Q000 Q001 From 83e6807fa01c15d545244493920fb63a74a941f0 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Fri, 10 Nov 2017 17:54:56 -0800 Subject: [PATCH 02/13] [docs] add StatsD setup instructions (#3813) --- docs/installation.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index 823600ba6ed0f..3d043b8bcf3e7 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -550,3 +550,20 @@ same server. return "Ok" BLUEPRINTS = [simple_page] + +StatsD logging +-------------- + +Superset is instrumented to log events to StatsD if desired. Most endpoints hit +are logged as well as key events like query start and end in SQL Lab. + +To setup StatsD logging, it's a matter of configuring the logger in your +``superset_config.py``. + +..code :: + + from superset.stats_logger import StatsdStatsLogger + STATS_LOGGER = StatsdStatsLogger(host='localhost', port=8125, prefix='superset') + +Note that it's also possible to implement you own logger by deriving +``superset.stats_logger.BaseStatsLogger``. From 4d48d5d854e23d469c80105a6e384e3c4db8942d Mon Sep 17 00:00:00 2001 From: Jeff Niu Date: Fri, 10 Nov 2017 21:33:31 -0800 Subject: [PATCH 03/13] [Explore] Altered Slice Tag (#3668) * Added altered tag to explore slice view and fixes #3616 * unit tests * Moved getDiffs logic into AlteredSliceTag * code style fixs --- .../components/AlteredSliceTag.jsx | 145 +++++++++++ .../explore/components/ExploreChartHeader.jsx | 10 + superset/assets/package.json | 1 + .../components/AlteredSliceTag_spec.jsx | 235 ++++++++++++++++++ superset/utils.py | 36 ++- tests/utils_tests.py | 87 ++++++- 6 files changed, 510 insertions(+), 4 deletions(-) create mode 100644 superset/assets/javascripts/components/AlteredSliceTag.jsx create mode 100644 superset/assets/spec/javascripts/components/AlteredSliceTag_spec.jsx diff --git a/superset/assets/javascripts/components/AlteredSliceTag.jsx b/superset/assets/javascripts/components/AlteredSliceTag.jsx new file mode 100644 index 0000000000000..eb24424e8d41f --- /dev/null +++ b/superset/assets/javascripts/components/AlteredSliceTag.jsx @@ -0,0 +1,145 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import { Table, Tr, Td, Thead, Th } from 'reactable'; +import { isEqual, isEmpty } from 'underscore'; + +import TooltipWrapper from './TooltipWrapper'; +import { controls } from '../explore/stores/controls'; +import ModalTrigger from './ModalTrigger'; +import { t } from '../locales'; + +const propTypes = { + origFormData: PropTypes.object.isRequired, + currentFormData: PropTypes.object.isRequired, +}; + +export default class AlteredSliceTag extends React.Component { + + constructor(props) { + super(props); + const diffs = this.getDiffs(props); + this.state = { diffs, hasDiffs: !isEmpty(diffs) }; + } + + componentWillReceiveProps(newProps) { + // Update differences if need be + if (isEqual(this.props, newProps)) { + return; + } + const diffs = this.getDiffs(newProps); + this.setState({ diffs, hasDiffs: !isEmpty(diffs) }); + } + + getDiffs(props) { + // Returns all properties that differ in the + // current form data and the saved form data + const ofd = props.origFormData; + const cfd = props.currentFormData; + const fdKeys = Object.keys(cfd); + const diffs = {}; + for (const fdKey of fdKeys) { + // Ignore values that are undefined/nonexisting in either + if (!ofd[fdKey] && !cfd[fdKey]) { + continue; + } + if (!isEqual(ofd[fdKey], cfd[fdKey])) { + diffs[fdKey] = { before: ofd[fdKey], after: cfd[fdKey] }; + } + } + return diffs; + } + + formatValue(value, key) { + // Format display value based on the control type + // or the value type + if (value === undefined) { + return 'N/A'; + } else if (value === null) { + return 'null'; + } else if (controls[key] && controls[key].type === 'FilterControl') { + if (!value.length) { + return '[]'; + } + return value.map((v) => { + const filterVal = v.val.constructor === Array ? `[${v.val.join(', ')}]` : v.val; + return `${v.col} ${v.op} ${filterVal}`; + }).join(', '); + } else if (controls[key] && controls[key].type === 'BoundsControl') { + return `Min: ${value[0]}, Max: ${value[1]}`; + } else if (controls[key] && controls[key].type === 'CollectionControl') { + return value.map(v => JSON.stringify(v)).join(', '); + } else if (typeof value === 'boolean') { + return value ? 'true' : 'false'; + } else if (value.constructor === Array) { + return value.length ? value.join(', ') : '[]'; + } else if (typeof value === 'string' || typeof value === 'number') { + return value; + } + return JSON.stringify(value); + } + + renderRows() { + const diffs = this.state.diffs; + const rows = []; + for (const key in diffs) { + rows.push( + + + {this.formatValue(diffs[key].before, key)} + {this.formatValue(diffs[key].after, key)} + , + ); + } + return rows; + } + + renderModalBody() { + return ( + + + + + + + {this.renderRows()} +
ControlBeforeAfter
+ ); + } + + renderTriggerNode() { + return ( + + + {t('Altered')} + + + ); + } + + render() { + // Return nothing if there are no differences + if (!this.state.hasDiffs) { + return null; + } + // Render the label-warning 'Altered' tag which the user may + // click to open a modal containing a table summarizing the + // differences in the slice + return ( + + ); + } +} + +AlteredSliceTag.propTypes = propTypes; diff --git a/superset/assets/javascripts/explore/components/ExploreChartHeader.jsx b/superset/assets/javascripts/explore/components/ExploreChartHeader.jsx index 8c2e1f3ca442f..3750fc029141f 100644 --- a/superset/assets/javascripts/explore/components/ExploreChartHeader.jsx +++ b/superset/assets/javascripts/explore/components/ExploreChartHeader.jsx @@ -4,6 +4,7 @@ import PropTypes from 'prop-types'; import { chartPropType } from '../../chart/chartReducer'; import ExploreActionButtons from './ExploreActionButtons'; import EditableTitle from '../../components/EditableTitle'; +import AlteredSliceTag from '../../components/AlteredSliceTag'; import FaveStar from '../../components/FaveStar'; import TooltipWrapper from '../../components/TooltipWrapper'; import Timer from '../../components/Timer'; @@ -54,6 +55,13 @@ class ExploreChartHeader extends React.PureComponent { }); } + renderAlteredTag() { + const origFormData = (this.props.slice && this.props.slice.form_data) || {}; + const currentFormData = this.props.form_data; + const tagProps = { origFormData, currentFormData }; + return (); + } + renderChartTitle() { let title; if (this.props.slice) { @@ -106,6 +114,8 @@ class ExploreChartHeader extends React.PureComponent { } + {this.renderAlteredTag()} +
{this.props.chart.chartStatus === 'success' && queryResponse && diff --git a/superset/assets/package.json b/superset/assets/package.json index 43aa268f288f4..99f8afc16f66a 100644 --- a/superset/assets/package.json +++ b/superset/assets/package.json @@ -96,6 +96,7 @@ "srcdoc-polyfill": "^1.0.0", "supercluster": "https://github.com/georgeke/supercluster/tarball/ac3492737e7ce98e07af679623aad452373bbc40", "urijs": "^1.18.10", + "underscore": "^1.8.3", "viewport-mercator-project": "^2.1.0" }, "devDependencies": { diff --git a/superset/assets/spec/javascripts/components/AlteredSliceTag_spec.jsx b/superset/assets/spec/javascripts/components/AlteredSliceTag_spec.jsx new file mode 100644 index 0000000000000..f3b746bbb361c --- /dev/null +++ b/superset/assets/spec/javascripts/components/AlteredSliceTag_spec.jsx @@ -0,0 +1,235 @@ +import React from 'react'; +import { shallow } from 'enzyme'; +import { describe, it } from 'mocha'; +import { expect } from 'chai'; + +import { Table, Thead, Td, Th, Tr } from 'reactable'; + +import AlteredSliceTag from '../../../javascripts/components/AlteredSliceTag'; +import ModalTrigger from '../../../javascripts/components/ModalTrigger'; +import TooltipWrapper from '../../../javascripts/components/TooltipWrapper'; + +const defaultProps = { + origFormData: { + filters: [{ col: 'a', op: '==', val: 'hello' }], + y_axis_bounds: [10, 20], + column_collection: [{ 1: 'a', b: ['6', 'g'] }], + bool: false, + alpha: undefined, + gucci: [1, 2, 3, 4], + never: 5, + ever: { a: 'b', c: 'd' }, + }, + currentFormData: { + filters: [{ col: 'b', op: 'in', val: ['hello', 'my', 'name'] }], + y_axis_bounds: [15, 16], + column_collection: [{ 1: 'a', b: [9, '15'], t: 'gggg' }], + bool: true, + alpha: null, + gucci: ['a', 'b', 'c', 'd'], + never: 10, + ever: { x: 'y', z: 'z' }, + }, +}; + +const expectedDiffs = { + filters: { + before: [{ col: 'a', op: '==', val: 'hello' }], + after: [{ col: 'b', op: 'in', val: ['hello', 'my', 'name'] }], + }, + y_axis_bounds: { + before: [10, 20], + after: [15, 16], + }, + column_collection: { + before: [{ 1: 'a', b: ['6', 'g'] }], + after: [{ 1: 'a', b: [9, '15'], t: 'gggg' }], + }, + bool: { + before: false, + after: true, + }, + gucci: { + before: [1, 2, 3, 4], + after: ['a', 'b', 'c', 'd'], + }, + never: { + before: 5, + after: 10, + }, + ever: { + before: { a: 'b', c: 'd' }, + after: { x: 'y', z: 'z' }, + }, +}; + +describe('AlteredSliceTag', () => { + let wrapper; + let props; + + beforeEach(() => { + props = Object.assign({}, defaultProps); + wrapper = shallow(); + }); + + it('correctly determines form data differences', () => { + const diffs = wrapper.instance().getDiffs(props); + expect(diffs).to.deep.equal(expectedDiffs); + expect(wrapper.instance().state.diffs).to.deep.equal(expectedDiffs); + expect(wrapper.instance().state.hasDiffs).to.equal(true); + }); + + it('does not run when there are no differences', () => { + props = { + origFormData: props.origFormData, + currentFormData: props.origFormData, + }; + wrapper = shallow(); + expect(wrapper.instance().state.diffs).to.deep.equal({}); + expect(wrapper.instance().state.hasDiffs).to.equal(false); + expect(wrapper.instance().render()).to.equal(null); + }); + + it('sets new diffs when receiving new props', () => { + const newProps = { + currentFormData: Object.assign({}, props.currentFormData), + origFormData: Object.assign({}, props.origFormData), + }; + newProps.currentFormData.beta = 10; + wrapper = shallow(); + wrapper.instance().componentWillReceiveProps(newProps); + const newDiffs = wrapper.instance().state.diffs; + const expectedBeta = { before: undefined, after: 10 }; + expect(newDiffs.beta).to.deep.equal(expectedBeta); + }); + + it('does not set new state when props are the same', () => { + const currentDiff = wrapper.instance().state.diffs; + wrapper.instance().componentWillReceiveProps(props); + // Check equal references + expect(wrapper.instance().state.diffs).to.equal(currentDiff); + }); + + it('renders a ModalTrigger', () => { + expect(wrapper.find(ModalTrigger)).to.have.lengthOf(1); + }); + + describe('renderTriggerNode', () => { + it('renders a TooltipWrapper', () => { + const triggerNode = shallow(
{wrapper.instance().renderTriggerNode()}
); + expect(triggerNode.find(TooltipWrapper)).to.have.lengthOf(1); + }); + }); + + describe('renderModalBody', () => { + it('renders a Table', () => { + const modalBody = shallow(
{wrapper.instance().renderModalBody()}
); + expect(modalBody.find(Table)).to.have.lengthOf(1); + }); + + it('renders a Thead', () => { + const modalBody = shallow(
{wrapper.instance().renderModalBody()}
); + expect(modalBody.find(Thead)).to.have.lengthOf(1); + }); + + it('renders Th', () => { + const modalBody = shallow(
{wrapper.instance().renderModalBody()}
); + const th = modalBody.find(Th); + expect(th).to.have.lengthOf(3); + ['control', 'before', 'after'].forEach((v, i) => { + expect(th.get(i).props.column).to.equal(v); + }); + }); + + it('renders the correct number of Tr', () => { + const modalBody = shallow(
{wrapper.instance().renderModalBody()}
); + const tr = modalBody.find(Tr); + expect(tr).to.have.lengthOf(7); + }); + + it('renders the correct number of Td', () => { + const modalBody = shallow(
{wrapper.instance().renderModalBody()}
); + const td = modalBody.find(Td); + expect(td).to.have.lengthOf(21); + ['control', 'before', 'after'].forEach((v, i) => { + expect(td.get(i).props.column).to.equal(v); + }); + }); + }); + + describe('renderRows', () => { + it('returns an array of rows with one Tr and three Td', () => { + const rows = wrapper.instance().renderRows(); + expect(rows).to.have.lengthOf(7); + const fakeRow = shallow(
{rows[0]}
); + expect(fakeRow.find(Tr)).to.have.lengthOf(1); + expect(fakeRow.find(Td)).to.have.lengthOf(3); + }); + }); + + describe('formatValue', () => { + it('returns "N/A" for undefined values', () => { + expect(wrapper.instance().formatValue(undefined, 'b')).to.equal('N/A'); + }); + + it('returns "null" for null values', () => { + expect(wrapper.instance().formatValue(null, 'b')).to.equal('null'); + }); + + it('returns "Max" and "Min" for BoundsControl', () => { + expect(wrapper.instance().formatValue([5, 6], 'y_axis_bounds')).to.equal( + 'Min: 5, Max: 6', + ); + }); + + it('returns stringified objects for CollectionControl', () => { + const value = [{ 1: 2, alpha: 'bravo' }, { sent: 'imental', w0ke: 5 }]; + const expected = '{"1":2,"alpha":"bravo"}, {"sent":"imental","w0ke":5}'; + expect(wrapper.instance().formatValue(value, 'column_collection')).to.equal(expected); + }); + + it('returns boolean values as string', () => { + expect(wrapper.instance().formatValue(true, 'b')).to.equal('true'); + expect(wrapper.instance().formatValue(false, 'b')).to.equal('false'); + }); + + it('returns Array joined by commas', () => { + const value = [5, 6, 7, 8, 'hello', 'goodbye']; + const expected = '5, 6, 7, 8, hello, goodbye'; + expect(wrapper.instance().formatValue(value)).to.equal(expected); + }); + + it('stringifies objects', () => { + const value = { 1: 2, alpha: 'bravo' }; + const expected = '{"1":2,"alpha":"bravo"}'; + expect(wrapper.instance().formatValue(value)).to.equal(expected); + }); + + it('does nothing to strings and numbers', () => { + expect(wrapper.instance().formatValue(5)).to.equal(5); + expect(wrapper.instance().formatValue('hello')).to.equal('hello'); + }); + + it('returns "[]" for empty filters', () => { + expect(wrapper.instance().formatValue([], 'filters')).to.equal('[]'); + }); + + it('correctly formats filters with array values', () => { + const filters = [ + { col: 'a', op: 'in', val: ['1', 'g', '7', 'ho'] }, + { col: 'b', op: 'not in', val: ['hu', 'ho', 'ha'] }, + ]; + const expected = 'a in [1, g, 7, ho], b not in [hu, ho, ha]'; + expect(wrapper.instance().formatValue(filters, 'filters')).to.equal(expected); + }); + + it('correctly formats filters with string values', () => { + const filters = [ + { col: 'a', op: '==', val: 'gucci' }, + { col: 'b', op: 'LIKE', val: 'moshi moshi' }, + ]; + const expected = 'a == gucci, b LIKE moshi moshi'; + expect(wrapper.instance().formatValue(filters, 'filters')).to.equal(expected); + }); + }); +}); diff --git a/superset/utils.py b/superset/utils.py index 74d8dfbb63dc2..f3f1a60f6a491 100644 --- a/superset/utils.py +++ b/superset/utils.py @@ -684,10 +684,40 @@ def merge_extra_filters(form_data): '__time_origin': 'druid_time_origin', '__granularity': 'granularity', } + # Grab list of existing filters 'keyed' on the column and operator + + def get_filter_key(f): + return f['col'] + '__' + f['op'] + existing_filters = {} + for existing in form_data['filters']: + existing_filters[get_filter_key(existing)] = existing['val'] for filtr in form_data['extra_filters']: - if date_options.get(filtr['col']): # merge date options + # Pull out time filters/options and merge into form data + if date_options.get(filtr['col']): if filtr.get('val'): form_data[date_options[filtr['col']]] = filtr['val'] - else: - form_data['filters'] += [filtr] # merge col filters + elif filtr['val'] and len(filtr['val']): + # Merge column filters + filter_key = get_filter_key(filtr) + if filter_key in existing_filters: + # Check if the filter already exists + if isinstance(filtr['val'], list): + if isinstance(existing_filters[filter_key], list): + # Add filters for unequal lists + # order doesn't matter + if ( + sorted(existing_filters[filter_key]) != + sorted(filtr['val']) + ): + form_data['filters'] += [filtr] + else: + form_data['filters'] += [filtr] + else: + # Do not add filter if same value already exists + if filtr['val'] != existing_filters[filter_key]: + form_data['filters'] += [filtr] + else: + # Filter not found, add it + form_data['filters'] += [filtr] + # Remove extra filters from the form data since no longer needed del form_data['extra_filters'] diff --git a/tests/utils_tests.py b/tests/utils_tests.py index 5096f80ad3525..dc6b454a36ff9 100644 --- a/tests/utils_tests.py +++ b/tests/utils_tests.py @@ -110,6 +110,89 @@ def test_merge_extra_filters(self): merge_extra_filters(form_data) self.assertEquals(form_data, expected) + def test_merge_extra_filters_ignores_empty_filters(self): + form_data = {'extra_filters': [ + {'col': 'a', 'op': 'in', 'val': ''}, + {'col': 'B', 'op': '==', 'val': []}, + ]} + expected = {'filters': []} + merge_extra_filters(form_data) + self.assertEquals(form_data, expected) + + def test_merge_extra_filters_ignores_equal_filters(self): + form_data = { + 'extra_filters': [ + {'col': 'a', 'op': 'in', 'val': 'someval'}, + {'col': 'B', 'op': '==', 'val': ['c1', 'c2']}, + ], + 'filters': [ + {'col': 'a', 'op': 'in', 'val': 'someval'}, + {'col': 'B', 'op': '==', 'val': ['c1', 'c2']}, + ], + } + expected = {'filters': [ + {'col': 'a', 'op': 'in', 'val': 'someval'}, + {'col': 'B', 'op': '==', 'val': ['c1', 'c2']}, + ]} + merge_extra_filters(form_data) + self.assertEquals(form_data, expected) + + def test_merge_extra_filters_merges_different_val_types(self): + form_data = { + 'extra_filters': [ + {'col': 'a', 'op': 'in', 'val': ['g1', 'g2']}, + {'col': 'B', 'op': '==', 'val': ['c1', 'c2']}, + ], + 'filters': [ + {'col': 'a', 'op': 'in', 'val': 'someval'}, + {'col': 'B', 'op': '==', 'val': ['c1', 'c2']}, + ], + } + expected = {'filters': [ + {'col': 'a', 'op': 'in', 'val': 'someval'}, + {'col': 'B', 'op': '==', 'val': ['c1', 'c2']}, + {'col': 'a', 'op': 'in', 'val': ['g1', 'g2']}, + ]} + merge_extra_filters(form_data) + self.assertEquals(form_data, expected) + form_data = { + 'extra_filters': [ + {'col': 'a', 'op': 'in', 'val': 'someval'}, + {'col': 'B', 'op': '==', 'val': ['c1', 'c2']}, + ], + 'filters': [ + {'col': 'a', 'op': 'in', 'val': ['g1', 'g2']}, + {'col': 'B', 'op': '==', 'val': ['c1', 'c2']}, + ], + } + expected = {'filters': [ + {'col': 'a', 'op': 'in', 'val': ['g1', 'g2']}, + {'col': 'B', 'op': '==', 'val': ['c1', 'c2']}, + {'col': 'a', 'op': 'in', 'val': 'someval'}, + ]} + merge_extra_filters(form_data) + self.assertEquals(form_data, expected) + + def test_merge_extra_filters_adds_unequal_lists(self): + form_data = { + 'extra_filters': [ + {'col': 'a', 'op': 'in', 'val': ['g1', 'g2', 'g3']}, + {'col': 'B', 'op': '==', 'val': ['c1', 'c2', 'c3']}, + ], + 'filters': [ + {'col': 'a', 'op': 'in', 'val': ['g1', 'g2']}, + {'col': 'B', 'op': '==', 'val': ['c1', 'c2']}, + ], + } + expected = {'filters': [ + {'col': 'a', 'op': 'in', 'val': ['g1', 'g2']}, + {'col': 'B', 'op': '==', 'val': ['c1', 'c2']}, + {'col': 'a', 'op': 'in', 'val': ['g1', 'g2', 'g3']}, + {'col': 'B', 'op': '==', 'val': ['c1', 'c2', 'c3']}, + ]} + merge_extra_filters(form_data) + self.assertEquals(form_data, expected) + def test_datetime_f(self): self.assertEquals( datetime_f(datetime(1990, 9, 21, 19, 11, 19, 626096)), @@ -119,7 +202,9 @@ def test_datetime_f(self): self.assertEquals(datetime_f(None), 'None') iso = datetime.now().isoformat()[:10].split('-') [a, b, c] = [int(v) for v in iso] - self.assertEquals(datetime_f(datetime(a, b, c)), '00:00:00') + self.assertEquals( + datetime_f(datetime(a, b, c)), '00:00:00', + ) def test_json_encoded_obj(self): obj = {'a': 5, 'b': ['a', 'g', 5]} From a3a4687ebf8163f3d72b8377b1db6ec19bc3ba92 Mon Sep 17 00:00:00 2001 From: John Bodley <4567245+john-bodley@users.noreply.github.com> Date: Sat, 11 Nov 2017 21:43:55 -0800 Subject: [PATCH 04/13] [viz] Fix payload force logic (#3839) --- superset/viz.py | 1 - 1 file changed, 1 deletion(-) diff --git a/superset/viz.py b/superset/viz.py index ee6c2fc8636ae..acc90d0b36839 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -244,7 +244,6 @@ def get_payload(self, force=False): """Handles caching around the json payload retrieval""" cache_key = self.cache_key payload = None - force = force if force else self.form_data.get('force') == 'true' if not force and cache: payload = cache.get(cache_key) From d908e48d61200cbacadbb0dcf8a8e50c309f8c10 Mon Sep 17 00:00:00 2001 From: John Bodley <4567245+john-bodley@users.noreply.github.com> Date: Sat, 11 Nov 2017 21:44:24 -0800 Subject: [PATCH 05/13] [slice] Removing deprecated argument (#3838) --- superset/models/core.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/superset/models/core.py b/superset/models/core.py index fdeea6b143c1f..e067da4c2d115 100644 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -229,12 +229,9 @@ def slice_link(self): name = escape(self.slice_name) return Markup('{name}'.format(**locals())) - def get_viz(self, url_params_multidict=None): + def get_viz(self): """Creates :py:class:viz.BaseViz object from the url_params_multidict. - :param werkzeug.datastructures.MultiDict url_params_multidict: - Contains the visualization params, they override the self.params - stored in the database :return: object of the 'viz_type' type that is taken from the url_params_multidict or self.params. :rtype: :py:class:viz.BaseViz From f7bf17290cca7482b82f3eab3ebda8672940bca3 Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Sun, 12 Nov 2017 06:44:55 +0100 Subject: [PATCH 06/13] run_tests.sh: call coveralls only on CI (#3836) --- run_tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 55760f633bb67..448a750fdf30f 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -9,4 +9,6 @@ set -e superset/bin/superset db upgrade superset/bin/superset version -v python setup.py nosetests -coveralls +if [ "$CI" = "true" ] ; then + coveralls +fi From 8459347bdc90e6f2459a08c87f00a6659f12cd48 Mon Sep 17 00:00:00 2001 From: Grace Guo Date: Sat, 11 Nov 2017 21:45:29 -0800 Subject: [PATCH 07/13] [Dashboard bug] should reset chartAlert when start new query (#3837) --- superset/assets/javascripts/chart/chartReducer.js | 1 + 1 file changed, 1 insertion(+) diff --git a/superset/assets/javascripts/chart/chartReducer.js b/superset/assets/javascripts/chart/chartReducer.js index 7d5b23c32feab..c67d66210a732 100644 --- a/superset/assets/javascripts/chart/chartReducer.js +++ b/superset/assets/javascripts/chart/chartReducer.js @@ -41,6 +41,7 @@ export default function chartReducer(charts = {}, action) { [actions.CHART_UPDATE_STARTED](state) { return { ...state, chartStatus: 'loading', + chartAlert: null, chartUpdateEndTime: null, chartUpdateStartTime: now(), queryRequest: action.queryRequest, From b4c9402737067bdddc0658019da7e90aafc8fa34 Mon Sep 17 00:00:00 2001 From: Grace Guo Date: Sat, 11 Nov 2017 21:51:25 -0800 Subject: [PATCH 08/13] [Dashboard bug] Fix Cache status and dttm information display for each slice (#3833) --- .../dashboard/components/GridCell.jsx | 9 ++- .../dashboard/components/GridLayout.jsx | 73 ++++++++++--------- .../dashboard/components/SliceHeader.jsx | 6 +- 3 files changed, 51 insertions(+), 37 deletions(-) diff --git a/superset/assets/javascripts/dashboard/components/GridCell.jsx b/superset/assets/javascripts/dashboard/components/GridCell.jsx index 1a59a92dd69d6..26d8eeb6eadc4 100644 --- a/superset/assets/javascripts/dashboard/components/GridCell.jsx +++ b/superset/assets/javascripts/dashboard/components/GridCell.jsx @@ -11,6 +11,8 @@ const propTypes = { timeout: PropTypes.number, datasource: PropTypes.object, isLoading: PropTypes.bool, + isCached: PropTypes.bool, + cachedDttm: PropTypes.number, isExpanded: PropTypes.bool, widgetHeight: PropTypes.number, widgetWidth: PropTypes.number, @@ -78,8 +80,9 @@ class GridCell extends React.PureComponent { render() { const { - exploreChartUrl, exportCSVUrl, isExpanded, isLoading, removeSlice, updateSliceName, - toggleExpandSlice, forceRefresh, chartKey, slice, datasource, formData, timeout, + exploreChartUrl, exportCSVUrl, isExpanded, isLoading, isCached, cachedDttm, + removeSlice, updateSliceName, toggleExpandSlice, forceRefresh, + chartKey, slice, datasource, formData, timeout, } = this.props; return (
{ + const chartKey = `slice_${slice.slice_id}`; + const currentChart = this.props.charts[chartKey]; + const queryResponse = currentChart.queryResponse || {}; + return ( +
+ +
); + }); + return ( - {this.props.dashboard.slices.map(slice => ( -
- -
- ))} + {cells} ); } diff --git a/superset/assets/javascripts/dashboard/components/SliceHeader.jsx b/superset/assets/javascripts/dashboard/components/SliceHeader.jsx index d1a2d9ec94274..8360017d44fa2 100644 --- a/superset/assets/javascripts/dashboard/components/SliceHeader.jsx +++ b/superset/assets/javascripts/dashboard/components/SliceHeader.jsx @@ -11,6 +11,8 @@ const propTypes = { exploreChartUrl: PropTypes.string, exportCSVUrl: PropTypes.string, isExpanded: PropTypes.bool, + isCached: PropTypes.bool, + cachedDttm: PropTypes.number, formDataExtra: PropTypes.object, removeSlice: PropTypes.func, updateSliceName: PropTypes.func, @@ -40,9 +42,9 @@ class SliceHeader extends React.PureComponent { render() { const slice = this.props.slice; - const isCached = slice.is_cached; + const isCached = this.props.isCached; const isExpanded = !!this.props.isExpanded; - const cachedWhen = moment.utc(slice.cached_dttm).fromNow(); + const cachedWhen = moment.utc(this.props.cachedDttm).fromNow(); const refreshTooltip = isCached ? t('Served from data cached %s . Click to force refresh.', cachedWhen) : t('Force refresh data'); From 7fce8eab3aa10850278440f4ef283480d416f95b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=B4=81=E7=8E=89?= Date: Sun, 12 Nov 2017 13:51:53 +0800 Subject: [PATCH 09/13] Update setup.py (#3510) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 00df4b4c3198b..217a0708ffef3 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup BASE_DIR = os.path.abspath(os.path.dirname(__file__)) -PACKAGE_DIR = os.path.join(BASE_DIR, 'superset', 'static', 'assets') +PACKAGE_DIR = os.path.join(BASE_DIR, 'superset', 'assets') PACKAGE_FILE = os.path.join(PACKAGE_DIR, 'package.json') with open(PACKAGE_FILE) as package_file: version_string = json.load(package_file)['version'] From 1a3a8daf499ad271d4e82f287f34e41ca188d95e Mon Sep 17 00:00:00 2001 From: Grace Guo Date: Sat, 11 Nov 2017 22:38:40 -0800 Subject: [PATCH 10/13] [Dashboard bug] should reset chartAlert when start new query (#3841) From 1b4f128f55e4eb0944a0caa255fa4a90b7db4716 Mon Sep 17 00:00:00 2001 From: John Bodley <4567245+john-bodley@users.noreply.github.com> Date: Sun, 12 Nov 2017 11:09:22 -0800 Subject: [PATCH 11/13] [flake8] Resolving F5?? errors (#3846) --- superset/config.py | 4 +++- superset/models/core.py | 20 ++++++++++++++------ superset/sql_lab.py | 5 ++++- superset/views/core.py | 16 ++++++++++------ tests/celery_tests.py | 3 ++- tests/core_tests.py | 16 ++++++++++++---- tests/db_engine_specs_test.py | 12 ++++++------ tox.ini | 1 - 8 files changed, 51 insertions(+), 26 deletions(-) diff --git a/superset/config.py b/superset/config.py index 89b024c314789..95ee3b0dc1173 100644 --- a/superset/config.py +++ b/superset/config.py @@ -348,7 +348,9 @@ class CeleryConfig(object): print('Loaded your LOCAL configuration at [{}]'.format( os.environ[CONFIG_PATH_ENV_VAR])) module = sys.modules[__name__] - override_conf = imp.load_source('superset_config', os.environ[CONFIG_PATH_ENV_VAR]) + override_conf = imp.load_source( + 'superset_config', + os.environ[CONFIG_PATH_ENV_VAR]) for key in dir(override_conf): if key.isupper(): setattr(module, key, getattr(override_conf, key)) diff --git a/superset/models/core.py b/superset/models/core.py index e067da4c2d115..130f63e0cbd02 100644 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -612,7 +612,10 @@ def get_effective_user(self, url, user_name=None): effective_username = url.username if user_name: effective_username = user_name - elif hasattr(g, 'user') and hasattr(g.user, 'username') and g.user.username is not None: + elif ( + hasattr(g, 'user') and hasattr(g.user, 'username') and + g.user.username is not None + ): effective_username = g.user.username return effective_username @@ -622,8 +625,12 @@ def get_sqla_engine(self, schema=None, nullpool=False, user_name=None): url = self.db_engine_spec.adjust_database_uri(url, schema) effective_username = self.get_effective_user(url, user_name) # If using MySQL or Presto for example, will set url.username - # If using Hive, will not do anything yet since that relies on a configuration parameter instead. - self.db_engine_spec.modify_url_for_impersonation(url, self.impersonate_user, effective_username) + # If using Hive, will not do anything yet since that relies on a + # configuration parameter instead. + self.db_engine_spec.modify_url_for_impersonation( + url, + self.impersonate_user, + effective_username) masked_url = self.get_password_masked_url(url) logging.info("Database.get_sqla_engine(). Masked URL: {0}".format(masked_url)) @@ -635,9 +642,10 @@ def get_sqla_engine(self, schema=None, nullpool=False, user_name=None): # If using Hive, this will set hive.server2.proxy.user=$effective_username configuration = {} configuration.update( - self.db_engine_spec.get_configuration_for_impersonation(str(url), - self.impersonate_user, - effective_username)) + self.db_engine_spec.get_configuration_for_impersonation( + str(url), + self.impersonate_user, + effective_username)) if configuration: params["connect_args"] = {"configuration": configuration} diff --git a/superset/sql_lab.py b/superset/sql_lab.py index 937f2222c6b2c..ab0f96009c756 100644 --- a/superset/sql_lab.py +++ b/superset/sql_lab.py @@ -176,7 +176,10 @@ def handle_error(msg): conn = None try: engine = database.get_sqla_engine( - schema=query.schema, nullpool=not ctask.request.called_directly, user_name=user_name) + schema=query.schema, + nullpool=not ctask.request.called_directly, + user_name=user_name, + ) conn = engine.raw_connection() cursor = conn.cursor() logging.info("Running query: \n{}".format(executed_sql)) diff --git a/superset/views/core.py b/superset/views/core.py index 3cbe7a62ecc87..c106556edc99b 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -238,10 +238,11 @@ class DatabaseView(SupersetModelView, DeleteMixin): # noqa "(http://docs.sqlalchemy.org/en/rel_1_0/core/metadata.html" "#sqlalchemy.schema.MetaData) call. ", True), 'impersonate_user': _( - "If Presto, all the queries in SQL Lab are going to be executed as the currently logged on user " - "who must have permission to run them.
" - "If Hive and hive.server2.enable.doAs is enabled, will run the queries as service account, " - "but impersonate the currently logged on user via hive.server2.proxy.user property."), + "If Presto, all the queries in SQL Lab are going to be executed as the " + "currently logged on user who must have permission to run them.
" + "If Hive and hive.server2.enable.doAs is enabled, will run the queries as " + "service account, but impersonate the currently logged on user " + "via hive.server2.proxy.user property."), } label_columns = { 'expose_in_sqllab': _("Expose in SQL Lab"), @@ -1102,7 +1103,9 @@ def explore(self, datasource_type, datasource_id): action = request.args.get('action') if action == 'overwrite' and not slice_overwrite_perm: - return json_error_response("You don't have the rights to alter this slice", status=400) + return json_error_response( + "You don't have the rights to alter this slice", + status=400) if action in ('saveas', 'overwrite'): return self.save_or_overwrite_slice( @@ -1462,7 +1465,8 @@ def testconn(self): if database and uri: url = make_url(uri) - db_engine = models.Database.get_db_engine_spec_for_backend(url.get_backend_name()) + db_engine = models.Database.get_db_engine_spec_for_backend( + url.get_backend_name()) db_engine.patch() masked_url = database.get_password_masked_url_from_uri(uri) diff --git a/tests/celery_tests.py b/tests/celery_tests.py index a825b4d345f6f..b171b5259db7b 100644 --- a/tests/celery_tests.py +++ b/tests/celery_tests.py @@ -27,7 +27,8 @@ class CeleryConfig(object): BROKER_URL = 'sqla+sqlite:///' + app.config.get('SQL_CELERY_DB_FILE_PATH') CELERY_IMPORTS = ('superset.sql_lab', ) - CELERY_RESULT_BACKEND = 'db+sqlite:///' + app.config.get('SQL_CELERY_RESULTS_DB_FILE_PATH') + CELERY_RESULT_BACKEND = ( + 'db+sqlite:///' + app.config.get('SQL_CELERY_RESULTS_DB_FILE_PATH')) CELERY_ANNOTATIONS = {'sql_lab.add': {'rate_limit': '10/s'}} CONCURRENCY = 1 diff --git a/tests/core_tests.py b/tests/core_tests.py index 04f4746c6cf00..44da324a80208 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -283,7 +283,10 @@ def test_testconn(self, username='admin'): 'name': 'main', 'impersonate_user': False, }) - response = self.client.post('/superset/testconn', data=data, content_type='application/json') + response = self.client.post( + '/superset/testconn', + data=data, + content_type='application/json') assert response.status_code == 200 assert response.headers['Content-Type'] == 'application/json' @@ -293,7 +296,10 @@ def test_testconn(self, username='admin'): 'name': 'main', 'impersonate_user': False, }) - response = self.client.post('/superset/testconn', data=data, content_type='application/json') + response = self.client.post( + '/superset/testconn', + data=data, + content_type='application/json') assert response.status_code == 200 assert response.headers['Content-Type'] == 'application/json' @@ -311,7 +317,8 @@ def custom_password_store(uri): assert conn.password != conn_pre.password def test_databaseview_edit(self, username='admin'): - # validate that sending a password-masked uri does not over-write the decrypted uri + # validate that sending a password-masked uri does not over-write the decrypted + # uri self.login(username=username) database = self.get_main_database(db.session) sqlalchemy_uri_decrypted = database.sqlalchemy_uri_decrypted @@ -740,7 +747,8 @@ def test_user_profile(self, username='admin'): self.assertNotIn('message', data) data = self.get_json_resp('/superset/fave_dashboards/{}/'.format(userid)) self.assertNotIn('message', data) - data = self.get_json_resp('/superset/fave_dashboards_by_username/{}/'.format(username)) + data = self.get_json_resp( + '/superset/fave_dashboards_by_username/{}/'.format(username)) self.assertNotIn('message', data) def test_slice_id_is_always_logged_correctly_on_web_request(self): diff --git a/tests/db_engine_specs_test.py b/tests/db_engine_specs_test.py index e348b90324a41..a2310d1d8098d 100644 --- a/tests/db_engine_specs_test.py +++ b/tests/db_engine_specs_test.py @@ -13,7 +13,7 @@ def test_0_progress(self): log = """ 17/02/07 18:26:27 INFO log.PerfLogger: 17/02/07 18:26:27 INFO log.PerfLogger: - """.split('\n') + """.split('\n') # noqa ignore: E501 self.assertEquals( 0, HiveEngineSpec.progress(log)) @@ -35,7 +35,7 @@ def test_job_1_launched_stage_1_0_progress(self): 17/02/07 19:15:55 INFO ql.Driver: Total jobs = 2 17/02/07 19:15:55 INFO ql.Driver: Launching Job 1 out of 2 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 0%, reduce = 0% - """.split('\n') + """.split('\n') # noqa ignore: E501 self.assertEquals(0, HiveEngineSpec.progress(log)) def test_job_1_launched_stage_1_map_40_progress(self): @@ -44,7 +44,7 @@ def test_job_1_launched_stage_1_map_40_progress(self): 17/02/07 19:15:55 INFO ql.Driver: Launching Job 1 out of 2 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 0%, reduce = 0% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 40%, reduce = 0% - """.split('\n') + """.split('\n') # noqa ignore: E501 self.assertEquals(10, HiveEngineSpec.progress(log)) def test_job_1_launched_stage_1_map_80_reduce_40_progress(self): @@ -54,7 +54,7 @@ def test_job_1_launched_stage_1_map_80_reduce_40_progress(self): 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 0%, reduce = 0% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 40%, reduce = 0% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 80%, reduce = 40% - """.split('\n') + """.split('\n') # noqa ignore: E501 self.assertEquals(30, HiveEngineSpec.progress(log)) def test_job_1_launched_stage_2_stages_progress(self): @@ -66,7 +66,7 @@ def test_job_1_launched_stage_2_stages_progress(self): 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 80%, reduce = 40% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-2 map = 0%, reduce = 0% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 100%, reduce = 0% - """.split('\n') + """.split('\n') # noqa ignore: E501 self.assertEquals(12, HiveEngineSpec.progress(log)) def test_job_2_launched_stage_2_stages_progress(self): @@ -77,5 +77,5 @@ def test_job_2_launched_stage_2_stages_progress(self): 17/02/07 19:15:55 INFO ql.Driver: Launching Job 2 out of 2 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 0%, reduce = 0% 17/02/07 19:16:09 INFO exec.Task: 2017-02-07 19:16:09,173 Stage-1 map = 40%, reduce = 0% - """.split('\n') + """.split('\n') # noqa ignore: E501 self.assertEquals(60, HiveEngineSpec.progress(log)) diff --git a/tox.ini b/tox.ini index fc926b97e270f..177858dfe3971 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,6 @@ exclude = superset/migrations superset/templates ignore = - E501 Q000 Q001 import-order-style = google From 500455fc72dc58f9c0bd32d2b3b4da9a915bfb8c Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Sun, 12 Nov 2017 11:28:37 -0800 Subject: [PATCH 12/13] Add CHANGELOG.md entries for 0.20.0 to 0.20.5 (#3842) --- CHANGELOG.md | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fac1cedd369ea..e1ed7cd24738a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,104 @@ ## Change Log +### 0.20.5 (2017/11/06 07:18 +00:00) +- [#3776](https://github.com/apache/incubator-superset/pull/3776) [flake8] Enabling flake8 linting (#3776) (@john-bodley) +- [#3774](https://github.com/apache/incubator-superset/pull/3774) [sql-lab] Fixing Run Query tooltip (#3774) (@john-bodley) +- [#3773](https://github.com/apache/incubator-superset/pull/3773) Fix dashboard export download (#3773) (@michellethomas) +- [#3767](https://github.com/apache/incubator-superset/pull/3767) [time table] add tooltip to sparkline (#3767) (@williaster) +- [#3748](https://github.com/apache/incubator-superset/pull/3748) Update to reflect new version of cryptography (#3748) (@SpyderRivera) +- [#3763](https://github.com/apache/incubator-superset/pull/3763) docs: reword the FAQ regarding table changes (#3763) (@xrmx) +- [#3764](https://github.com/apache/incubator-superset/pull/3764) add stackoverflow tag (#3764) (@dmigo) +- [#3759](https://github.com/apache/incubator-superset/pull/3759) Add dummy file to fix symlink (#3759) (@mistercrunch) +- [#3751](https://github.com/apache/incubator-superset/pull/3751) fix https://github.com/apache/incubator-superset/pull/3726 (#3751) (@graceguo-supercat) +- [#3750](https://github.com/apache/incubator-superset/pull/3750) Consolidate all translation config (#3750) (@alanmcruickshank) +- [#3726](https://github.com/apache/incubator-superset/pull/3726) Bumping react-select to rc10 (#3726) (@mistercrunch) +- [#3741](https://github.com/apache/incubator-superset/pull/3741) Fix has_table method (#3741) (@mxmzdlv) +- [#3736](https://github.com/apache/incubator-superset/pull/3736) Escape columns names for time grains - postgres (#3736) (@Ryanthegiantlion) +- [#3739](https://github.com/apache/incubator-superset/pull/3739) Fix 3657 (#3739) (@baldoalessandro) +- [#3733](https://github.com/apache/incubator-superset/pull/3733) Using indexOf instead of includes for isXAxisString (#3733) (@michellethomas) +- [#3723](https://github.com/apache/incubator-superset/pull/3723) bump react-bootstrap version (#3723) (@graceguo-supercat) +- [#3721](https://github.com/apache/incubator-superset/pull/3721) Add CRUD action to refresh table metadata (#3721) (@mistercrunch) +- [#3720](https://github.com/apache/incubator-superset/pull/3720) Validate JSON in slice's params on save (#3720) (@mistercrunch) +- [#3722](https://github.com/apache/incubator-superset/pull/3722) Fix box_plot NaN issue (#3722) (@mistercrunch) +- [#3715](https://github.com/apache/incubator-superset/pull/3715) Update messages.po (#3715) (@magicansk) +- [#3686](https://github.com/apache/incubator-superset/pull/3686) Missing the data of one province and two regions of China (#3686) (@roganw) +- [#3685](https://github.com/apache/incubator-superset/pull/3685) Fix the ISO code description of region/province/department (#3685) (@roganw) +- [#3662](https://github.com/apache/incubator-superset/pull/3662) Set logging level to debug for DummyStatsLogger (#3662) (@mistercrunch) +- [#3692](https://github.com/apache/incubator-superset/pull/3692) fixes for bugs in #3689 (#3692) (@Mogball) +- [#3703](https://github.com/apache/incubator-superset/pull/3703) add VIPKID to the orgs. (#3703) (@killpanda) +- [#3696](https://github.com/apache/incubator-superset/pull/3696) changed metric heading from h1 to h3 (#3696) (@Mogball) +- [#3713](https://github.com/apache/incubator-superset/pull/3713) [translation] added japanese support (#3713) (@xiaoyugit) +- [#3663](https://github.com/apache/incubator-superset/pull/3663) [minor] fix label showing description in time_table's URL (#3663) (@mistercrunch) +- [#3711](https://github.com/apache/incubator-superset/pull/3711) fix the slice permission issue after user click-edit new slice title (#3711) (@graceguo-supercat) +- [#3701](https://github.com/apache/incubator-superset/pull/3701) [form-data] Quoting form data (#3701) (@john-bodley) +- [#3698](https://github.com/apache/incubator-superset/pull/3698) fixing the datasource inconsistence but in visualize flow (#3698) (@graceguo-supercat) +- [#3683](https://github.com/apache/incubator-superset/pull/3683) [cleanup] removing print() artefacts (#3683) (@mistercrunch) +- [#3702](https://github.com/apache/incubator-superset/pull/3702) Add support for IE 11 for markup slices (#3702) (@jaylindquist) +- [#3693](https://github.com/apache/incubator-superset/pull/3693) defaultSort should be false when no sort is necessary (#3693) (@michellethomas) +- [#3586](https://github.com/apache/incubator-superset/pull/3586) [Feature] Percentage columns in Table Viz (#3586) (@Mogball) +- [#3652](https://github.com/apache/incubator-superset/pull/3652) DI-1113. Authentication: Enable user impersonation for Superset to HiveServer2 using hive.server2.proxy.user (a.fernandez) (#3652) (@afernandez) +- [#3664](https://github.com/apache/incubator-superset/pull/3664) [minor] fix padding in Time Table (#3664) (@mistercrunch) +- [#3678](https://github.com/apache/incubator-superset/pull/3678) unit tests for OptionDescription component (#3678) (@Mogball) +- [#3679](https://github.com/apache/incubator-superset/pull/3679) Avoid dividing by zero for sparkline in time table viz (#3679) (@michellethomas) +- [#3680](https://github.com/apache/incubator-superset/pull/3680) Sqllab error troubleshooting (#3680) (@timifasubaa) +- [#3653](https://github.com/apache/incubator-superset/pull/3653) Add a ColorPickerControl (#3653) (@mistercrunch) +- [#3642](https://github.com/apache/incubator-superset/pull/3642) [New Viz] Partition Diagram (#3642) (@Mogball) +- [#3665](https://github.com/apache/incubator-superset/pull/3665) Add description for running specific test (#3665) (@timifasubaa) +- [#3661](https://github.com/apache/incubator-superset/pull/3661) Making the sort order for metrics pull from fd for time table viz (#3661) (@michellethomas) +- [#3417](https://github.com/apache/incubator-superset/pull/3417) Make columns that return an exception on click unsortable. (#3417) (@aliavni) +- [#3651](https://github.com/apache/incubator-superset/pull/3651) Adding sort time table (#3651) (@michellethomas) +- [#3647](https://github.com/apache/incubator-superset/pull/3647) added aihello as superset user. (#3647) (@ganeshkrishnan1) +- [#3646](https://github.com/apache/incubator-superset/pull/3646) Fix #3612 - reverse sign in difference calculation (#3646) (@mistercrunch) +- [#3648](https://github.com/apache/incubator-superset/pull/3648) Fixing some warnings during tests (#3648) (@dennybiasiolli) + +### 0.20.4 (2017/10/12 04:04 +00:00) +- [#3645](https://github.com/apache/incubator-superset/pull/3645) [Translations] Restored lost French translations (#3645) (@Mogball) +- [#3644](https://github.com/apache/incubator-superset/pull/3644) [sql lab] fix impersonation + template issue (#3644) (@mistercrunch) +- [#3641](https://github.com/apache/incubator-superset/pull/3641) Pin moment.js library since 2.19.0 creates problem (#3641) (@mistercrunch) +- [#3600](https://github.com/apache/incubator-superset/pull/3600) [time_table] adding support for URLs / links (#3600) (@mistercrunch) +- [#3626](https://github.com/apache/incubator-superset/pull/3626) Set tooltip to show extent of sparkData (#3626) (@michellethomas) +- [#3631](https://github.com/apache/incubator-superset/pull/3631) add explicit message display for 'Fetching Annotation Layer' error (#3631) (@graceguo-supercat) +- [#3637](https://github.com/apache/incubator-superset/pull/3637) [bugfix] Template rendering failed: '_AppCtxGlobals' object has no attribute 'user' (#3637) (@mistercrunch) +- [#3638](https://github.com/apache/incubator-superset/pull/3638) fix long title text wrapping in editable-title component (#3638) (@graceguo-supercat) +- [#3625](https://github.com/apache/incubator-superset/pull/3625) [minor] proper tooltip on ControlHeader's instant re-render trigger (#3625) (@mistercrunch) +- [#3634](https://github.com/apache/incubator-superset/pull/3634) add annotation option and a linear color map for heatmap viz. (#3634) (@xiaoyugit) +- [#3633](https://github.com/apache/incubator-superset/pull/3633) [bugfix] empty From date filter NoneType error (#3633) (@mistercrunch) +- [#3621](https://github.com/apache/incubator-superset/pull/3621) remove unused imports (#3621) (@xrmx) +- [#3611](https://github.com/apache/incubator-superset/pull/3611) fixing date/time filter keys (#3611) (@Mogball) + +### 0.20.2 (2017/10/06 07:46 +00:00) +- [#3606](https://github.com/apache/incubator-superset/pull/3606) [bugfix] #3593 'Chart Options' panel is missing (#3606) (@mistercrunch) +- [#3601](https://github.com/apache/incubator-superset/pull/3601) Removing git artifact (#3601) (@mistercrunch) +- [#3599](https://github.com/apache/incubator-superset/pull/3599) [hotfix] fixing issues around new time_table viz (#3599) (@mistercrunch) +- [#3598](https://github.com/apache/incubator-superset/pull/3598) [hofix] work around circular deps (#3598) (@mistercrunch) +- [#3597](https://github.com/apache/incubator-superset/pull/3597) [time table] fix reversed ratio (#3597) (@mistercrunch) +- [#3508](https://github.com/apache/incubator-superset/pull/3508) [Feature/Bugfix] Datepicker and time granularity options to dashboard filters (#3508) (@Mogball) +- [#3596](https://github.com/apache/incubator-superset/pull/3596) updating react-alert dependency to v2.3.0 (#3596) (@dennybiasiolli) +- [#3577](https://github.com/apache/incubator-superset/pull/3577) [translations] generating missing strings (#3577) (@mistercrunch) +- [#3478](https://github.com/apache/incubator-superset/pull/3478) [Bugfix/Feature] Fixed slice render staggering on dashboard first load (#3478) (@Mogball) +- [#3543](https://github.com/apache/incubator-superset/pull/3543) New "Time Series - Table" visualization (#3543) (@mistercrunch) +- [#3587](https://github.com/apache/incubator-superset/pull/3587) [sql lab] fix numeric sort in data table (#3587) (@mistercrunch) +- [#3594](https://github.com/apache/incubator-superset/pull/3594) Fxing bug in label generation for multiple groupbys (#3594) (@fabianmenges) +- [#3591](https://github.com/apache/incubator-superset/pull/3591) update immutable.js to v3.8.2 (MIT license) (#3591) (@naoyak) +- [#3571](https://github.com/apache/incubator-superset/pull/3571) [Feature] Copy-to-clipboard button in View Query (#3571) (@Mogball) +- [#3585](https://github.com/apache/incubator-superset/pull/3585) Allow users to see query string when query returns no data (#3585) (@Mogball) +- [#3582](https://github.com/apache/incubator-superset/pull/3582) [Bugfix]: Explore view does not respect custom timeout. (#3582) (@fabianmenges) +- [#3584](https://github.com/apache/incubator-superset/pull/3584) Fixed creating new filter options in FilterBox (#3584) (@Mogball) +- [#3562](https://github.com/apache/incubator-superset/pull/3562) Added custom pasteSelect to handle paste events (#3562) (@Mogball) +- [#3569](https://github.com/apache/incubator-superset/pull/3569) Bumping React to 15.6.2 (MIT license) (#3569) (@mistercrunch) + +### 0.20.1 (2017/10/03 07:04 +00:00) +- [#3576](https://github.com/apache/incubator-superset/pull/3576) v0.20.1 (#3576) (@mistercrunch) +- [#3572](https://github.com/apache/incubator-superset/pull/3572) After saving slice fixing redirect (#3572) (@michellethomas) +- [#3565](https://github.com/apache/incubator-superset/pull/3565) Added label+percent and label+value display options to pie chart (#3565) (@Mogball) +- [#3567](https://github.com/apache/incubator-superset/pull/3567) Removing yarn warnings during install (#3567) (@dennybiasiolli) +- [#3563](https://github.com/apache/incubator-superset/pull/3563) [nvd3] fix single metric showing up in legend (#3563) (@mistercrunch) +- [#3558](https://github.com/apache/incubator-superset/pull/3558) Add Pronto Tools to user list (#3558) (@zkan) +- [#3553](https://github.com/apache/incubator-superset/pull/3553) Minor documentation fix (#3553) (@gaborhermann) +- [#3545](https://github.com/apache/incubator-superset/pull/3545) CHANGELOG for 0.20.0 (#3545) (@mistercrunch) +- [#3534](https://github.com/apache/incubator-superset/pull/3534) Explore update button labels (#3534) (@timifasubaa) +- [#3547](https://github.com/apache/incubator-superset/pull/3547) Fixing missing messages.json file (#3547) (@mistercrunch) + ### 0.20.0 (2017/09/28 04:26 +00:00) - [#3528](https://github.com/apache/incubator-superset/pull/3528) try to fix problem that chrome window not opening after ajax requrest (#3528) (@graceguo-supercat) - [#3521](https://github.com/apache/incubator-superset/pull/3521) Time Series Annotation Layers (#3521) (@graceguo-supercat) From 068c343be0e7236117b8ed99b58abef841ed4cf5 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Sun, 12 Nov 2017 21:24:20 -0800 Subject: [PATCH 13/13] [sqllab] fix wrong error msg (#3849) I was getting some "Could not connect to server" when there was a proper json payload with an `error` key, the change here makes sure to prioritize those messages over the generic one. --- superset/assets/javascripts/SqlLab/actions.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/superset/assets/javascripts/SqlLab/actions.js b/superset/assets/javascripts/SqlLab/actions.js index e5563020b674d..2541ee585646b 100644 --- a/superset/assets/javascripts/SqlLab/actions.js +++ b/superset/assets/javascripts/SqlLab/actions.js @@ -153,10 +153,12 @@ export function runQuery(query) { msg = err.responseText; } } - if (textStatus === 'error' && errorThrown === '') { - msg = t('Could not connect to server'); - } else if (msg === null) { - msg = `[${textStatus}] ${errorThrown}`; + if (msg === null) { + if (errorThrown) { + msg = `[${textStatus}] ${errorThrown}`; + } else { + msg = t('Unknown error'); + } } if (msg.indexOf('CSRF token') > 0) { msg = t('Your session timed out, please refresh your page and try again.');