From c605d33dff77fb0315b8ad332ad976562c39a0cc Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Fri, 1 Mar 2019 11:27:23 -0800 Subject: [PATCH 1/8] Lightweight pipelines POC --- superset/assets/package-lock.json | 46 +++++++- superset/assets/package.json | 3 +- .../SqlLab/components/ScheduleQueryButton.jsx | 109 ++++++++++++++++++ .../src/SqlLab/components/SqlEditor.jsx | 14 +++ superset/assets/src/featureFlags.ts | 1 + superset/views/sql_lab.py | 7 +- 6 files changed, 172 insertions(+), 8 deletions(-) create mode 100644 superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx diff --git a/superset/assets/package-lock.json b/superset/assets/package-lock.json index 704e6f3afd8aa..e5075d9699665 100644 --- a/superset/assets/package-lock.json +++ b/superset/assets/package-lock.json @@ -5791,8 +5791,7 @@ "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=", - "dev": true + "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" }, "coa": { "version": "2.0.2", @@ -6071,8 +6070,7 @@ "core-js": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.0.tgz", - "integrity": "sha512-kLRC6ncVpuEW/1kwrOXYX6KQASCVtrh1gQr/UiaVgFlf9WE5Vp+lNe5+h3LuMr5PAucWnnEXwH0nQHRH/gpGtw==", - "dev": true + "integrity": "sha512-kLRC6ncVpuEW/1kwrOXYX6KQASCVtrh1gQr/UiaVgFlf9WE5Vp+lNe5+h3LuMr5PAucWnnEXwH0nQHRH/gpGtw==" }, "core-util-is": { "version": "1.0.2", @@ -13233,6 +13231,11 @@ "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=", "dev": true }, + "lodash.topath": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/lodash.topath/-/lodash.topath-4.5.2.tgz", + "integrity": "sha1-NhY1Hzu6YZlKCTGYlmC9AyVP0Ak=" + }, "lodash.uniq": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", @@ -17458,6 +17461,41 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.6.3.tgz", "integrity": "sha512-u7FDWtthB4rWibG/+mFbVd5FvdI20yde86qKGx4lVUTWmPlSWQ4QxbBIrrs+HnXGbxOUlUzTAP/VDmvCwaP2yA==" }, + "react-jsonschema-form": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/react-jsonschema-form/-/react-jsonschema-form-1.2.0.tgz", + "integrity": "sha512-rR77qoFiQ5TxDYwsJz8UWmDner4jQ4xMnDqeV6Nvg7GtoEyOUoTVkI/SBMEzfXuF/piWZXYjquP96Hy/2L7C+Q==", + "requires": { + "ajv": "^5.2.3", + "babel-runtime": "^6.26.0", + "core-js": "^2.5.7", + "lodash.topath": "^4.5.2", + "prop-types": "^15.5.8" + }, + "dependencies": { + "ajv": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", + "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", + "requires": { + "co": "^4.6.0", + "fast-deep-equal": "^1.0.0", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.3.0" + } + }, + "fast-deep-equal": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", + "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=" + }, + "json-schema-traverse": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", + "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" + } + } + }, "react-lifecycles-compat": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz", diff --git a/superset/assets/package.json b/superset/assets/package.json index 6914c736490b9..16fb3b26a44a5 100644 --- a/superset/assets/package.json +++ b/superset/assets/package.json @@ -117,7 +117,8 @@ "react-dom": "^16.4.1", "react-gravatar": "^2.6.1", "react-hot-loader": "^4.3.6", - "react-map-gl": "^4.0.10", + "react-jsonschema-form": "^1.2.0", + "react-map-gl": "^3.0.4", "react-markdown": "^3.3.0", "react-redux": "^5.0.2", "react-search-input": "^0.11.3", diff --git a/superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx b/superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx new file mode 100644 index 0000000000000..f2a4a924bc807 --- /dev/null +++ b/superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import Form from 'react-jsonschema-form'; +import { t } from '@superset-ui/translation'; + +import Button from '../../components/Button'; +import ModalTrigger from '../../components/ModalTrigger'; + +const propTypes = { + defaultLabel: PropTypes.string, + sql: PropTypes.string, + schema: PropTypes.string, + dbId: PropTypes.number, + animation: PropTypes.bool, + onSchedule: PropTypes.func, +}; +const defaultProps = { + defaultLabel: t('Undefined'), + animation: true, + onSchedule: () => {}, +}; + +class ScheduleQueryButton extends React.PureComponent { + constructor(props) { + super(props); + this.state = { + description: '', + label: props.defaultLabel, + showSchedule: false, + }; + this.toggleSchedule = this.toggleSchedule.bind(this); + this.onSchedule = this.onSchedule.bind(this); + this.onCancel = this.onCancel.bind(this); + this.onLabelChange = this.onLabelChange.bind(this); + this.onDescriptionChange = this.onDescriptionChange.bind(this); + } + onSchedule({ formData }) { + const query = { + label: this.state.label, + description: this.state.description, + db_id: this.props.dbId, + schema: this.props.schema, + sql: this.props.sql, + extra_json: JSON.stringify({ schedule_info: formData }), + }; + this.props.onSchedule(query); + this.saveModal.close(); + } + onCancel() { + this.saveModal.close(); + } + onLabelChange(e) { + this.setState({ label: e.target.value }); + } + onDescriptionChange(e) { + this.setState({ description: e.target.value }); + } + toggleSchedule(e) { + this.setState({ target: e.target, showSchedule: !this.state.showSchedule }); + } + renderModalBody() { + return ( +
+ ); + } + render() { + return ( + + { this.saveModal = ref; }} + modalTitle={t('Schedule Query')} + modalBody={this.renderModalBody()} + triggerNode={ + + } + bsSize="medium" + /> + + ); + } +} +ScheduleQueryButton.propTypes = propTypes; +ScheduleQueryButton.defaultProps = defaultProps; + +export default ScheduleQueryButton; diff --git a/superset/assets/src/SqlLab/components/SqlEditor.jsx b/superset/assets/src/SqlLab/components/SqlEditor.jsx index 960a4af15ef50..cfab50eb6d6d3 100644 --- a/superset/assets/src/SqlLab/components/SqlEditor.jsx +++ b/superset/assets/src/SqlLab/components/SqlEditor.jsx @@ -36,6 +36,7 @@ import LimitControl from './LimitControl'; import TemplateParamsEditor from './TemplateParamsEditor'; import SouthPane from './SouthPane'; import SaveQuery from './SaveQuery'; +import ScheduleQueryButton from './ScheduleQueryButton'; import ShareSqlLabQuery from './ShareSqlLabQuery'; import Timer from '../../components/Timer'; import Hotkeys from '../../components/Hotkeys'; @@ -43,6 +44,7 @@ import SqlEditorLeftBar from './SqlEditorLeftBar'; import AceEditorWrapper from './AceEditorWrapper'; import { STATE_BSSTYLE_MAP } from '../constants'; import RunQueryActionButton from './RunQueryActionButton'; +import { isFeatureEnabled } from '../../featureFlags'; const SQL_EDITOR_PADDING = 10; const SQL_TOOLBAR_HEIGHT = 51; @@ -313,6 +315,18 @@ class SqlEditor extends React.PureComponent { sql={this.state.sql} /> + {isFeatureEnabled('SCHEDULED_QUERIES') && + + + + } Date: Tue, 30 Apr 2019 13:07:14 -0700 Subject: [PATCH 2/8] Add docs --- docs/installation.rst | 78 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index a06da876dc729..a5562a89b9365 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -816,6 +816,84 @@ in this dictionary are made available for users to use in their SQL. 'my_crazy_macro': lambda x: x*2, } +**Scheduling queries** + +You can optionally allow your users to schedule queries directly in SQL Lab. +This is done by addding extra metadata to saved queries, which are then picked +up by an external scheduled (like [Apache Airflow](https://airflow.apache.org/)). + +To allow scheduled queries, add the following to your `config.py`: + +.. code-block:: python + + FEATURE_FLAGS = { + # Configuration for scheduling queries from SQL Lab. This information is + # collected when the user clicks "Schedule query", and saved into the `extra` + # field of saved queries. + # See: https://github.com/mozilla-services/react-jsonschema-form + 'SCHEDULED_QUERIES': { + 'JSONSCHEMA': { + 'title': 'Schedule', + 'description': ( + 'In order to schedule a query, you need to specify when it ' + 'should start running, when it should stop running, and how ' + 'often it should run. You can also optionally specify ' + 'dependencies that should be met before the query is ' + 'executed. Please read the documentation for best practices ' + 'and more information on how to specify dependencies.' + ), + 'type': 'object', + 'properties': { + 'output_table': { + 'type': 'string', + 'title': 'Output table name', + }, + 'start_date': { + 'type': 'string', + 'format': 'date-time', + 'title': 'Start date', + }, + 'end_date': { + 'type': 'string', + 'format': 'date-time', + 'title': 'End date', + }, + 'schedule_interval': { + 'type': 'string', + 'title': 'Schedule interval', + }, + 'dependencies': { + 'type': 'array', + 'title': 'Dependencies', + 'items': { + 'type': 'string', + }, + }, + }, + }, + 'UISCHEMA': { + 'schedule_interval': { + 'ui:placeholder': '@daily, @weekly, etc.', + }, + 'dependencies': { + 'ui:help': ( + 'Check the documentation for the correct format when ' + 'defining dependencies.' + ), + }, + }, + }, + } + +This feature flag is based on [react-jsonschema-form](https://github.com/mozilla-services/react-jsonschema-form), +and will add a button called "Schedule Query" to SQL Lab. When the button is +clicked, a modal will show up where the user can add the metadata required for +scheduling the query. + +This information can then be retrieved from the endpoint https://superset-staging.lyft.net/savedqueryviewapi/api/read +and used to schedule the queries that have `scheduled_queries` in their JSON +metadata. For schedulers other than Airflow, additional fields can be easily +added to the configuration file above. Celery Flower ------------- From bc038833a2764038ed530b97eb61cc67a6f6f8f9 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Tue, 30 Apr 2019 13:09:33 -0700 Subject: [PATCH 3/8] Minor fixes --- docs/installation.rst | 116 +++++++++++++++++------------------ superset/assets/package.json | 2 +- 2 files changed, 59 insertions(+), 59 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index a5562a89b9365..456df0cfd930d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -826,64 +826,64 @@ To allow scheduled queries, add the following to your `config.py`: .. code-block:: python - FEATURE_FLAGS = { - # Configuration for scheduling queries from SQL Lab. This information is - # collected when the user clicks "Schedule query", and saved into the `extra` - # field of saved queries. - # See: https://github.com/mozilla-services/react-jsonschema-form - 'SCHEDULED_QUERIES': { - 'JSONSCHEMA': { - 'title': 'Schedule', - 'description': ( - 'In order to schedule a query, you need to specify when it ' - 'should start running, when it should stop running, and how ' - 'often it should run. You can also optionally specify ' - 'dependencies that should be met before the query is ' - 'executed. Please read the documentation for best practices ' - 'and more information on how to specify dependencies.' - ), - 'type': 'object', - 'properties': { - 'output_table': { - 'type': 'string', - 'title': 'Output table name', - }, - 'start_date': { - 'type': 'string', - 'format': 'date-time', - 'title': 'Start date', - }, - 'end_date': { - 'type': 'string', - 'format': 'date-time', - 'title': 'End date', - }, - 'schedule_interval': { - 'type': 'string', - 'title': 'Schedule interval', - }, - 'dependencies': { - 'type': 'array', - 'title': 'Dependencies', - 'items': { - 'type': 'string', - }, - }, - }, - }, - 'UISCHEMA': { - 'schedule_interval': { - 'ui:placeholder': '@daily, @weekly, etc.', - }, - 'dependencies': { - 'ui:help': ( - 'Check the documentation for the correct format when ' - 'defining dependencies.' - ), - }, - }, - }, - } + FEATURE_FLAGS = { + # Configuration for scheduling queries from SQL Lab. This information is + # collected when the user clicks "Schedule query", and saved into the `extra` + # field of saved queries. + # See: https://github.com/mozilla-services/react-jsonschema-form + 'SCHEDULED_QUERIES': { + 'JSONSCHEMA': { + 'title': 'Schedule', + 'description': ( + 'In order to schedule a query, you need to specify when it ' + 'should start running, when it should stop running, and how ' + 'often it should run. You can also optionally specify ' + 'dependencies that should be met before the query is ' + 'executed. Please read the documentation for best practices ' + 'and more information on how to specify dependencies.' + ), + 'type': 'object', + 'properties': { + 'output_table': { + 'type': 'string', + 'title': 'Output table name', + }, + 'start_date': { + 'type': 'string', + 'format': 'date-time', + 'title': 'Start date', + }, + 'end_date': { + 'type': 'string', + 'format': 'date-time', + 'title': 'End date', + }, + 'schedule_interval': { + 'type': 'string', + 'title': 'Schedule interval', + }, + 'dependencies': { + 'type': 'array', + 'title': 'Dependencies', + 'items': { + 'type': 'string', + }, + }, + }, + }, + 'UISCHEMA': { + 'schedule_interval': { + 'ui:placeholder': '@daily, @weekly, etc.', + }, + 'dependencies': { + 'ui:help': ( + 'Check the documentation for the correct format when ' + 'defining dependencies.' + ), + }, + }, + }, + } This feature flag is based on [react-jsonschema-form](https://github.com/mozilla-services/react-jsonschema-form), and will add a button called "Schedule Query" to SQL Lab. When the button is diff --git a/superset/assets/package.json b/superset/assets/package.json index 16fb3b26a44a5..6edb971331dc9 100644 --- a/superset/assets/package.json +++ b/superset/assets/package.json @@ -118,7 +118,7 @@ "react-gravatar": "^2.6.1", "react-hot-loader": "^4.3.6", "react-jsonschema-form": "^1.2.0", - "react-map-gl": "^3.0.4", + "react-map-gl": "^4.0.10", "react-markdown": "^3.3.0", "react-redux": "^5.0.2", "react-search-input": "^0.11.3", From 5af5c94ab2299f5cd2ac47396fd8e2acd54c3f95 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Tue, 30 Apr 2019 13:22:49 -0700 Subject: [PATCH 4/8] Remove Lyft URL --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 456df0cfd930d..0f372bdbed1d3 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -890,7 +890,7 @@ and will add a button called "Schedule Query" to SQL Lab. When the button is clicked, a modal will show up where the user can add the metadata required for scheduling the query. -This information can then be retrieved from the endpoint https://superset-staging.lyft.net/savedqueryviewapi/api/read +This information can then be retrieved from the endpoint `/savedqueryviewapi/api/read` and used to schedule the queries that have `scheduled_queries` in their JSON metadata. For schedulers other than Airflow, additional fields can be easily added to the configuration file above. From 393bc42647d5010e6d0773ccb98dae1382d5c9eb Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Tue, 30 Apr 2019 13:29:13 -0700 Subject: [PATCH 5/8] Use enum --- superset/assets/src/SqlLab/components/SqlEditor.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/superset/assets/src/SqlLab/components/SqlEditor.jsx b/superset/assets/src/SqlLab/components/SqlEditor.jsx index cfab50eb6d6d3..f4495af2a4dc6 100644 --- a/superset/assets/src/SqlLab/components/SqlEditor.jsx +++ b/superset/assets/src/SqlLab/components/SqlEditor.jsx @@ -44,7 +44,7 @@ import SqlEditorLeftBar from './SqlEditorLeftBar'; import AceEditorWrapper from './AceEditorWrapper'; import { STATE_BSSTYLE_MAP } from '../constants'; import RunQueryActionButton from './RunQueryActionButton'; -import { isFeatureEnabled } from '../../featureFlags'; +import { FeatureFlag, isFeatureEnabled } from '../../featureFlags'; const SQL_EDITOR_PADDING = 10; const SQL_TOOLBAR_HEIGHT = 51; @@ -315,7 +315,7 @@ class SqlEditor extends React.PureComponent { sql={this.state.sql} /> - {isFeatureEnabled('SCHEDULED_QUERIES') && + {isFeatureEnabled(FeatureFlag.SCHEDULED_QUERIES) && Date: Tue, 30 Apr 2019 14:59:12 -0700 Subject: [PATCH 6/8] Minor fix --- superset/assets/src/SqlLab/components/SqlEditor.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/assets/src/SqlLab/components/SqlEditor.jsx b/superset/assets/src/SqlLab/components/SqlEditor.jsx index f4495af2a4dc6..d24a70dbeee97 100644 --- a/superset/assets/src/SqlLab/components/SqlEditor.jsx +++ b/superset/assets/src/SqlLab/components/SqlEditor.jsx @@ -315,7 +315,7 @@ class SqlEditor extends React.PureComponent { sql={this.state.sql} /> - {isFeatureEnabled(FeatureFlag.SCHEDULED_QUERIES) && + {FeatureFlag && isFeatureEnabled(FeatureFlag.SCHEDULED_QUERIES) && Date: Tue, 30 Apr 2019 21:15:02 -0700 Subject: [PATCH 7/8] Fix unit tests --- superset/assets/src/SqlLab/components/SqlEditor.jsx | 2 +- superset/assets/src/featureFlags.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/superset/assets/src/SqlLab/components/SqlEditor.jsx b/superset/assets/src/SqlLab/components/SqlEditor.jsx index d24a70dbeee97..f4495af2a4dc6 100644 --- a/superset/assets/src/SqlLab/components/SqlEditor.jsx +++ b/superset/assets/src/SqlLab/components/SqlEditor.jsx @@ -315,7 +315,7 @@ class SqlEditor extends React.PureComponent { sql={this.state.sql} /> - {FeatureFlag && isFeatureEnabled(FeatureFlag.SCHEDULED_QUERIES) && + {isFeatureEnabled(FeatureFlag.SCHEDULED_QUERIES) && Date: Wed, 1 May 2019 12:29:46 -0700 Subject: [PATCH 8/8] Mark props as required --- .../assets/src/SqlLab/components/ScheduleQueryButton.jsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx b/superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx index f2a4a924bc807..2e7e16e3167af 100644 --- a/superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx +++ b/superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx @@ -26,9 +26,9 @@ import ModalTrigger from '../../components/ModalTrigger'; const propTypes = { defaultLabel: PropTypes.string, - sql: PropTypes.string, - schema: PropTypes.string, - dbId: PropTypes.number, + sql: PropTypes.string.isRequired, + schema: PropTypes.string.isRequired, + dbId: PropTypes.number.isRequired, animation: PropTypes.bool, onSchedule: PropTypes.func, };