From 90eef519f3bd7e04bd5cea8201893d42c8757f5f Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Fri, 3 May 2019 15:10:43 -0700 Subject: [PATCH] feat: Scheduling queries from SQL Lab (#7416) * Lightweight pipelines POC * Add docs * Minor fixes * Remove Lyft URL * Use enum * Minor fix * Fix unit tests * Mark props as required --- docs/installation.rst | 78 +++++++++++++ superset/assets/package-lock.json | 46 +++++++- superset/assets/package.json | 1 + .../SqlLab/components/ScheduleQueryButton.jsx | 109 ++++++++++++++++++ .../src/SqlLab/components/SqlEditor.jsx | 14 +++ superset/assets/src/featureFlags.ts | 3 +- superset/views/sql_lab.py | 7 +- 7 files changed, 250 insertions(+), 8 deletions(-) create mode 100644 superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx diff --git a/docs/installation.rst b/docs/installation.rst index a06da876dc729..0f372bdbed1d3 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -816,6 +816,84 @@ in this dictionary are made available for users to use in their SQL. 'my_crazy_macro': lambda x: x*2, } +**Scheduling queries** + +You can optionally allow your users to schedule queries directly in SQL Lab. +This is done by addding extra metadata to saved queries, which are then picked +up by an external scheduled (like [Apache Airflow](https://airflow.apache.org/)). + +To allow scheduled queries, add the following to your `config.py`: + +.. code-block:: python + + FEATURE_FLAGS = { + # Configuration for scheduling queries from SQL Lab. This information is + # collected when the user clicks "Schedule query", and saved into the `extra` + # field of saved queries. + # See: https://github.com/mozilla-services/react-jsonschema-form + 'SCHEDULED_QUERIES': { + 'JSONSCHEMA': { + 'title': 'Schedule', + 'description': ( + 'In order to schedule a query, you need to specify when it ' + 'should start running, when it should stop running, and how ' + 'often it should run. You can also optionally specify ' + 'dependencies that should be met before the query is ' + 'executed. Please read the documentation for best practices ' + 'and more information on how to specify dependencies.' + ), + 'type': 'object', + 'properties': { + 'output_table': { + 'type': 'string', + 'title': 'Output table name', + }, + 'start_date': { + 'type': 'string', + 'format': 'date-time', + 'title': 'Start date', + }, + 'end_date': { + 'type': 'string', + 'format': 'date-time', + 'title': 'End date', + }, + 'schedule_interval': { + 'type': 'string', + 'title': 'Schedule interval', + }, + 'dependencies': { + 'type': 'array', + 'title': 'Dependencies', + 'items': { + 'type': 'string', + }, + }, + }, + }, + 'UISCHEMA': { + 'schedule_interval': { + 'ui:placeholder': '@daily, @weekly, etc.', + }, + 'dependencies': { + 'ui:help': ( + 'Check the documentation for the correct format when ' + 'defining dependencies.' + ), + }, + }, + }, + } + +This feature flag is based on [react-jsonschema-form](https://github.com/mozilla-services/react-jsonschema-form), +and will add a button called "Schedule Query" to SQL Lab. When the button is +clicked, a modal will show up where the user can add the metadata required for +scheduling the query. + +This information can then be retrieved from the endpoint `/savedqueryviewapi/api/read` +and used to schedule the queries that have `scheduled_queries` in their JSON +metadata. For schedulers other than Airflow, additional fields can be easily +added to the configuration file above. Celery Flower ------------- diff --git a/superset/assets/package-lock.json b/superset/assets/package-lock.json index 704e6f3afd8aa..e5075d9699665 100644 --- a/superset/assets/package-lock.json +++ b/superset/assets/package-lock.json @@ -5791,8 +5791,7 @@ "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=", - "dev": true + "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" }, "coa": { "version": "2.0.2", @@ -6071,8 +6070,7 @@ "core-js": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.0.tgz", - "integrity": "sha512-kLRC6ncVpuEW/1kwrOXYX6KQASCVtrh1gQr/UiaVgFlf9WE5Vp+lNe5+h3LuMr5PAucWnnEXwH0nQHRH/gpGtw==", - "dev": true + "integrity": "sha512-kLRC6ncVpuEW/1kwrOXYX6KQASCVtrh1gQr/UiaVgFlf9WE5Vp+lNe5+h3LuMr5PAucWnnEXwH0nQHRH/gpGtw==" }, "core-util-is": { "version": "1.0.2", @@ -13233,6 +13231,11 @@ "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=", "dev": true }, + "lodash.topath": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/lodash.topath/-/lodash.topath-4.5.2.tgz", + "integrity": "sha1-NhY1Hzu6YZlKCTGYlmC9AyVP0Ak=" + }, "lodash.uniq": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", @@ -17458,6 +17461,41 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.6.3.tgz", "integrity": "sha512-u7FDWtthB4rWibG/+mFbVd5FvdI20yde86qKGx4lVUTWmPlSWQ4QxbBIrrs+HnXGbxOUlUzTAP/VDmvCwaP2yA==" }, + "react-jsonschema-form": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/react-jsonschema-form/-/react-jsonschema-form-1.2.0.tgz", + "integrity": "sha512-rR77qoFiQ5TxDYwsJz8UWmDner4jQ4xMnDqeV6Nvg7GtoEyOUoTVkI/SBMEzfXuF/piWZXYjquP96Hy/2L7C+Q==", + "requires": { + "ajv": "^5.2.3", + "babel-runtime": "^6.26.0", + "core-js": "^2.5.7", + "lodash.topath": "^4.5.2", + "prop-types": "^15.5.8" + }, + "dependencies": { + "ajv": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", + "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", + "requires": { + "co": "^4.6.0", + "fast-deep-equal": "^1.0.0", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.3.0" + } + }, + "fast-deep-equal": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", + "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=" + }, + "json-schema-traverse": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", + "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" + } + } + }, "react-lifecycles-compat": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz", diff --git a/superset/assets/package.json b/superset/assets/package.json index 6914c736490b9..6edb971331dc9 100644 --- a/superset/assets/package.json +++ b/superset/assets/package.json @@ -117,6 +117,7 @@ "react-dom": "^16.4.1", "react-gravatar": "^2.6.1", "react-hot-loader": "^4.3.6", + "react-jsonschema-form": "^1.2.0", "react-map-gl": "^4.0.10", "react-markdown": "^3.3.0", "react-redux": "^5.0.2", diff --git a/superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx b/superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx new file mode 100644 index 0000000000000..2e7e16e3167af --- /dev/null +++ b/superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import Form from 'react-jsonschema-form'; +import { t } from '@superset-ui/translation'; + +import Button from '../../components/Button'; +import ModalTrigger from '../../components/ModalTrigger'; + +const propTypes = { + defaultLabel: PropTypes.string, + sql: PropTypes.string.isRequired, + schema: PropTypes.string.isRequired, + dbId: PropTypes.number.isRequired, + animation: PropTypes.bool, + onSchedule: PropTypes.func, +}; +const defaultProps = { + defaultLabel: t('Undefined'), + animation: true, + onSchedule: () => {}, +}; + +class ScheduleQueryButton extends React.PureComponent { + constructor(props) { + super(props); + this.state = { + description: '', + label: props.defaultLabel, + showSchedule: false, + }; + this.toggleSchedule = this.toggleSchedule.bind(this); + this.onSchedule = this.onSchedule.bind(this); + this.onCancel = this.onCancel.bind(this); + this.onLabelChange = this.onLabelChange.bind(this); + this.onDescriptionChange = this.onDescriptionChange.bind(this); + } + onSchedule({ formData }) { + const query = { + label: this.state.label, + description: this.state.description, + db_id: this.props.dbId, + schema: this.props.schema, + sql: this.props.sql, + extra_json: JSON.stringify({ schedule_info: formData }), + }; + this.props.onSchedule(query); + this.saveModal.close(); + } + onCancel() { + this.saveModal.close(); + } + onLabelChange(e) { + this.setState({ label: e.target.value }); + } + onDescriptionChange(e) { + this.setState({ description: e.target.value }); + } + toggleSchedule(e) { + this.setState({ target: e.target, showSchedule: !this.state.showSchedule }); + } + renderModalBody() { + return ( +
+ ); + } + render() { + return ( + + { this.saveModal = ref; }} + modalTitle={t('Schedule Query')} + modalBody={this.renderModalBody()} + triggerNode={ + + } + bsSize="medium" + /> + + ); + } +} +ScheduleQueryButton.propTypes = propTypes; +ScheduleQueryButton.defaultProps = defaultProps; + +export default ScheduleQueryButton; diff --git a/superset/assets/src/SqlLab/components/SqlEditor.jsx b/superset/assets/src/SqlLab/components/SqlEditor.jsx index 960a4af15ef50..f4495af2a4dc6 100644 --- a/superset/assets/src/SqlLab/components/SqlEditor.jsx +++ b/superset/assets/src/SqlLab/components/SqlEditor.jsx @@ -36,6 +36,7 @@ import LimitControl from './LimitControl'; import TemplateParamsEditor from './TemplateParamsEditor'; import SouthPane from './SouthPane'; import SaveQuery from './SaveQuery'; +import ScheduleQueryButton from './ScheduleQueryButton'; import ShareSqlLabQuery from './ShareSqlLabQuery'; import Timer from '../../components/Timer'; import Hotkeys from '../../components/Hotkeys'; @@ -43,6 +44,7 @@ import SqlEditorLeftBar from './SqlEditorLeftBar'; import AceEditorWrapper from './AceEditorWrapper'; import { STATE_BSSTYLE_MAP } from '../constants'; import RunQueryActionButton from './RunQueryActionButton'; +import { FeatureFlag, isFeatureEnabled } from '../../featureFlags'; const SQL_EDITOR_PADDING = 10; const SQL_TOOLBAR_HEIGHT = 51; @@ -313,6 +315,18 @@ class SqlEditor extends React.PureComponent { sql={this.state.sql} /> + {isFeatureEnabled(FeatureFlag.SCHEDULED_QUERIES) && + + + + }