Enable databricks support on Web

Website/Packages/datax-common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datax-common",
-  "version": "1.2.2-SNAPSHOT-5-21",
+  "version": "1.3.0-SNAPSHOT-1",
   "description": "Common UX, styles, utilities, and modules",
   "author": "Microsoft",
   "license": "MIT",

Website/Packages/datax-home/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datax-home",
-  "version": "1.2.2-SNAPSHOT-5-21",
+  "version": "1.3.0-SNAPSHOT-1",
   "description": "Home page",
   "author": "Microsoft",
   "license": "MIT",
@@ -30,7 +30,7 @@
     "css-loader": "1.0.1",
     "file-loader": "2.0.0",
     "mini-css-extract-plugin": "0.4.4",
-    "datax-common": "1.2.2-SNAPSHOT-5-21",
+    "datax-common": "1.3.0-SNAPSHOT-1",
     "office-ui-fabric-react": "6.111.2",
     "q": "1.5.1",
     "react": "16.6.3",

Website/Packages/datax-jobs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datax-jobs",
-  "version": "1.2.2-SNAPSHOT-5-21",
+  "version": "1.3.0-SNAPSHOT-1",
   "description": "Job features",
   "author": "Microsoft",
   "license": "MIT",
@@ -30,7 +30,7 @@
     "css-loader": "1.0.1",
     "file-loader": "2.0.0",
     "mini-css-extract-plugin": "0.4.4",
-    "datax-common": "1.2.2-SNAPSHOT-5-21",
+    "datax-common": "1.3.0-SNAPSHOT-1",
     "office-ui-fabric-react": "6.111.2",
     "q": "1.5.1",
     "prop-types": "15.6.2",

Website/Packages/datax-metrics/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datax-metrics",
-  "version": "1.2.2-SNAPSHOT-5-21",
+  "version": "1.3.0-SNAPSHOT-1",
   "description": "Metric features",
   "author": "Microsoft",
   "license": "MIT",
@@ -33,7 +33,7 @@
     "d3": "4.10.2",
     "file-loader": "2.0.0",
     "mini-css-extract-plugin": "0.4.4",
-    "datax-common": "1.2.2-SNAPSHOT-5-21",
+    "datax-common": "1.3.0-SNAPSHOT-1",
     "office-ui-fabric-react": "6.111.2",
     "q": "1.5.1",
     "plotly.js": "1.44.4",

Website/Packages/datax-pipeline/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datax-pipeline",
-  "version": "1.2.2-SNAPSHOT-5-21",
+  "version": "1.3.0-SNAPSHOT-1",
   "description": "Pipeline features",
   "author": "Microsoft",
   "license": "MIT",
@@ -34,7 +34,7 @@
     "jsoneditor-react": "1.0.0",
     "mini-css-extract-plugin": "0.4.4",
     "monaco-editor-webpack-plugin": "1.6.0",
-    "datax-common": "1.2.2-SNAPSHOT-5-21",
+    "datax-common": "1.3.0-SNAPSHOT-1",
     "office-ui-fabric-react": "6.111.2",
     "q": "1.5.1",
     "promise-polyfill": "8.1.0",

Website/Packages/datax-pipeline/src/common/api.js

@@ -6,3 +6,4 @@ import { nodeServiceGetApi } from 'datax-common';
 import { ApiNames } from './apiConstants';
 
 export const functionEnabled = () => nodeServiceGetApi(ApiNames.FunctionEnabled);
+export const isDatabricksSparkType = () => nodeServiceGetApi(ApiNames.IsDatabricksSparkType);

Website/Packages/datax-pipeline/src/common/apiConstants.js

@@ -20,7 +20,7 @@ export const Constants = {
 
 export const ApiNames = {
     FunctionEnabled: 'functionenabled',
-
+    IsDatabricksSparkType: 'isdatabrickssparktype',
     // OneBox
     EnableLocalOneBox: 'enableLocalOneBox'
 };

Website/Packages/datax-pipeline/src/modules/flowDefinition/components/flowDefinitionPanel.jsx

@@ -27,7 +27,7 @@ import QuerySettingsContent from './query/querySettingsContent';
 import ScaleSettingsContent from './scale/scaleSettingsContent';
 import OutputSettingsContent from './output/outputSettingsContent';
 import RulesSettingsContent from './rule/rulesSettingsContent';
-import { functionEnabled } from '../../../common/api';
+import { functionEnabled, isDatabricksSparkType } from '../../../common/api';
 import {
     Colors,
     Panel,
@@ -88,7 +88,8 @@ class FlowDefinitionPanel extends React.Component {
             addOutputSinkButtonEnabled: false,
             deleteOutputSinkButtonEnabled: false,
             scaleNumExecutorsSliderEnabled: false,
-            scaleExecutorMemorySliderEnabled: false
+            scaleExecutorMemorySliderEnabled: false,
+            isDatabricksSparkType: false
         };
 
         this.handleWindowClose = this.handleWindowClose.bind(this);
@@ -184,6 +185,10 @@ class FlowDefinitionPanel extends React.Component {
                 scaleExecutorMemorySliderEnabled: response.scaleExecutorMemorySliderEnabled ? true : false
             });
         });
+
+        isDatabricksSparkType().then(response => {
+            this.setState({ isDatabricksSparkType: response });
+        });
     }
 
     render() {
@@ -323,8 +328,11 @@ class FlowDefinitionPanel extends React.Component {
                                 displayName={this.props.flow.displayName}
                                 name={this.props.flow.name}
                                 owner={this.props.flow.owner}
+                                databricksToken={this.props.flow.databricksToken}
                                 onUpdateDisplayName={this.props.onUpdateDisplayName}
+                                onUpdateDatabricksToken={this.props.onUpdateDatabricksToken}
                                 flowNameTextboxEnabled={this.state.flowNameTextboxEnabled}
+                                isDatabricksSparkType={this.state.isDatabricksSparkType}
                             />
                         </VerticalTabItem>
 
@@ -489,6 +497,10 @@ class FlowDefinitionPanel extends React.Component {
                                 onUpdateExecutorMemory={this.props.onUpdateExecutorMemory}
                                 scaleNumExecutorsSliderEnabled={this.state.scaleNumExecutorsSliderEnabled}
                                 scaleExecutorMemorySliderEnabled={this.state.scaleExecutorMemorySliderEnabled}
+                                isDatabricksSparkType={this.state.isDatabricksSparkType}
+                                onUpdateDatabricksAutoScale={this.props.onUpdateDatabricksAutoScale}
+                                onUpdateDatabricksMinWorkers={this.props.onUpdateDatabricksMinWorkers}
+                                onUpdateDatabricksMaxWorkers={this.props.onUpdateDatabricksMaxWorkers}
                             />
                         </VerticalTabItem>
                     </VerticalTabs>
@@ -783,6 +795,7 @@ const mapDispatchToProps = dispatch => ({
 
     // Info Actions
     onUpdateDisplayName: displayName => dispatch(Actions.updateDisplayName(displayName)),
+    onUpdateDatabricksToken: databricksToken => dispatch(Actions.updateDatabricksToken(databricksToken)),
 
     // Input Actions
     onUpdateInputMode: mode => dispatch(Actions.updateInputMode(mode)),
@@ -845,6 +858,9 @@ const mapDispatchToProps = dispatch => ({
     // Scale Actions
     onUpdateNumExecutors: numExecutors => dispatch(Actions.updateNumExecutors(numExecutors)),
     onUpdateExecutorMemory: executorMemory => dispatch(Actions.updateExecutorMemory(executorMemory)),
+    onUpdateDatabricksAutoScale: databricksAutoScale => dispatch(Actions.updateDatabricksAutoScale(databricksAutoScale)),
+    onUpdateDatabricksMinWorkers: databricksMinWorkers => dispatch(Actions.updateDatabricksMinWorkers(databricksMinWorkers)),
+    onUpdateDatabricksMaxWorkers: databricksMaxWorkers => dispatch(Actions.updateDatabricksMaxWorkers(databricksMaxWorkers)),
 
     // Output Actions
     onNewSinker: type => dispatch(Actions.newSinker(type)),

Website/Packages/datax-pipeline/src/modules/flowDefinition/components/info/infoSettingsContent.jsx

@@ -35,6 +35,20 @@ export default class InfoSettingsContent extends React.Component {
                     />
                 </div>
 
+                {this.props.isDatabricksSparkType && (
+                    <div style={sectionStyle}>
+                        <TextField
+                            type="password"
+                            className="ms-font-m info-settings-textbox"
+                            spellCheck={false}
+                            label="Databricks Token"
+                            disabled={false}
+                            value={this.props.databricksToken}
+                            onChange={(event, databricksToken) => this.props.onUpdateDatabricksToken(databricksToken)}
+                        />
+                    </div>
+                )}
+
                 <div style={sectionStyle}>
                     <TextField
                         className="ms-font-m info-settings-textbox"
@@ -66,8 +80,10 @@ InfoSettingsContent.propTypes = {
     displayName: PropTypes.string.isRequired,
     name: PropTypes.string.isRequired,
     owner: PropTypes.string.isRequired,
+    databricksToken: PropTypes.string.isRequired,
 
     onUpdateDisplayName: PropTypes.func.isRequired,
+    onUpdateDatabricksToken: PropTypes.func.isRequired,
     flowNameTextboxEnabled: PropTypes.bool.isRequired
 };
 

Website/Packages/datax-pipeline/src/modules/flowDefinition/components/scale/scaleSettingsContent.jsx

@@ -4,7 +4,7 @@
 // *********************************************************************
 import React from 'react';
 import PropTypes from 'prop-types';
-import { Slider } from 'office-ui-fabric-react';
+import { Slider, Toggle } from 'office-ui-fabric-react';
 import { Colors, ScrollableContentPane, StatementBox } from 'datax-common';
 
 export default class ScaleSettingsContent extends React.Component {
@@ -22,6 +22,15 @@ export default class ScaleSettingsContent extends React.Component {
     }
 
     renderContent() {
+        return (
+            <div style={rootStyle}>
+                {this.props.isDatabricksSparkType && this.renderContentForDatabricks()}
+                {!this.props.isDatabricksSparkType && this.renderContentContentForHDInsight()}
+            </div>
+        );
+    }
+
+    renderContentContentForHDInsight() {
         return (
             <div style={contentStyle}>
                 <div style={sectionStyle}>
@@ -54,6 +63,51 @@ export default class ScaleSettingsContent extends React.Component {
             </div>
         );
     }
+
+    renderContentForDatabricks() {
+        return (
+            <div style={contentStyle}>
+                <div style={toggleSectionStyle}>
+                    <Toggle
+                        onText="Autoscale"
+                        offText="Autoscale"
+                        checked={this.props.scale.jobDatabricksAutoScale}
+                        onChange={(event, value) => this.props.onUpdateDatabricksAutoScale(value)}
+                    />
+                </div>
+
+                <div style={sectionStyle}>
+                    <Slider
+                        className="ms-font-m info-settings-slider"
+                        label={this.props.scale.jobDatabricksAutoScale ? 'Min Number of Workers' : 'Number of Workers'}
+                        disabled={false}
+                        min={1}
+                        max={100}
+                        step={1}
+                        value={Number(this.props.scale.jobDatabricksMinWorkers)}
+                        showValue={true}
+                        onChange={value => this.props.onUpdateDatabricksMinWorkers(value.toString())}
+                    />
+                </div>
+
+                {this.props.scale.jobDatabricksAutoScale && (
+                    <div style={sectionStyle}>
+                        <Slider
+                            className="ms-font-m info-settings-slider"
+                            label="Max Number of Workers"
+                            disabled={false}
+                            min={1}
+                            max={100}
+                            step={1}
+                            value={Number(this.props.scale.jobDatabricksMaxWorkers)}
+                            showValue={true}
+                            onChange={value => this.props.onUpdateDatabricksMaxWorkers(value.toString())}
+                        />
+                    </div>
+                )}
+            </div>
+        );
+    }
 }
 
 // Props
@@ -62,6 +116,9 @@ ScaleSettingsContent.propTypes = {
 
     onUpdateNumExecutors: PropTypes.func.isRequired,
     onUpdateExecutorMemory: PropTypes.func.isRequired,
+    onUpdateDatabricksAutoScale: PropTypes.func.isRequired,
+    onUpdateDatabricksMinWorkers: PropTypes.func.isRequired,
+    onUpdateDatabricksMaxWorkers: PropTypes.func.isRequired,
 
     scaleNumExecutorsSliderEnabled: PropTypes.bool.isRequired,
     scaleExecutorMemorySliderEnabled: PropTypes.bool.isRequired
@@ -83,3 +140,8 @@ const contentStyle = {
 const sectionStyle = {
     paddingBottom: 15
 };
+
+const toggleSectionStyle = {
+    paddingTop: 10,
+    paddingBottom: 15
+};

Website/Packages/datax-pipeline/src/modules/flowDefinition/flowActions.js

@@ -24,6 +24,7 @@ export const FLOW_NEW = 'FLOW_NEW';
 // Info
 export const FLOW_UPDATE_DISPLAY_NAME = 'FLOW_UPDATE_DISPLAY_NAME';
 export const FLOW_UPDATE_OWNER = 'FLOW_UPDATE_OWNER';
+export const FLOW_UPDATE_DATABRICKSTOKEN = 'FLOW_UPDATE_DATABRICKSTOKEN';
 
 // Query
 export const FLOW_UPDATE_QUERY = 'FLOW_UPDATE_QUERY';
@@ -118,6 +119,13 @@ export const updateOwner = () => (dispatch, getState) => {
     });
 };
 
+export const updateDatabricksToken = databricksToken => dispatch => {
+    return dispatch({
+        type: FLOW_UPDATE_DATABRICKSTOKEN,
+        payload: databricksToken
+    });
+};
+
 // Input Actions
 export const updateInputMode = mode => (dispatch, getState) => {
     updateInput(
@@ -782,6 +790,33 @@ export const updateExecutorMemory = executorMemory => (dispatch, getState) => {
     );
 };
 
+export const updateDatabricksAutoScale = databricksAutoScale => (dispatch, getState) => {
+    updateScale(
+        dispatch,
+        Object.assign({}, Selectors.getFlowScale(getState()), {
+            jobDatabricksAutoScale: databricksAutoScale
+        })
+    );
+};
+
+export const updateDatabricksMinWorkers = databricksMinWorkers => (dispatch, getState) => {
+    updateScale(
+        dispatch,
+        Object.assign({}, Selectors.getFlowScale(getState()), {
+            jobDatabricksMinWorkers: databricksMinWorkers
+        })
+    );
+};
+
+export const updateDatabricksMaxWorkers = databricksMaxWorkers => (dispatch, getState) => {
+    updateScale(
+        dispatch,
+        Object.assign({}, Selectors.getFlowScale(getState()), {
+            jobDatabricksMaxWorkers: databricksMaxWorkers
+        })
+    );
+};
+
 function updateScale(dispatch, scale) {
     return dispatch({
         type: FLOW_UPDATE_SCALE,

Website/Packages/datax-pipeline/src/modules/flowDefinition/flowHelpers.js

@@ -460,6 +460,7 @@ export function convertFlowToConfig(flow) {
         name: flow.name,
         displayName: flow.displayName.trim(),
         owner: flow.owner,
+        databricksToken: flow.databricksToken,
         input: Object.assign({}, flow.input, { referenceData: flow.referenceData }),
         process: {
             timestampColumn: flow.input.properties.timestampColumn,
@@ -491,6 +492,7 @@ export function convertConfigToFlow(config) {
         name: config.name,
         displayName: config.displayName,
         owner: config.owner,
+        databricksToken: config.databricksToken,
         input: input,
         referenceData: input.referenceData ? input.referenceData : [],
         functions: config.process.functions ? config.process.functions : [],

Website/Packages/datax-pipeline/src/modules/flowDefinition/flowReducer.js

@@ -10,13 +10,17 @@ const INITIAL_FLOW_STATE = {
     name: '',
     displayName: Models.getDefaultName(),
     owner: '',
+    databricksToken: '',
     input: Models.defaultInput,
     referenceData: [],
     functions: [],
     query: Models.defaultQuery,
     scale: {
         jobNumExecutors: '4',
-        jobExecutorMemory: '1000'
+        jobExecutorMemory: '1000',
+        jobDatabricksAutoScale: true,
+        jobDatabricksMinWorkers: '1',
+        jobDatabricksMaxWorkers: '8'
     },
     outputs: [Models.getMetricSinker()],
     outputTemplates: [],
@@ -70,6 +74,12 @@ export default (state = INITIAL_FLOW_STATE, action) => {
                 displayName: action.payload
             });
 
+        case Actions.FLOW_UPDATE_DATABRICKSTOKEN:
+            return Object.assign({}, state, {
+                isDirty: true,
+                databricksToken: action.payload
+            });
+
         case Actions.FLOW_UPDATE_OWNER:
             return Object.assign({}, state, { owner: action.payload });