fix bugs-metrics dashboard and switching mode, and also enable scro… (#…

…132)

* fix bugs-metrics dashboard and switching mode, and also enabling scroll for job page

* updated the package versions

* update based on feedback

* added some unit tests for the helper functions in ConfigDeleter API

* Enable scrollbar for Input panel

Services/DataX.Config/DataX.Config.Test/CustomRuntimeConfigGenerationTest.cs

@@ -86,7 +86,7 @@ public override int GetOrder()
 
             public override async Task<string> Process(FlowDeploymentSession flowToDeploy)
             {
-                return "done";
+                return await Task.FromResult("done");
             }
         }
 

Services/DataX.Config/DataX.Config.Test/DataX.Config.Test.csproj

@@ -37,6 +37,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\DataX.Contract\DataX.Contract.csproj" />
+    <ProjectReference Include="..\..\DataX.Flow\DataX.Flow.DeleteHelper\DataX.Flow.DeleteHelper.csproj" />
     <ProjectReference Include="..\DataX.Config.KeyVault\DataX.Config.KeyVault.csproj" />
     <ProjectReference Include="..\DataX.Config.Test.Utility\DataX.Config.Test.Utility.csproj" />
     <ProjectReference Include="..\DataX.Config\DataX.Config.csproj" />

Services/DataX.Flow/DataX.Flow.Common.Tests/GeneratorHelperTests.cs

@@ -0,0 +1,73 @@
+// *********************************************************************
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+// Licensed under the MIT License
+// *********************************************************************
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using DataX.Flow.Common;
+using DataX.Config.ConfigDataModel;
+
+namespace DataX.Flow.Generator.Tests
+{
+    [TestClass]
+    public class FlowGeneratorHelperTests
+    {
+        [TestMethod]
+        public void IsKeyVaultTest()
+        {
+            Assert.IsTrue(Helper.IsKeyVault("keyvault://abc"));
+            Assert.IsTrue(Helper.IsKeyVault("secretscope://abc"));
+            Assert.IsFalse(Helper.IsKeyVault("abc//keyvault://abc"));
+            Assert.IsFalse(Helper.IsKeyVault("abc.secretscope://abc"));
+        }
+
+        [TestMethod]
+        public void ParseEventHubTest()
+        {
+            Assert.AreEqual("testeventhubname", Helper.ParseEventHub(@"Endpoint=sb://testnamespace.servicebus.windows.net/;SharedAccessKeyName=policyname;SharedAccessKey=12345=;EntityPath=testeventhubname"));
+
+            Assert.AreEqual(null, Helper.ParseEventHub(@"Endpoint=sb://testnamespace.servicebus.windows.net/;SharedAccessKeyName=policyname;SharedAccessKey=12345"));
+        }
+
+        [TestMethod]
+        public void ParseEventHubNamespaceTest()
+        {
+            Assert.AreEqual("testnamespace", Helper.ParseEventHubNamespace(@"Endpoint=sb://testnamespace.servicebus.windows.net/;SharedAccessKeyName=policyname;SharedAccessKey=12345=;EntityPath=testeventhubname"));
+
+            Assert.AreEqual("testnamespace", Helper.ParseEventHubNamespace(@"Endpoint=sb://testnamespace.servicebus.windows.net/;SharedAccessKeyName=policyname;SharedAccessKey=12345"));
+        }
+
+        [TestMethod]
+        public void ParseEventHubPolicyNameTest()
+        {
+            Assert.AreEqual("policyname", Helper.ParseEventHubPolicyName(@"Endpoint=sb://testnamespace.servicebus.windows.net/;SharedAccessKeyName=policyname;SharedAccessKey=12345=;EntityPath=testeventhubname"));
+
+            Assert.AreEqual("policyname", Helper.ParseEventHubPolicyName(@"Endpoint =sb://testnamespace.servicebus.windows.net/;SharedAccessKeyName=policyname;SharedAccessKey=12345"));
+        }
+
+        [TestMethod]
+        public void ParseEventHubAccessKeyTest()
+        {
+            Assert.AreEqual("12345=", Helper.ParseEventHubAccessKey(@"Endpoint=sb://testnamespace.servicebus.windows.net/;SharedAccessKeyName=policyname;SharedAccessKey=12345=;EntityPath=testeventhubname"));
+
+            Assert.AreEqual("12345", Helper.ParseEventHubAccessKey(@"Endpoint =sb://testnamespace.servicebus.windows.net/;SharedAccessKeyName=policyname;SharedAccessKey=12345"));
+        }
+
+        [TestMethod]
+        public void GetSecretFromKeyvaultIfNeededTest()
+        {
+            Assert.AreEqual("value", Helper.GetSecretFromKeyvaultIfNeeded(@"value"));
+        }
+
+        [TestMethod]
+        public void GetKeyVaultNameTest()
+        {
+            Assert.AreEqual("keyvault://somekeyvalut/test-input-connectionstring-CD42404D52AD55CCFA9ACA4ADC828AA5", Helper.GetKeyVaultName("somekeyvalut", "test-input-connectionstring", Constants.SparkTypeHDInsight, "value"));
+
+            Assert.AreEqual("keyvault://somekeyvalut/test-input-connectionstring", Helper.GetKeyVaultName("somekeyvalut", "test-input-connectionstring", Constants.SparkTypeHDInsight, "value", false));
+
+            Assert.AreEqual("secretscope://somekeyvalut/test-input-connectionstring-CD42404D52AD55CCFA9ACA4ADC828AA5", Helper.GetKeyVaultName("somekeyvalut", "test-input-connectionstring", Constants.SparkTypeDataBricks, "value"));
+
+            Assert.AreEqual("secretscope://somekeyvalut/test-input-connectionstring", Helper.GetKeyVaultName("somekeyvalut", "test-input-connectionstring", Constants.SparkTypeDataBricks, "value", false));
+        }
+    }
+}

Services/DataX.Flow/DataX.Flow.DeleteHelper/ConfigDeleter.cs

@@ -99,8 +99,8 @@ public async Task<ApiResult> DeleteFlow(JObject jObject)
             _inputEventhubConnectionStringRef = Helper.IsKeyVault(diag.EventhubConnectionString) ? diag.EventhubConnectionString : Helper.GenerateNewSecret(_keySecretList, _engineEnvironment.EngineFlowConfig.SparkKeyVaultName, ConfigName + "-input-eventhubconnectionstring", _engineEnvironment.EngineFlowConfig.SparkType, diag.EventhubConnectionString, false);
             diag.EventhubConnectionString = _inputEventhubConnectionStringRef;
 
-            if (diag.InputType == Constants.InputType_EventHub)
-            { 
+            if (diag.InputType == Constants.InputType_EventHub && !string.IsNullOrEmpty(inputEventhubConnection))
+            {
                 var ehName = Helper.ParseEventHub(inputEventhubConnection);
                 _eventHubNamespace = Helper.ParseEventHubNamespace(inputEventhubConnection);
                 _eventHubNameRole = Helper.ParseEventHubPolicyName(inputEventhubConnection);
@@ -115,7 +115,7 @@ public async Task<ApiResult> DeleteFlow(JObject jObject)
 
                 _eventHubNames = new List<string>() { ehName };
             }
-            else
+            else if (diag.InputType == Constants.InputType_IoTHub && !string.IsNullOrEmpty(diag.EventhubNames) && !string.IsNullOrEmpty(inputEventhubConnection))
             {
                 _eventHubNames = Helper.ParseEventHubNames(diag.EventhubNames);
                 _eventHubNamespace = Helper.ParseEventHubNamespace(inputEventhubConnection);
@@ -139,7 +139,7 @@ public async Task<ApiResult> DeleteFlow(JObject jObject)
 
             // ResourceCreation is one of the environment variables.
             // If you don't want to create resource, you can set this to false.
-            if (_engineEnvironment.ResourceCreation && (diag.InputType == Constants.InputType_EventHub || diag.InputType == Constants.InputType_IoTHub))
+            if (_engineEnvironment.ResourceCreation && _eventHubNames != null && (diag.InputType == Constants.InputType_EventHub || diag.InputType == Constants.InputType_IoTHub))
             {
                 var inputSubscriptionId = string.IsNullOrEmpty(diag.InputSubscriptionId) ? Helper.GetSecretFromKeyvaultIfNeeded(_engineEnvironment.EngineFlowConfig.SubscriptionId) : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputSubscriptionId);
                 var inputResourceGroup = string.IsNullOrEmpty(diag.InputResourceGroup) ? _engineEnvironment.EngineFlowConfig.EventHubResourceGroupName : Helper.GetSecretFromKeyvaultIfNeeded(diag.InputResourceGroup);

Website/Packages/datax-common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datax-common",
-  "version": "1.3.0-SNAPSHOT-9",
+  "version": "1.3.0-SNAPSHOT-10",
   "description": "Common UX, styles, utilities, and modules",
   "author": "Microsoft",
   "license": "MIT",

Website/Packages/datax-home/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datax-home",
-  "version": "1.3.0-SNAPSHOT-9",
+  "version": "1.3.0-SNAPSHOT-10",
   "description": "Home page",
   "author": "Microsoft",
   "license": "MIT",
@@ -30,7 +30,7 @@
     "css-loader": "1.0.1",
     "file-loader": "2.0.0",
     "mini-css-extract-plugin": "0.4.4",
-    "datax-common": "1.3.0-SNAPSHOT-9",
+    "datax-common": "1.3.0-SNAPSHOT-10",
     "office-ui-fabric-react": "6.111.2",
     "q": "1.5.1",
     "react": "16.6.3",

Website/Packages/datax-jobs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datax-jobs",
-  "version": "1.3.0-SNAPSHOT-9",
+  "version": "1.3.0-SNAPSHOT-10",
   "description": "Job features",
   "author": "Microsoft",
   "license": "MIT",
@@ -30,7 +30,7 @@
     "css-loader": "1.0.1",
     "file-loader": "2.0.0",
     "mini-css-extract-plugin": "0.4.4",
-    "datax-common": "1.3.0-SNAPSHOT-9",
+    "datax-common": "1.3.0-SNAPSHOT-10",
     "office-ui-fabric-react": "6.111.2",
     "q": "1.5.1",
     "prop-types": "15.6.2",

Website/Packages/datax-jobs/src/modules/jobs/components/sparkJobs.jsx

@@ -58,7 +58,7 @@ class SparkJobs extends React.Component {
             }
 
             return (
-                <Panel>
+                <div style={rootStyle}>
                     <div style={contentStyle}>
                         <div style={filterContainerStyle}>
                             <SearchBox
@@ -73,7 +73,7 @@ class SparkJobs extends React.Component {
 
                         <div style={detailsContainerStyle}>{details}</div>
                     </div>
-                </Panel>
+                </div>
             );
         }
     }
@@ -134,6 +134,13 @@ const mapStateToProps = state => ({});
 const mapDispatchToProps = () => ({});
 
 // Styles
+const rootStyle = {
+    display: 'flex',
+    flexDirection: 'column',
+    overflowX: 'hidden',
+    overflowY: 'auto'
+};
+
 const contentStyle = {
     paddingLeft: 30,
     paddingRight: 30,

Website/Packages/datax-metrics/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datax-metrics",
-  "version": "1.3.0-SNAPSHOT-9",
+  "version": "1.3.0-SNAPSHOT-10",
   "description": "Metric features",
   "author": "Microsoft",
   "license": "MIT",
@@ -33,7 +33,7 @@
     "d3": "4.10.2",
     "file-loader": "2.0.0",
     "mini-css-extract-plugin": "0.4.4",
-    "datax-common": "1.3.0-SNAPSHOT-9",
+    "datax-common": "1.3.0-SNAPSHOT-10",
     "office-ui-fabric-react": "6.111.2",
     "q": "1.5.1",
     "plotly.js": "1.44.4",

Website/Packages/datax-metrics/src/modules/metrics/components/metricDashboard.jsx

@@ -35,7 +35,11 @@ class MetricDashboard extends React.Component {
     }
 
     componentDidMount() {
-        if (!this.props.product) {
+        if (
+            !this.props.product ||
+            !this.props.product.jobNames ||
+            (this.props.product.jobNames && this.props.product.jobNames.length < 1)
+        ) {
             return;
         }
 
@@ -60,6 +64,14 @@ class MetricDashboard extends React.Component {
 
         if (!product) {
             return null;
+        } else if (!product.jobNames || (product.jobNames && product.jobNames.length < 1)) {
+            const message = 'No metrics to display. No job for the flow is created. Please start the job to see metrics.';
+            return (
+                <div>
+                    <div style={headerStyle}>{product.displayName} Metrics</div>
+                    <StatementBox icon="IncidentTriangle" overrideRootStyle={messageStyle} statement={message} />
+                </div>
+            );
         } else if (product.gui.input.mode === Constants.batching) {
             const message = 'Metrics is not supported for the batch job.';
             return (

Website/Packages/datax-metrics/src/modules/metrics/components/metricExplorer.jsx

@@ -129,8 +129,7 @@ const contentStyle = {
     width: '100%',
     display: 'flex',
     flexDirection: 'column',
-    overflowY: 'auto',
-    paddingBottom: 80
+    overflowY: 'auto'
 };
 
 export default withRouter(

Website/Packages/datax-pipeline/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datax-pipeline",
-  "version": "1.3.0-SNAPSHOT-9",
+  "version": "1.3.0-SNAPSHOT-10",
   "description": "Pipeline features",
   "author": "Microsoft",
   "license": "MIT",
@@ -34,8 +34,8 @@
     "css-loader": "1.0.1",
     "file-loader": "2.0.0",
     "mini-css-extract-plugin": "0.4.4",
-    "datax-common": "1.3.0-SNAPSHOT-9",
-    "datax-query": "1.3.0-SNAPSHOT-9",
+    "datax-common": "1.3.0-SNAPSHOT-10",
+    "datax-query": "1.3.0-SNAPSHOT-10",
     "office-ui-fabric-react": "6.111.2",
     "q": "1.5.1",
     "promise-polyfill": "8.1.0",

Website/Packages/datax-pipeline/src/modules/flowDefinition/components/function/functionSettingsContent.jsx

@@ -113,7 +113,7 @@ export default class FunctionSettingsContent extends React.Component {
     renderAddFunctionButton() {
         const menuItems = this.props.enableLocalOneBox
             ? Models.functionTypes
-                  .filter(functionType => functionType.name != 'Azure Function') // disable Azure function in OneBox mode
+                  .filter(functionType => functionType.name !== 'Azure Function') // disable Azure function in OneBox mode
                   .map(functionType => {
                       return Object.assign({}, functionType, {
                           onClick: () => this.props.onNewFunction(functionType.key)

Website/Packages/datax-pipeline/src/modules/flowDefinition/components/input/inputSettingsContent.jsx

@@ -53,7 +53,7 @@ export default class InputSettingsContent extends React.Component {
 
     renderContent() {
         return (
-            <div style={{ display: 'flex', flexDirection: 'column', flex: 1 }}>
+            <div style={{ display: 'flex', flexDirection: 'column', flex: 1, overflowY: 'auto' }}>
                 <div style={contentStyle}>
                     {this.renderLeftPane()}
                     {this.renderRightPane()}
@@ -65,7 +65,7 @@ export default class InputSettingsContent extends React.Component {
     renderLeftPane() {
         if (this.props.input.mode === Models.inputModeEnum.batching) {
             let batchData = undefined;
-            if (this.props.batchInputs != undefined && this.props.selectedFlowBatchInputIndex != undefined) {
+            if (this.props.batchInputs && this.props.selectedFlowBatchInputIndex !== undefined) {
                 batchData = this.props.batchInputs[this.props.selectedFlowBatchInputIndex];
             }
             return (
@@ -528,7 +528,10 @@ export default class InputSettingsContent extends React.Component {
         const enableButton =
             ((this.props.input.mode === Models.inputModeEnum.streaming && this.props.input.properties.inputEventhubConnection !== '') ||
                 (this.props.input.mode === Models.inputModeEnum.batching &&
-                    this.props.batchInputs[this.props.selectedFlowBatchInputIndex].properties.connection !== '')) &&
+                    this.props.batchInputs &&
+                    this.props.selectedFlowBatchInputIndex !== undefined &&
+                    this.props.batchInputs[this.props.selectedFlowBatchInputIndex].properties.connection !== '' &&
+                    this.props.batchInputs[this.props.selectedFlowBatchInputIndex].properties.path !== '')) &&
             !this.props.fetchingInputSchema &&
             this.props.getInputSchemaButtonEnabled;
 

Website/Packages/datax-pipeline/src/modules/flowDefinition/components/output/outputSettingsContent.jsx

@@ -125,14 +125,14 @@ export default class OutputSettingsContent extends React.Component {
     renderAddSinkerButton() {
         const menuItems = this.props.enableLocalOneBox
             ? Models.outputSinkerTypes
-                  .filter(sinkerType => sinkerType.name == 'Local')
+                  .filter(sinkerType => sinkerType.name === 'Local')
                   .map(sinkerType => {
                       return Object.assign({}, sinkerType, {
                           onClick: () => this.props.onNewSinker(sinkerType.key)
                       });
                   })
             : Models.outputSinkerTypes
-                  .filter(sinkerType => sinkerType.name != 'Local')
+                  .filter(sinkerType => sinkerType.name !== 'Local')
                   .map(sinkerType => {
                       return Object.assign({}, sinkerType, {
                           onClick: () => this.props.onNewSinker(sinkerType.key)

Website/Packages/datax-pipeline/src/modules/flowDefinition/components/schedule/scheduleSettingsContent.jsx

@@ -132,7 +132,7 @@ export default class ScheduleSettingsContent extends React.Component {
                       });
                   })
                 : Models.batchTypes
-                      .filter(batchType => batchType.key != Models.batchTypeEnum.recurring)
+                      .filter(batchType => batchType.key !== Models.batchTypeEnum.recurring)
                       .map(batchType => {
                           return Object.assign({}, batchType, {
                               onClick: () => this.props.onNewBatch(batchType.key)

Website/Packages/datax-pipeline/src/modules/flowDefinition/flowHelpers.js

@@ -441,6 +441,7 @@ export function convertFlowToConfig(flow, query) {
     let outputTemplates = [...flow.outputTemplates];
     let rules = [...flow.rules];
     let batchList = flow.batchList ? [...flow.batchList] : [];
+    let batchInputs = flow.batchInputs ? [...flow.batchInputs] : [Models.getDefaultBatchInputSettings()];
 
     // sort by name
     referenceData.sort((a, b) => a.id.localeCompare(b.id));
@@ -457,7 +458,7 @@ export function convertFlowToConfig(flow, query) {
         displayName: flow.displayName.trim(),
         owner: flow.owner,
         databricksToken: flow.databricksToken,
-        input: Object.assign({}, flow.input, { referenceData: flow.referenceData, batch: flow.batchInputs }),
+        input: Object.assign({}, flow.input, { referenceData: flow.referenceData, batch: batchInputs }),
         process: {
             timestampColumn: flow.input.properties.timestampColumn,
             watermark: `${flow.input.properties.watermarkValue} ${flow.input.properties.watermarkUnit}`,
@@ -492,7 +493,7 @@ export function convertConfigToFlow(config) {
         owner: config.owner,
         databricksToken: config.databricksToken,
         input: input,
-        batchInputs: input.batch,
+        batchInputs: input.batch ? input.batch : [Models.getDefaultBatchInputSettings()],
         batchList: config.batchList ? config.batchList : [],
         referenceData: input.referenceData ? input.referenceData : [],
         functions: config.process.functions ? config.process.functions : [],

Website/Packages/datax-pipeline/src/modules/flowDefinition/flowReducer.js

@@ -4,7 +4,7 @@
 // *********************************************************************
 import * as Actions from './flowActions';
 import * as Models from './flowModels';
-import {QueryModels, QueryActions} from 'datax-query';
+import { QueryModels, QueryActions } from 'datax-query';
 
 const INITIAL_FLOW_STATE = {
     // Flow Config
@@ -56,7 +56,7 @@ export default (state = INITIAL_FLOW_STATE, action) => {
             return Object.assign({}, INITIAL_FLOW_STATE, flow, {
                 isNew: false,
                 isDirty: false,
-                selectedFlowBatchInputIndex: flow.input.batch && flow.input.batch.length > 0 ? 0 : undefined,
+                selectedFlowBatchInputIndex: flow.batchInputs && flow.batchInputs.length > 0 ? 0 : undefined,
                 selectedReferenceDataIndex: flow.referenceData && flow.referenceData.length > 0 ? 0 : undefined,
                 selectedFunctionIndex: flow.functions && flow.functions.length > 0 ? 0 : undefined,
                 selectedSinkerIndex: flow.outputs && flow.outputs.length > 0 ? 0 : undefined,
@@ -109,7 +109,7 @@ export default (state = INITIAL_FLOW_STATE, action) => {
         case Actions.FLOW_UPDATE_SAMPLING_INPUT_DURATION:
             return Object.assign({}, state, {
                 samplingInputDuration: action.duration
-            });     
+            });
 
         case Actions.FLOW_UPDATE_REFERENCE_DATA_LIST:
             return Object.assign({}, state, {