Tylake/configuration

DataProcessing/DataX.Utilities/.gitignore

@@ -0,0 +1 @@
+*/obj

Services/.gitignore

@@ -128,4 +128,7 @@ BundleArtifacts/
 /DeploymentCloud/Deployment.DataX/cachedVariables
 /DeploymentCloud/Deployment.DataX/Temp
 /DeploymentCloud/Deployment.DataX/Outputs
-/DeploymentCloud/Deployment.DataX/Packages
+/DeploymentCloud/Deployment.DataX/Packages
+
+# Development settings
+appsettings.Development.json

Services/AspnetCore/DataX.FlowManagement/DataX.FlowManagement/Startup.cs

@@ -27,8 +27,9 @@ public class Startup
     {
 
         private const string _MetricsHttpEndpointRelativeUri = "/api/data/upload";
+        private readonly ILoggerFactory _loggerFactory;
 
-        public Startup(IHostingEnvironment env)
+        public Startup(IHostingEnvironment env, ILoggerFactory loggerFactory)
         {
             var builder = new ConfigurationBuilder()
                 .SetBasePath(env.ContentRootPath)
@@ -37,6 +38,8 @@ public Startup(IHostingEnvironment env)
                 .AddEnvironmentVariables();
 
             Configuration = builder.Build();
+
+            _loggerFactory = loggerFactory;
         }
 
         public IConfiguration Configuration { get; }
@@ -51,7 +54,7 @@ public void ConfigureServices(IServiceCollection services)
 
             // Export the Config dependencies
             Type[] exportTypes = new Type[] { typeof(FlowOperation), typeof(RuntimeConfigGeneration), typeof(JobOperation) };
-            services.AddMefExportsFromAssemblies(ServiceLifetime.Scoped, GetOneBoxModeDependencyAssemblies(), exportTypes, new object[] { });
+            services.AddMefExportsFromAssemblies(ServiceLifetime.Scoped, GetOneBoxModeDependencyAssemblies(), exportTypes, null, _loggerFactory, true);
             var result = InitTemplatesForLocal(services);
             Ensure.IsSuccessResult(result);
         }

Services/DataX.Config/DataX.Config.Local/LocalSparkClientFactory.cs

@@ -17,7 +17,7 @@ public class LocalSparkClientFactory : ISparkJobClientFactory
         private ILogger _logger { get; }
 
         [ImportingConstructor]
-        public LocalSparkClientFactory(ILogger logger)
+        public LocalSparkClientFactory(ILogger<LocalSparkClientFactory> logger)
         {
             this._logger = logger;
         }

Services/DataX.Config/DataX.Config/InternalService/SparkJobOperation.cs

@@ -21,7 +21,7 @@ namespace DataX.Config
     public class SparkJobOperation
     {
         [ImportingConstructor]
-        public SparkJobOperation(SparkJobData jobData, SparkClusterManager clusterManager, ILogger logger)
+        public SparkJobOperation(SparkJobData jobData, SparkClusterManager clusterManager, ILogger<SparkJobOperation> logger)
         {
             this.JobData= jobData;
             this.ClusterManager = clusterManager;

Services/DataX.Config/DataX.Config/PublicService/RuntimeConfigGeneration.cs

@@ -25,7 +25,7 @@ public class RuntimeConfigGeneration
         /// </summary>
         [ImportingConstructor]
         public RuntimeConfigGeneration(
-            ILogger logger,
+            ILogger<RuntimeConfigGeneration> logger,
             FlowDataManager flows,
             JobDataManager jobData,
             [ImportMany] IEnumerable<IFlowDeploymentProcessor> flowProcessors

Services/DataX.Config/DataX.Config/Utility/LoggerExportDescriptorProvider.cs

@@ -0,0 +1,76 @@
+// *********************************************************************
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+// Licensed under the MIT License
+// *********************************************************************
+using Microsoft.Extensions.Logging;
+using System;
+using System.Collections.Generic;
+using System.Composition.Hosting.Core;
+using System.Linq;
+using System.Reflection;
+
+namespace DataX.Config.Utility
+{
+    /// <summary>
+    /// Using a LoggerFactory, dynamically creates Logger<T> to resolve for MEF
+    /// Falls back to InstanceExportDescriptorProvider implementation if not a Logger
+    /// </summary>
+    public class LoggerExportDescriptorProvider : InstanceExportDescriptorProvider
+    {
+        private static readonly Type _ILoggerType = typeof(ILogger);
+
+        private static readonly MethodInfo _CreateLogger =
+            typeof(Microsoft.Extensions.Logging.LoggerFactoryExtensions)
+            .GetMethods()
+            .Where(m => m.Name == nameof(LoggerFactoryExtensions.CreateLogger) && m.IsGenericMethod)
+            .FirstOrDefault();
+
+        private readonly ILoggerFactory _loggerFactory;
+        private readonly bool _hasInstances;
+
+        public LoggerExportDescriptorProvider(object[] instances, ILoggerFactory loggerFactory)
+            : base(instances)
+        {
+            _hasInstances = instances?.Length > 0;
+            _loggerFactory = loggerFactory;
+        }
+
+        ///<inheritdoc />
+        public override IEnumerable<ExportDescriptorPromise> GetExportDescriptors(CompositionContract contract, DependencyAccessor descriptorAccessor)
+        {
+            if (_loggerFactory != null && _ILoggerType.IsAssignableFrom(contract.ContractType))
+            {
+                ILogger logger;
+
+                if (contract.ContractType.GenericTypeArguments.Length > 0)
+                {
+                    logger = CreateLogger(contract.ContractType.GenericTypeArguments.FirstOrDefault());
+                }
+                else
+                {
+                    logger = _loggerFactory.CreateLogger(contract.ContractType);
+                }
+
+                yield return new ExportDescriptorPromise(
+                        contract,
+                        contract.ContractType.FullName,
+                        true,
+                        NoDependencies,
+                        dependencies => ExportDescriptor.Create((context, operation) => logger, NoMetadata));
+            }
+            else if(_hasInstances)
+            {
+                foreach (var descriptor in base.GetExportDescriptors(contract, descriptorAccessor))
+                {
+                    yield return descriptor;
+                }
+            }
+        }
+
+        private ILogger CreateLogger(Type t)
+        {
+            var genericMethod = _CreateLogger.MakeGenericMethod(t);
+            return genericMethod.Invoke(null, new object[] { _loggerFactory }) as ILogger;
+        }
+    }
+}

Services/DataX.Flow/DataX.Flow/ApplicationPackageRoot/ApplicationManifest.xml

@@ -22,6 +22,7 @@ Licensed under the MIT License
     <Parameter Name="MefStorageAccountName" DefaultValue="" />
     <Parameter Name="MefContainerName" DefaultValue="" />
     <Parameter Name="MefBlobDirectory" DefaultValue="" />
+    <Parameter Name="EnableOneBox" DefaultValue="false" />
   </Parameters>
   <!-- Import the ServiceManifest from the ServicePackage. The ServiceManifestName and ServiceManifestVersion 
        should match the Name and Version attributes of the ServiceManifest element defined in the 
@@ -37,6 +38,7 @@ Licensed under the MIT License
             <Parameter Name="cosmosDBConfigDatabaseName" Value="[cosmosDBConfigDatabaseName]" />
             <Parameter Name="cosmosDBConfigCollectionName" Value="[cosmosDBConfigCollectionName]" />
             <Parameter Name="AppInsightsIntrumentationKey" Value="[AppInsightsIntrumentationKey]" />
+            <Parameter Name="EnableOneBox" Value="[EnableOneBox]" />
           </Section>
         </Settings>
       </ConfigOverride>
@@ -57,6 +59,7 @@ Licensed under the MIT License
             <Parameter Name="cosmosDBConfigDatabaseName" Value="[cosmosDBConfigDatabaseName]" />
             <Parameter Name="cosmosDBConfigCollectionName" Value="[cosmosDBConfigCollectionName]" />
             <Parameter Name="AppInsightsIntrumentationKey" Value="[AppInsightsIntrumentationKey]" />
+            <Parameter Name="EnableOneBox" Value="[EnableOneBox]" />
           </Section>
         </Settings>
       </ConfigOverride>
@@ -77,6 +80,7 @@ Licensed under the MIT License
             <Parameter Name="cosmosDBConfigDatabaseName" Value="[cosmosDBConfigDatabaseName]" />
             <Parameter Name="cosmosDBConfigCollectionName" Value="[cosmosDBConfigCollectionName]" />
             <Parameter Name="AppInsightsIntrumentationKey" Value="[AppInsightsIntrumentationKey]" />
+            <Parameter Name="EnableOneBox" Value="[EnableOneBox]" />
           </Section>
         </Settings>
       </ConfigOverride>
@@ -100,6 +104,7 @@ Licensed under the MIT License
             <Parameter Name="MefStorageAccountName" Value="[MefStorageAccountName]" />
             <Parameter Name="MefContainerName" Value="[MefContainerName]" />
             <Parameter Name="MefBlobDirectory" Value="[MefBlobDirectory]" />
+            <Parameter Name="EnableOneBox" Value="[EnableOneBox]" />
           </Section>
         </Settings>
       </ConfigOverride>

Services/DataX.Flow/DataX.Flow/ApplicationParameters/Cloud.xml

@@ -18,5 +18,6 @@ Licensed under the MIT License
     <Parameter Name="MefStorageAccountName" Value="" />
     <Parameter Name="MefContainerName" Value="" />
     <Parameter Name="MefBlobDirectory" Value="" />
+    <Parameter Name="EnableOneBox" Value="" />
   </Parameters>
 </Application>

Services/DataX.Flow/DataX.Flow/ApplicationParameters/Local.1Node.xml

@@ -22,5 +22,6 @@ Licensed under the MIT License
     <Parameter Name="MefStorageAccountName" Value="" />
     <Parameter Name="MefContainerName" Value="" />
     <Parameter Name="MefBlobDirectory" Value="" />
+    <Parameter Name="EnableOneBox" Value="" />
   </Parameters>
 </Application>

Services/DataX.Flow/DataX.Flow/ApplicationParameters/Local.5Node.xml

@@ -22,5 +22,6 @@ Licensed under the MIT License
     <Parameter Name="MefStorageAccountName" Value="" />
     <Parameter Name="MefContainerName" Value="" />
     <Parameter Name="MefBlobDirectory" Value="" />
+    <Parameter Name="EnableOneBox" Value="" />
   </Parameters>
 </Application>

Services/DataX.Flow/DataX.Flow/DataX.Flow.sfproj

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" InitialTargets=";ValidateMSBuildFiles">
-  <Import Project="..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props" Condition="Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" />
+  <Import Project="..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props" Condition="Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" />
   <PropertyGroup Label="Globals">
     <ProjectGuid>34e85436-8f73-44fc-9c03-ca205077d68d</ProjectGuid>
     <ProjectVersion>2.3</ProjectVersion>
@@ -42,9 +42,9 @@
     <ApplicationProjectTargetsPath>$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Service Fabric Tools\Microsoft.VisualStudio.Azure.Fabric.ApplicationProject.targets</ApplicationProjectTargetsPath>
   </PropertyGroup>
   <Import Project="$(ApplicationProjectTargetsPath)" Condition="Exists('$(ApplicationProjectTargetsPath)')" />
-  <Import Project="..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets" Condition="Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" />
-  <Target Name="ValidateMSBuildFiles">
-    <Error Condition="!Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" Text="Unable to find the '..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
-    <Error Condition="!Exists('..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" Text="Unable to find the '..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
+  <Import Project="..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets" Condition="Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" />
+  <Target Name="ValidateMSBuildFiles" BeforeTargets="PrepareForBuild">
+    <Error Condition="!Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props')" Text="Unable to find the '..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.props' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
+    <Error Condition="!Exists('..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets')" Text="Unable to find the '..\..\packages\Microsoft.VisualStudio.Azure.Fabric.MSBuild.1.6.7\build\Microsoft.VisualStudio.Azure.Fabric.Application.targets' file. Please restore the 'Microsoft.VisualStudio.Azure.Fabric.MSBuild' Nuget package." />
   </Target>
 </Project>

Services/DataX.Flow/Flow.ManagementService/Controllers/FlowManagementController.cs

@@ -17,14 +17,17 @@
 using System.Linq;
 using DataX.Utilities.Web;
 using System.Collections.Generic;
+using Microsoft.AspNetCore.Authorization;
+using DataX.ServiceHost.AspNetCore.Authorization.Roles;
 
 namespace Flow.Management.Controllers
 {
     [Route("api")]
+    [DataXReader]
     public partial class FlowManagementController : Controller
     {
         private readonly ILogger<FlowManagementController> _logger;
-        private FlowOperation _flowOperation;
+        private readonly FlowOperation _flowOperation;
         private JobOperation _jobOperation;
         private RuntimeConfigGeneration _runtimeConfigGenerator;
         private bool _isLocal = false;
@@ -44,6 +47,7 @@ public FlowManagementController(ILogger<FlowManagementController> logger, FlowOp
 
         [HttpPost]
         [Route("flow/save")] // save flow config
+        [DataXWriter]
         public async Task<ApiResult> SaveFlow([FromBody]JObject config)
         {
             try
@@ -75,6 +79,7 @@ public async Task<ApiResult> SaveFlow([FromBody]JObject config)
 
         [HttpPost]
         [Route("flow/generateconfigs")] // generate flow configs
+        [DataXWriter]
         public async Task<ApiResult> GenerateConfigs([FromBody] string flowName)
         {
             try
@@ -154,6 +159,7 @@ public async Task<ApiResult> GetAllFlowsMin()
 
         [HttpPost]
         [Route("flow/startjobs")]
+        [DataXWriter]
         public async Task<ApiResult> StartJobsForFlow([FromBody] string flowName)
         {
             try
@@ -174,6 +180,7 @@ public async Task<ApiResult> StartJobsForFlow([FromBody] string flowName)
 
         [HttpPost]
         [Route("flow/restartjobs")]
+        [DataXWriter]
         public async Task<ApiResult> RestartJobsForFlow([FromBody] string flowName)
         {
             try
@@ -194,6 +201,7 @@ public async Task<ApiResult> RestartJobsForFlow([FromBody] string flowName)
 
         [HttpPost]
         [Route("flow/stopjobs")]
+        [DataXWriter]
         public async Task<ApiResult> StopJobsForFlow([FromBody] string flowName)
         {
             try
@@ -215,6 +223,7 @@ public async Task<ApiResult> StopJobsForFlow([FromBody] string flowName)
 
         [HttpPost]
         [Route("userqueries/schema")] // generator (sqlparser)
+        [DataXWriter]
         public async Task<ApiResult> GetSchema([FromBody]JObject config)
         {
             try
@@ -242,6 +251,7 @@ public async Task<ApiResult> GetSchema([FromBody]JObject config)
 
         [HttpPost]
         [Route("userqueries/codegen")] // generator
+        [DataXWriter]
         public async Task<ApiResult> GetCodeGen([FromBody]JObject config)
         {
             try
@@ -328,6 +338,7 @@ public async Task<ApiResult> GetJobsByNames([FromBody] string[] jobNames)
 
         [HttpPost]
         [Route("job/start")] // start the job
+        [DataXWriter]
         public async Task<ApiResult> StartJob([FromBody] string jobName)
         {
             try
@@ -349,6 +360,7 @@ public async Task<ApiResult> StartJob([FromBody] string jobName)
 
         [HttpPost]
         [Route("job/stop")] // stop the job
+        [DataXWriter]
         public async Task<ApiResult> StopJob([FromBody] string jobName)
         {
             try
@@ -369,6 +381,7 @@ public async Task<ApiResult> StopJob([FromBody] string jobName)
 
         [HttpPost]
         [Route("job/restart")]
+        [DataXWriter]
         public async Task<ApiResult> RestartJob([FromBody] string jobName)
         {
             try
@@ -390,6 +403,7 @@ public async Task<ApiResult> RestartJob([FromBody] string jobName)
 
         [HttpGet]
         [Route("job/syncall")]
+        [DataXWriter]
         public async Task<ApiResult> SyncAllJobs()
         {
             try

Services/DataX.Flow/Flow.ManagementService/Flow.ManagementService.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
+<Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
     <TargetFramework>netcoreapp2.1</TargetFramework>
@@ -46,9 +46,11 @@
     <ProjectReference Include="..\..\DataX.Config\DataX.Config.Input.EventHub\DataX.Config.Input.EventHub.csproj" />
     <ProjectReference Include="..\..\DataX.Config\DataX.Config.KeyVault\DataX.Config.KeyVault.csproj" />
     <ProjectReference Include="..\..\DataX.Config\DataX.Config.LivyClient\DataX.Config.LivyClient.csproj" />
+    <ProjectReference Include="..\..\DataX.Config\DataX.Config.Local\DataX.Config.Local.csproj" />
     <ProjectReference Include="..\..\DataX.Config\DataX.Config.Storage\DataX.Config.Storage.csproj" />
     <ProjectReference Include="..\..\DataX.Config\DataX.Config\DataX.Config.csproj" />
     <ProjectReference Include="..\..\DataX.ServiceHost\DataX.ServiceHost.AspNetCore\DataX.ServiceHost.AspNetCore.csproj" />
+    <ProjectReference Include="..\..\DataX.ServiceHost\DataX.ServiceHost\DataX.ServiceHost.csproj" />
     <ProjectReference Include="..\..\DataX.Utilities\DataX.Utilities.KeyVault\DataX.Utilities.KeyVault.csproj" />
     <ProjectReference Include="..\..\DataX.Utilities\DataX.Utilities.Telemetry\DataX.Utilities.Telemetry.csproj" />
     <ProjectReference Include="..\..\DataX.Utilities\DataX.Utilities.Web\DataX.Utilities.Web.csproj" />

Services/DataX.Flow/Flow.ManagementService/FlowManagementService.cs

@@ -26,9 +26,13 @@ namespace Flow.Management
     /// </summary>
     internal sealed class FlowManagementService : StatelessService
     {
-        public FlowManagementService(StatelessServiceContext context)
+        private readonly IWebHostBuilder _webHostBuilder;
+
+        public FlowManagementService(StatelessServiceContext context, IWebHostBuilder webHostBuilder)
             : base(context)
-        { }
+        {
+            _webHostBuilder = webHostBuilder;
+        }
 
         /// <summary>
         /// Optional override to create listeners (like tcp, http) for this service instance.
@@ -43,15 +47,13 @@ protected override IEnumerable<ServiceInstanceListener> CreateServiceInstanceLis
                     {
                         ServiceEventSource.Current.ServiceMessage(serviceContext, $"Starting Kestrel on {url}");
 
-                        return new WebHostBuilder()
-                                    .UseKestrel()
+
+
+                        return _webHostBuilder
                                     .ConfigureServices(
                                         services => services
                                             .AddSingleton<StatelessServiceContext>(serviceContext)
                                              )
-                                    .UseContentRoot(Directory.GetCurrentDirectory())
-                                    .UseStartup<Startup>()
-
                                     .UseServiceFabricIntegration(listener, ServiceFabricIntegrationOptions.None)
                                     .UseUrls(url)
                                     .Build();