Merge pull request dotnet#3 from dotnet/master

merge with latest master
ericstj · Aug 20, 2018 · bad9cd2 · bad9cd2
2 parents 7d0ea81 + 94401d5
commit bad9cd2
Show file tree

Hide file tree

Showing 220 changed files with 78,792 additions and 2,889 deletions.
diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml
@@ -11,5 +11,3 @@ phases:
     buildScript: build.cmd
     queue:
       name: Hosted VS2017
-      demands:
-        - agent.os -equals Windows_NT
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -16,7 +16,7 @@ If you are new to GitHub [here](https://help.github.com/categories/collaborating
 
 As a first time contributor, you will be invited to sign the Contributor License Agreement (CLA). Please follow the instructions of the dotnet foundation bot reviewer on your PR to sign the agreement indicating that you have appropriate rights to your contribution.
 
-Your pull request needs to reference a filed issue. Please fill in the template that is populated for the pull request. Only pull requests adressing small typos can have no issues associated with them.
+Your pull request needs to reference a filed issue. Please fill in the template that is populated for the pull request. Only pull requests addressing small typos can have no issues associated with them.
 
 An ML.NET team member will be assigned to your pull request once the continuous integration checks have passed successfully.
 

diff --git a/Directory.Build.props b/Directory.Build.props
@@ -114,4 +114,11 @@
     <PublicSign Condition="'$(OS)' != 'Windows_NT'">true</PublicSign>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <UseIntrinsics Condition="'$(UseIntrinsics)' == ''">$(Configuration.EndsWith('-Intrinsics'))</UseIntrinsics>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <CustomAfterMicrosoftCommonTargets>$(RepoRoot)build\AfterCommonTargets.targets</CustomAfterMicrosoftCommonTargets>
+  </PropertyGroup>
 </Project>
diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln
@@ -97,6 +97,19 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer",
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer.Tests", "test\Microsoft.ML.CodeAnalyzer.Tests\Microsoft.ML.CodeAnalyzer.Tests.csproj", "{3E4ABF07-7970-4BE6-B45B-A13D3C397545}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CpuMath.PerformanceTests", "test\Microsoft.ML.CpuMath.PerformanceTests\Microsoft.ML.CpuMath.PerformanceTests.csproj", "{7333EDEF-4144-405C-A5EC-6F42201857D8}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CpuMath.UnitTests.netstandard", "test\Microsoft.ML.CpuMath.UnitTests.netstandard\Microsoft.ML.CpuMath.UnitTests.netstandard.csproj", "{A0E562A9-0E6D-470D-B180-6EB44BA84D60}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CpuMath.UnitTests.netcoreapp", "test\Microsoft.ML.CpuMath.UnitTests.netcoreapp\Microsoft.ML.CpuMath.UnitTests.netcoreapp.csproj", "{5F81A2A4-73AD-494C-B387-07D605EC8826}"
+EndProject
+
+Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Microsoft.ML.FSharp.Tests", "test\Microsoft.ML.FSharp.Tests\Microsoft.ML.FSharp.Tests.fsproj", "{802233D6-8CC0-46AD-9F23-FEE1E9AED9B3}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.ImageAnalytics", "src\Microsoft.ML.ImageAnalytics\Microsoft.ML.ImageAnalytics.csproj", "{00E38F77-1E61-4CDF-8F97-1417D4E85053}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.HalLearners", "src\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.csproj", "{A7222F41-1CF0-47D9-B80C-B4D77B027A61}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -329,6 +342,54 @@ Global
 		{3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release|Any CPU.Build.0 = Release|Any CPU
 		{3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
 		{3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
+		{7333EDEF-4144-405C-A5EC-6F42201857D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7333EDEF-4144-405C-A5EC-6F42201857D8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7333EDEF-4144-405C-A5EC-6F42201857D8}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU
+		{7333EDEF-4144-405C-A5EC-6F42201857D8}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU
+		{7333EDEF-4144-405C-A5EC-6F42201857D8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7333EDEF-4144-405C-A5EC-6F42201857D8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7333EDEF-4144-405C-A5EC-6F42201857D8}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
+		{7333EDEF-4144-405C-A5EC-6F42201857D8}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
+		{A0E562A9-0E6D-470D-B180-6EB44BA84D60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A0E562A9-0E6D-470D-B180-6EB44BA84D60}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A0E562A9-0E6D-470D-B180-6EB44BA84D60}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU
+		{A0E562A9-0E6D-470D-B180-6EB44BA84D60}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU
+		{A0E562A9-0E6D-470D-B180-6EB44BA84D60}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A0E562A9-0E6D-470D-B180-6EB44BA84D60}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A0E562A9-0E6D-470D-B180-6EB44BA84D60}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
+		{A0E562A9-0E6D-470D-B180-6EB44BA84D60}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
+		{5F81A2A4-73AD-494C-B387-07D605EC8826}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5F81A2A4-73AD-494C-B387-07D605EC8826}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5F81A2A4-73AD-494C-B387-07D605EC8826}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU
+		{5F81A2A4-73AD-494C-B387-07D605EC8826}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU
+		{5F81A2A4-73AD-494C-B387-07D605EC8826}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5F81A2A4-73AD-494C-B387-07D605EC8826}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5F81A2A4-73AD-494C-B387-07D605EC8826}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
+		{5F81A2A4-73AD-494C-B387-07D605EC8826}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
+		{802233D6-8CC0-46AD-9F23-FEE1E9AED9B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{802233D6-8CC0-46AD-9F23-FEE1E9AED9B3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{802233D6-8CC0-46AD-9F23-FEE1E9AED9B3}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU
+		{802233D6-8CC0-46AD-9F23-FEE1E9AED9B3}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU
+		{802233D6-8CC0-46AD-9F23-FEE1E9AED9B3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{802233D6-8CC0-46AD-9F23-FEE1E9AED9B3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{802233D6-8CC0-46AD-9F23-FEE1E9AED9B3}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
+		{802233D6-8CC0-46AD-9F23-FEE1E9AED9B3}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release|Any CPU.Build.0 = Release|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -365,8 +426,14 @@ Global
 		{001F3B4E-FBE4-4001-AFD2-A6A989CD1C25} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{DCF46B79-1FDB-4DBA-A263-D3D64E3AAA27} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{BF66A305-DF10-47E4-8D81-42049B149D2B} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
+		{7333EDEF-4144-405C-A5EC-6F42201857D8} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{A0E562A9-0E6D-470D-B180-6EB44BA84D60} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{5F81A2A4-73AD-494C-B387-07D605EC8826} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 		{B4E55B2D-2A92-46E7-B72F-E76D6FD83440} = {7F13E156-3EBA-4021-84A5-CD56BA72F99E}
 		{3E4ABF07-7970-4BE6-B45B-A13D3C397545} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{802233D6-8CC0-46AD-9F23-FEE1E9AED9B3} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
+		{A7222F41-1CF0-47D9-B80C-B4D77B027A61} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

diff --git a/README.md b/README.md
@@ -18,7 +18,7 @@ Along with these ML capabilities this first release of ML.NET also brings the fi
 
 ML.NET runs on Windows, Linux, and macOS - any platform where 64 bit [.NET Core](https://github.com/dotnet/core) or later is available.
 
-The current release is 0.3. Check out the [release notes](docs/release-notes/0.3/release-0.3.md).
+The current release is 0.4. Check out the [release notes](docs/release-notes/0.4/release-0.4.md).
 
 First ensure you have installed [.NET Core 2.0](https://www.microsoft.com/net/learn/get-started) or later. ML.NET also works on the .NET Framework. Note that ML.NET currently must run in a 64 bit process.
 

diff --git a/build.proj b/build.proj
@@ -34,24 +34,26 @@
       BuildNative;
       $(TraversalBuildDependsOn);
       DownloadExternalTestFiles;
-      RunTests;
     </TraversalBuildDependsOn>
   </PropertyGroup>
 
   <Target Name="RestoreProjects" Condition="'$(RestoreDuringBuild)'=='true'">
     <Message Importance="High" Text="Restoring all projects..." />
     <MSBuild Projects="@(Project)"
-             Targets="Restore" />
+             Targets="Restore" 
+             Properties="MSBuildWarningsAsMessages=NU1503" />
   </Target>
 
-  <Target Name="BuildNative" Condition="'$(SkipNativeBuild)' != 'true'">
+  <Target Name="BuildNative"
+          Condition="'$(SkipNativeBuild)' != 'true'"
+          DependsOnTargets="RestoreProjects">
     <Message Importance="High" Text="Building native components..." />
     <MSBuild Projects="src/Native/build.proj"
              Targets="Build" />
   </Target>
 
   <Target Name="BuildPackages"
-          DependsOnTargets="CreateOrUpdateCurrentVersionFile">
+          DependsOnTargets="CreateOrUpdateCurrentVersionFile;RestoreProjects">
     <Message Importance="High" Text="Building packages ..." />
 
     <ItemGroup>
@@ -77,7 +79,7 @@
                          TreatErrorsAsWarnings="true"/>
   </Target>
 
-  <Target Name="RunTests" Condition="'$(RunTests)'=='true'">
+  <Target Name="RunTests">
     <MSBuild Projects="test\run-tests.proj"
              Targets="RunTests" />
   </Target>

diff --git a/build/AfterCommonTargets.targets b/build/AfterCommonTargets.targets
@@ -0,0 +1,13 @@
+<Project>
+  <PropertyGroup> 
+    <MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
+  </PropertyGroup> 
+
+  <!-- 
+  We use netcoreapp3.0 for C# intrinsics, but 3.0 isn't supported in CI or in normal development 
+  environments yet. So when we are targeting netcoreapp3.0, but aren't building for intrinsics, 
+  we need to skip the project. 
+  --> 
+  <Import Condition="'$(UseIntrinsics)' != 'true' and '$(TargetFramework)' == 'netcoreapp3.0'" 
+          Project="$(RepoRoot)build\Empty.targets" />
+</Project>
diff --git a/build/BranchInfo.props b/build/BranchInfo.props
@@ -1,7 +1,7 @@
 <Project>
   <PropertyGroup>
     <MajorVersion>0</MajorVersion>
-    <MinorVersion>4</MinorVersion>
+    <MinorVersion>5</MinorVersion>
     <PatchVersion>0</PatchVersion>
     <PreReleaseLabel>preview</PreReleaseLabel>
   </PropertyGroup>

diff --git a/build/Dependencies.props b/build/Dependencies.props
@@ -8,5 +8,7 @@
     <SystemReflectionEmitLightweightPackageVersion>4.3.0</SystemReflectionEmitLightweightPackageVersion>
     <PublishSymbolsPackageVersion>1.0.0-beta-62824-02</PublishSymbolsPackageVersion>
     <LightGBMPackageVersion>2.1.2.2</LightGBMPackageVersion>
+    <MlNetMklDepsPackageVersion>0.0.0.5</MlNetMklDepsPackageVersion>
+    <SystemDrawingCommonPackageVersion>4.5.0</SystemDrawingCommonPackageVersion>
   </PropertyGroup>
 </Project>
diff --git a/build/Empty.targets b/build/Empty.targets
@@ -0,0 +1,29 @@
+<Project> 
+  <PropertyGroup> 
+    <MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
+    <!--
+      In the SDK ImportAfter folder, this property is declared to point to Microsoft.TestPlatform.targets, which is the file containing the original VSTest target.
+      Since the Microsoft.TestPlatform.targets are in the ImportAfter folder, they would be imported after this file hence our empty VSTest target would be overriden,
+      in order to be able to override this target, we set the VSTestTargets property to an inexistent file path, so nothing will be imported and that way we successfully
+      overrode the VSTest target.
+    -->
+    <VSTestTargets>ignore.targets</VSTestTargets>
+  </PropertyGroup>
+
+  <!-- 
+    Copied from https://github.com/dotnet/arcade/blob/master/src/Microsoft.DotNet.Arcade.Sdk/tools/Empty.targets 
+
+    Import this file to suppress all targets while allowing the project to participate in the build. 
+    Workaround for https://github.com/dotnet/sdk/issues/2071. 
+
+    The targets defined here are not sufficient for the project to be open in Visual Studio without issues though.     
+  --> 
+
+  <Target Name="_IsProjectRestoreSupported"/>
+  <Target Name="_CheckForInvalidConfigurationAndPlatform"/>
+  <Target Name="Restore"/>
+  <Target Name="Build"/>
+  <Target Name="Test"/>
+  <Target Name="VSTest"/>
+  <Target Name="Pack"/>
+</Project>
diff --git a/build/ci/phase-template.yml b/build/ci/phase-template.yml
@@ -13,16 +13,18 @@ phases:
       ${{ if ne(parameters.dockerImage, '') }}:
         _PREVIEW_VSTS_DOCKER_IMAGE: ${{ parameters.dockerImage }}
     queue:
-      parallel: 2
+      parallel: 99
       matrix:
         Build_Debug:
           _configuration: Debug
         Build_Release:
-          _configuration: Release 
+          _configuration: Release
       ${{ insert }}: ${{ parameters.queue }}
     steps:
+    - script: $(_buildScript) -$(_configuration)
+      displayName: Build
     - script: $(_buildScript) -$(_configuration) -runtests
-      displayName: Build and Test
+      displayName: Run Tests
     - task: PublishTestResults@2
       displayName: Publish Test Results
       condition: succeededOrFailed()

diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml
@@ -10,7 +10,7 @@ phases:
     DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
     DOTNET_MULTILEVEL_LOOKUP: 0
   queue:
-    name: DotNetCore-Test
+    name: DotNet-Build
     demands:
     - agent.os -equals linux
   steps:

diff --git a/config.json b/config.json
@@ -67,8 +67,8 @@
       "defaultValue": ""
     },
     "RunTests": {
-      "description": "Run tests after building.",
-      "valueType": "property",
+      "description": "MsBuild target that run the tests. Call this after building.",
+      "valueType": "target",
       "values": [],
       "defaultValue": ""
     },
@@ -113,9 +113,9 @@
           }
         },
         "runtests": {
-          "description": "Runs the tests after building.",
+          "description": "Runs the tests. Call this after building.",
           "settings": {
-            "RunTests": "true"
+            "RunTests": "default"
           }
         },
         "verbose": {

diff --git a/docs/building/windows-instructions.md b/docs/building/windows-instructions.md
@@ -46,14 +46,14 @@ You can use the Developer Command Prompt, Powershell or work in any regular cmd.
 From a (non-admin) Command Prompt window:
 
 - `build.cmd` - builds the assemblies
-- `build.cmd -runTests` - builds the assemblies and runs tests
+- `build.cmd -runTests` - called after a normal "build.cmd" will run all tests
 - `build.cmd -buildPackages` called after a normal “build.cmd” will create the NuGet packages with the assemblies in “bin"
 
 **Note**: Before working on individual projects or test projects you **must** run `build.cmd` from the root once before beginning that work. It is also a good idea to run `build.cmd` whenever you pull a large set of unknown changes into your branch.
 
 ### Running tests from the command line
 
-From the root, use `build.cmd -runTests`.
+From the root, run `build.cmd` and then `build.cmd -runTests`.
 For more details, or to test an individual project, you can navigate to the test project directory and then use `dotnet test`
 
 ### Running tests from Visual Studio

diff --git a/docs/project-docs/developer-guide.md b/docs/project-docs/developer-guide.md
@@ -32,7 +32,8 @@ build.cmd -Release -TargetArchitecture:x64
 
 - Building the src and then building and running the tests
 ```
-build.cmd -RunTests
+build.cmd
+build.cmd -runTests
 ```
 
 ### Building individual projects