diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 5c2dcf30cb..5d5bfe7c0a 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -6,47 +6,47 @@ resources: containers: - container: CentosContainer image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-8bba86b-20190314145033 - + - container: UbuntuContainer image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-20200515184230-2c829e8 jobs: -- template: /build/ci/job-template.yml - parameters: - name: Centos_x64_NetCoreApp31 - buildScript: ./build.sh - container: CentosContainer - customMatrixes: - Debug_Build: - _configuration: Debug-netcoreapp3_1 - _config_short: DI - _includeBenchmarkData: false - _targetFramework: netcoreapp3.1 - Release_Build: - _configuration: Release-netcoreapp3_1 - _config_short: RI - _includeBenchmarkData: true - _targetFramework: netcoreapp3.1 - innerLoop: true - pool: - name: Hosted Ubuntu 1604 +# - template: /build/ci/job-template.yml +# parameters: +# name: Centos_x64_NetCoreApp31 +# buildScript: ./build.sh +# container: CentosContainer +# customMatrixes: +# Debug_Build: +# _configuration: Debug-netcoreapp3_1 +# _config_short: DI +# _includeBenchmarkData: false +# _targetFramework: netcoreapp3.1 +# Release_Build: +# _configuration: Release-netcoreapp3_1 +# _config_short: RI +# _includeBenchmarkData: true +# _targetFramework: netcoreapp3.1 +# innerLoop: true +# pool: +# name: Hosted Ubuntu 1604 -- template: /build/ci/job-template.yml - parameters: - name: Ubuntu_x64_NetCoreApp21 - buildScript: ./build.sh - container: UbuntuContainer - innerLoop: true - pool: - name: Hosted Ubuntu 1604 +# - template: /build/ci/job-template.yml +# parameters: +# name: Ubuntu_x64_NetCoreApp21 +# buildScript: ./build.sh +# container: UbuntuContainer +# innerLoop: true +# pool: +# name: Hosted Ubuntu 1604 -- template: /build/ci/job-template.yml - parameters: - name: MacOS_x64_NetCoreApp21 - buildScript: ./build.sh - innerLoop: true - pool: - name: Hosted macOS +# - template: /build/ci/job-template.yml +# parameters: +# name: MacOS_x64_NetCoreApp21 +# buildScript: ./build.sh +# innerLoop: true +# pool: +# name: Hosted macOS - template: /build/ci/job-template.yml parameters: diff --git a/build.cmd b/build.cmd index 5e655ca47a..9aef81fd1f 100644 --- a/build.cmd +++ b/build.cmd @@ -1,3 +1,3 @@ @echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build -pack -warnAsError 0 %*" +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build -warnAsError 0 %*" exit /b %ErrorLevel% diff --git a/build/ci/job-template.yml b/build/ci/job-template.yml index 5f7b888fba..1358f25296 100644 --- a/build/ci/job-template.yml +++ b/build/ci/job-template.yml @@ -1,3 +1,4 @@ +#TODO: Need to update build documentation. parameters: name: '' architecture: x64 @@ -21,7 +22,7 @@ jobs: timeoutInMinutes: 120 cancelTimeoutInMinutes: 10 variables: - dotnetPath: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet + dotnetPath: $(Build.SourcesDirectory)/.dotnet/dotnet nugetFeed: https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json nightlyBuildProjPath: $(Build.SourcesDirectory)/test/Microsoft.ML.NightlyBuild.Tests/Microsoft.ML.NightlyBuild.Tests.csproj nightlyBuildRunPath: $(Build.SourcesDirectory)/bin/AnyCPU.$(_configuration)/Microsoft.ML.NightlyBuild.Tests/$(_targetFramework) @@ -43,7 +44,7 @@ jobs: _targetFramework: netcoreapp2.1 ${{ if ne(parameters.customMatrixes, '') }}: ${{ insert }}: ${{ parameters.customMatrixes }} - + pool: ${{ parameters.pool }} ${{ if ne(parameters.container, '') }}: container: ${{ parameters.container }} @@ -63,7 +64,7 @@ jobs: - ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.pool.name, 'Hosted Ubuntu 1604')) }}: - bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH" displayName: Set LD_LIBRARY_PATH for Ubuntu and CentOS to locate Native shared library in current running path - - script: ${{ parameters.buildScript }} -$(_configuration) -buildArch=${{ parameters.architecture }} + - script: ${{ parameters.buildScript }} -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} displayName: Build - ${{ if eq(parameters.pool.name, 'Hosted macOS') }}: - task: Bash@3 @@ -78,11 +79,12 @@ jobs: script: cd packages;find . -type d -path "*/runtimes/osx-*" -exec rm -rv {} +;find . -type d -path "*/runtimes/win-*" -exec rm -rv {} +;cd .. displayName: Clean up non-Linux runtime folders of NuGet Packages to save disk space - ${{ if eq(parameters.buildScript, 'build.cmd') }}: - - task: PowerShell@2 - inputs: - targetType: inline - script: Get-ChildItem -Path '.\packages\*\runtimes\*' -Recurse | Select -ExpandProperty FullName | Where {$_ -notlike '*\win-*'} | sort length -Descending | Remove-Item -Recurse -Confirm:$false -Force - displayName: Clean up non-Windows runtime folders of NuGet Packages to save disk space + # TODO: We need to do this. THe packages are restored in a different folder so we need find the correct path. + # - task: PowerShell@2 + # inputs: + # targetType: inline + # script: Get-ChildItem -Path '.\packages\*\runtimes\*' -Recurse | Select -ExpandProperty FullName | Where {$_ -notlike '*\win-*'} | sort length -Descending | Remove-Item -Recurse -Confirm:$false -Force + # displayName: Clean up non-Windows runtime folders of NuGet Packages to save disk space - script: dir /s "bin" displayName: show bin folder disk usage - ${{ if eq(parameters.nightlyBuild, 'true') }}: @@ -97,27 +99,22 @@ jobs: Get-ChildItem -Path '.\bin\AnyCPU.*' -Recurse | Select -ExpandProperty FullName | Where {$_ -notlike '*\Microsoft.ML.NightlyBuild.Tests*'} | - sort length -Descending | - Remove-Item -force + sort length -Descending | + Remove-Item -force Write-Output "Done cleaning up usless project..." displayName: Clean up useless project - - script: $(dotnetPath) msbuild -restore $(nightlyBuildProjPath) /p:ReferenceTypeForTestFramework="Nuget" /p:Configuration=$(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} + - script: $(dotnetPath) msbuild -restore $(nightlyBuildProjPath) /p:ReferenceTypeForTestFramework="Nuget" -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} displayName: Build Nightly-Build Project with latest package versions - script: ${{ parameters.buildScript }} -$(_configuration) -runnightlybuildtests displayName: Run Nightly Build Tests - ${{ if eq(parameters.nightlyBuild, 'false') }}: - - script: ${{ parameters.buildScript }} -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData) - displayName: Download Benchmark Data - timeoutInMinutes: 10 - - script: ${{ parameters.buildScript }} -- /t:DownloadTensorflowMetaFiles /p:IncludeTensorflowMetaFile=true - displayName: Download Tensorflow Meta File - timeoutInMinutes: 20 - ${{ if eq(parameters.innerLoop, 'false') }}: - ${{ if and(eq(parameters.runSpecific, 'false'), eq(parameters.useVSTestTask, 'false')) }}: - - script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }} + # TODO: Code coverage needs to be fixed. + - script: ${{ parameters.buildScript }} -configuration $(_configuration) -test -ci /p:TestRunnerAdditionalArguments=-notrait%20Category=SkipInCI #-coverage=${{ parameters.codeCoverage }} displayName: Run All Tests. - ${{ if and(eq(parameters.runSpecific, 'true'), eq(parameters.useVSTestTask, 'false')) }}: - - script: ${{ parameters.buildScript }} -$(_configuration) -runSpecificTests -coverage=${{ parameters.codeCoverage }} + - script: ${{ parameters.buildScript }} -configuration $(_configuration) -ci /p:TestRunnerAdditionalArguments=-trait%20Category=RunSpecificTest #-coverage=${{ parameters.codeCoverage }} displayName: Run Specific Tests. - ${{ if and(eq(parameters.buildScript, 'build.cmd'), eq(parameters.useVSTestTask, 'true')) }}: - task: VSTest@2 @@ -127,10 +124,10 @@ jobs: testAssemblyVer2: | **\*test.dll **\*tests.dll - !**\obj\** + !**\obj\** runSettingsFile: $(Build.SourcesDirectory)/tools-local/vstest.runsettings searchFolder: '$(System.DefaultWorkingDirectory)' - vstestLocationMethod: 'version' + vstestLocationMethod: 'version' vsTestVersion: 'latest' runInParallel: False runTestsInIsolation: True @@ -141,9 +138,9 @@ jobs: collectDumpOn: onAbortOnly publishRunAttachments: true - ${{ if eq(parameters.innerLoop, 'true') }}: - - script: ${{ parameters.buildScript }} -$(_configuration) -runCITests -coverage=${{ parameters.codeCoverage }} + - script: ${{ parameters.buildScript }} -configuration $(_configuration) -test -ci /p:TestRunnerAdditionalArguments=-notrait%20Category=SkipInCI #-coverage=${{ parameters.codeCoverage }} displayName: Run CI Tests. - - script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet msbuild -restore build/Codecoverage.proj + - script: $(dotnetPath) msbuild -restore build/Codecoverage.proj displayName: Upload coverage to codecov.io condition: and(succeeded(), eq(${{ parameters.codeCoverage }}, True)) - task: PublishTestResults@2 @@ -151,7 +148,7 @@ jobs: condition: succeededOrFailed() inputs: testRunner: 'vSTest' - searchFolder: '$(System.DefaultWorkingDirectory)/bin' + searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults' testResultsFiles: '**/*.trx' testRunTitle: Machinelearning_Tests_${{ parameters.name }}_$(_configuration)_$(Build.BuildNumber) configuration: $(_configuration) @@ -160,17 +157,16 @@ jobs: displayName: Stage build logs condition: not(succeeded()) inputs: - sourceFolder: $(Build.SourcesDirectory) - contents: '?(msbuild.*|binclash.log|init-tools.log)' - targetFolder: $(Build.ArtifactStagingDirectory) + sourceFolder: $(Build.SourcesDirectory)/artifacts/log/ + contents: '**' + targetFolder: $(Build.ArtifactStagingDirectory)artifacts/log/ - task: CopyFiles@2 displayName: Stage test output condition: not(succeeded()) inputs: - sourceFolder: $(Build.SourcesDirectory)/bin + sourceFolder: $(Build.SourcesDirectory)/artifacts/TestResults contents: | - **/TestOutput/**/* - **/*.trx + ** targetFolder: $(Build.ArtifactStagingDirectory) - task: CopyFiles@2 displayName: Stage process dump and pdb if any @@ -180,7 +176,7 @@ jobs: contents: | *.dmp CrashDumps/*.dmp - bin/**/*.pdb + artifacts/bin/**/*.pdb targetFolder: $(Build.ArtifactStagingDirectory) - task: PublishBuildArtifacts@1 displayName: Publish build and test logs @@ -190,5 +186,5 @@ jobs: artifactName: ${{ parameters.name }} $(_config_short) artifactType: container - ${{ if eq(parameters.nightlyBuild, 'false') }}: - - script: ${{ parameters.buildScript }} -buildPackages + - script: ${{ parameters.buildScript }} -pack -ci displayName: Build Packages diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml index 98dfa10e4a..c464d8c644 100644 --- a/build/vsts-ci.yml +++ b/build/vsts-ci.yml @@ -1,5 +1,5 @@ ################################################################################ -# ML.NET's official, signed build +# ML.NET's official, signed build ################################################################################ resources: @@ -80,7 +80,7 @@ phases: _TeamName: DotNetCore queue: name: DotNetCore-Build - demands: + demands: - agent.os -equals Windows_NT steps: @@ -98,7 +98,7 @@ phases: # Only build native assets to avoid conflicts. - script: ./build.cmd -buildNative -$(BuildConfig) -buildArch=x86 -skipRIDAgnosticAssets displayName: Build - + - task: MSBuild@1 displayName: Sign Windows_x86 Binaries inputs: @@ -132,7 +132,7 @@ phases: _TeamName: DotNetCore queue: name: DotNetCore-Build - demands: + demands: - agent.os -equals Windows_NT steps: @@ -147,7 +147,7 @@ phases: continueOnError: false condition: and(succeeded(), in(variables._SignType, 'real', 'test')) - # Build both native and managed assets. + # Build both native and managed assets. - script: ./build.cmd -$(BuildConfig) displayName: Build @@ -157,7 +157,7 @@ phases: verbosity: 'Verbose' alertWarningLevel: 'High' - + - task: MSBuild@1 displayName: Sign Windows_x64 Binaries inputs: @@ -199,7 +199,7 @@ phases: _MsdlSymbolServerPath: https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection queue: name: DotNetCore-Build - demands: + demands: - agent.os -equals Windows_NT steps: @@ -228,7 +228,7 @@ phases: inputs: sourceFolder: $(Build.SourcesDirectory)/bin/obj/packages/PackageAssets targetFolder: $(Build.SourcesDirectory)/bin/obj/packages - + - script: ./build.cmd -buildPackages displayName: Create Packages