Skip to content

Commit

Permalink
Windows CI working (#5477)
Browse files Browse the repository at this point in the history
* ci testing changes

* comments from pr
  • Loading branch information
michaelgsharp committed Nov 18, 2020
1 parent 76d9a63 commit 0ab9dc4
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 76 deletions.
70 changes: 35 additions & 35 deletions .vsts-dotnet-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,47 +6,47 @@ resources:
containers:
- container: CentosContainer
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-8bba86b-20190314145033

- container: UbuntuContainer
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-20200515184230-2c829e8

jobs:
- template: /build/ci/job-template.yml
parameters:
name: Centos_x64_NetCoreApp31
buildScript: ./build.sh
container: CentosContainer
customMatrixes:
Debug_Build:
_configuration: Debug-netcoreapp3_1
_config_short: DI
_includeBenchmarkData: false
_targetFramework: netcoreapp3.1
Release_Build:
_configuration: Release-netcoreapp3_1
_config_short: RI
_includeBenchmarkData: true
_targetFramework: netcoreapp3.1
innerLoop: true
pool:
name: Hosted Ubuntu 1604
# - template: /build/ci/job-template.yml
# parameters:
# name: Centos_x64_NetCoreApp31
# buildScript: ./build.sh
# container: CentosContainer
# customMatrixes:
# Debug_Build:
# _configuration: Debug-netcoreapp3_1
# _config_short: DI
# _includeBenchmarkData: false
# _targetFramework: netcoreapp3.1
# Release_Build:
# _configuration: Release-netcoreapp3_1
# _config_short: RI
# _includeBenchmarkData: true
# _targetFramework: netcoreapp3.1
# innerLoop: true
# pool:
# name: Hosted Ubuntu 1604

- template: /build/ci/job-template.yml
parameters:
name: Ubuntu_x64_NetCoreApp21
buildScript: ./build.sh
container: UbuntuContainer
innerLoop: true
pool:
name: Hosted Ubuntu 1604
# - template: /build/ci/job-template.yml
# parameters:
# name: Ubuntu_x64_NetCoreApp21
# buildScript: ./build.sh
# container: UbuntuContainer
# innerLoop: true
# pool:
# name: Hosted Ubuntu 1604

- template: /build/ci/job-template.yml
parameters:
name: MacOS_x64_NetCoreApp21
buildScript: ./build.sh
innerLoop: true
pool:
name: Hosted macOS
# - template: /build/ci/job-template.yml
# parameters:
# name: MacOS_x64_NetCoreApp21
# buildScript: ./build.sh
# innerLoop: true
# pool:
# name: Hosted macOS

- template: /build/ci/job-template.yml
parameters:
Expand Down
2 changes: 1 addition & 1 deletion build.cmd
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
@echo off
powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build -pack -warnAsError 0 %*"
powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build -warnAsError 0 %*"
exit /b %ErrorLevel%
60 changes: 28 additions & 32 deletions build/ci/job-template.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#TODO: Need to update build documentation.
parameters:
name: ''
architecture: x64
Expand All @@ -21,7 +22,7 @@ jobs:
timeoutInMinutes: 120
cancelTimeoutInMinutes: 10
variables:
dotnetPath: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet
dotnetPath: $(Build.SourcesDirectory)/.dotnet/dotnet
nugetFeed: https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json
nightlyBuildProjPath: $(Build.SourcesDirectory)/test/Microsoft.ML.NightlyBuild.Tests/Microsoft.ML.NightlyBuild.Tests.csproj
nightlyBuildRunPath: $(Build.SourcesDirectory)/bin/AnyCPU.$(_configuration)/Microsoft.ML.NightlyBuild.Tests/$(_targetFramework)
Expand All @@ -43,7 +44,7 @@ jobs:
_targetFramework: netcoreapp2.1
${{ if ne(parameters.customMatrixes, '') }}:
${{ insert }}: ${{ parameters.customMatrixes }}

pool: ${{ parameters.pool }}
${{ if ne(parameters.container, '') }}:
container: ${{ parameters.container }}
Expand All @@ -63,7 +64,7 @@ jobs:
- ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.pool.name, 'Hosted Ubuntu 1604')) }}:
- bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH"
displayName: Set LD_LIBRARY_PATH for Ubuntu and CentOS to locate Native shared library in current running path
- script: ${{ parameters.buildScript }} -$(_configuration) -buildArch=${{ parameters.architecture }}
- script: ${{ parameters.buildScript }} -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }}
displayName: Build
- ${{ if eq(parameters.pool.name, 'Hosted macOS') }}:
- task: Bash@3
Expand All @@ -78,11 +79,12 @@ jobs:
script: cd packages;find . -type d -path "*/runtimes/osx-*" -exec rm -rv {} +;find . -type d -path "*/runtimes/win-*" -exec rm -rv {} +;cd ..
displayName: Clean up non-Linux runtime folders of NuGet Packages to save disk space
- ${{ if eq(parameters.buildScript, 'build.cmd') }}:
- task: PowerShell@2
inputs:
targetType: inline
script: Get-ChildItem -Path '.\packages\*\runtimes\*' -Recurse | Select -ExpandProperty FullName | Where {$_ -notlike '*\win-*'} | sort length -Descending | Remove-Item -Recurse -Confirm:$false -Force
displayName: Clean up non-Windows runtime folders of NuGet Packages to save disk space
# TODO: We need to do this. THe packages are restored in a different folder so we need find the correct path.
# - task: PowerShell@2
# inputs:
# targetType: inline
# script: Get-ChildItem -Path '.\packages\*\runtimes\*' -Recurse | Select -ExpandProperty FullName | Where {$_ -notlike '*\win-*'} | sort length -Descending | Remove-Item -Recurse -Confirm:$false -Force
# displayName: Clean up non-Windows runtime folders of NuGet Packages to save disk space
- script: dir /s "bin"
displayName: show bin folder disk usage
- ${{ if eq(parameters.nightlyBuild, 'true') }}:
Expand All @@ -97,27 +99,22 @@ jobs:
Get-ChildItem -Path '.\bin\AnyCPU.*' -Recurse |
Select -ExpandProperty FullName |
Where {$_ -notlike '*\Microsoft.ML.NightlyBuild.Tests*'} |
sort length -Descending |
Remove-Item -force
sort length -Descending |
Remove-Item -force
Write-Output "Done cleaning up usless project..."
displayName: Clean up useless project
- script: $(dotnetPath) msbuild -restore $(nightlyBuildProjPath) /p:ReferenceTypeForTestFramework="Nuget" /p:Configuration=$(_configuration) /p:TargetArchitecture=${{ parameters.architecture }}
- script: $(dotnetPath) msbuild -restore $(nightlyBuildProjPath) /p:ReferenceTypeForTestFramework="Nuget" -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }}
displayName: Build Nightly-Build Project with latest package versions
- script: ${{ parameters.buildScript }} -$(_configuration) -runnightlybuildtests
displayName: Run Nightly Build Tests
- ${{ if eq(parameters.nightlyBuild, 'false') }}:
- script: ${{ parameters.buildScript }} -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData)
displayName: Download Benchmark Data
timeoutInMinutes: 10
- script: ${{ parameters.buildScript }} -- /t:DownloadTensorflowMetaFiles /p:IncludeTensorflowMetaFile=true
displayName: Download Tensorflow Meta File
timeoutInMinutes: 20
- ${{ if eq(parameters.innerLoop, 'false') }}:
- ${{ if and(eq(parameters.runSpecific, 'false'), eq(parameters.useVSTestTask, 'false')) }}:
- script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }}
# TODO: Code coverage needs to be fixed.
- script: ${{ parameters.buildScript }} -configuration $(_configuration) -test -ci /p:TestRunnerAdditionalArguments=-notrait%20Category=SkipInCI #-coverage=${{ parameters.codeCoverage }}
displayName: Run All Tests.
- ${{ if and(eq(parameters.runSpecific, 'true'), eq(parameters.useVSTestTask, 'false')) }}:
- script: ${{ parameters.buildScript }} -$(_configuration) -runSpecificTests -coverage=${{ parameters.codeCoverage }}
- script: ${{ parameters.buildScript }} -configuration $(_configuration) -ci /p:TestRunnerAdditionalArguments=-trait%20Category=RunSpecificTest #-coverage=${{ parameters.codeCoverage }}
displayName: Run Specific Tests.
- ${{ if and(eq(parameters.buildScript, 'build.cmd'), eq(parameters.useVSTestTask, 'true')) }}:
- task: VSTest@2
Expand All @@ -127,10 +124,10 @@ jobs:
testAssemblyVer2: |
**\*test.dll
**\*tests.dll
!**\obj\**
!**\obj\**
runSettingsFile: $(Build.SourcesDirectory)/tools-local/vstest.runsettings
searchFolder: '$(System.DefaultWorkingDirectory)'
vstestLocationMethod: 'version'
vstestLocationMethod: 'version'
vsTestVersion: 'latest'
runInParallel: False
runTestsInIsolation: True
Expand All @@ -141,17 +138,17 @@ jobs:
collectDumpOn: onAbortOnly
publishRunAttachments: true
- ${{ if eq(parameters.innerLoop, 'true') }}:
- script: ${{ parameters.buildScript }} -$(_configuration) -runCITests -coverage=${{ parameters.codeCoverage }}
- script: ${{ parameters.buildScript }} -configuration $(_configuration) -test -ci /p:TestRunnerAdditionalArguments=-notrait%20Category=SkipInCI #-coverage=${{ parameters.codeCoverage }}
displayName: Run CI Tests.
- script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet msbuild -restore build/Codecoverage.proj
- script: $(dotnetPath) msbuild -restore build/Codecoverage.proj
displayName: Upload coverage to codecov.io
condition: and(succeeded(), eq(${{ parameters.codeCoverage }}, True))
- task: PublishTestResults@2
displayName: Publish Test Results
condition: succeededOrFailed()
inputs:
testRunner: 'vSTest'
searchFolder: '$(System.DefaultWorkingDirectory)/bin'
searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults'
testResultsFiles: '**/*.trx'
testRunTitle: Machinelearning_Tests_${{ parameters.name }}_$(_configuration)_$(Build.BuildNumber)
configuration: $(_configuration)
Expand All @@ -160,17 +157,16 @@ jobs:
displayName: Stage build logs
condition: not(succeeded())
inputs:
sourceFolder: $(Build.SourcesDirectory)
contents: '?(msbuild.*|binclash.log|init-tools.log)'
targetFolder: $(Build.ArtifactStagingDirectory)
sourceFolder: $(Build.SourcesDirectory)/artifacts/log/
contents: '**'
targetFolder: $(Build.ArtifactStagingDirectory)artifacts/log/
- task: CopyFiles@2
displayName: Stage test output
condition: not(succeeded())
inputs:
sourceFolder: $(Build.SourcesDirectory)/bin
sourceFolder: $(Build.SourcesDirectory)/artifacts/TestResults
contents: |
**/TestOutput/**/*
**/*.trx
**
targetFolder: $(Build.ArtifactStagingDirectory)
- task: CopyFiles@2
displayName: Stage process dump and pdb if any
Expand All @@ -180,7 +176,7 @@ jobs:
contents: |
*.dmp
CrashDumps/*.dmp
bin/**/*.pdb
artifacts/bin/**/*.pdb
targetFolder: $(Build.ArtifactStagingDirectory)
- task: PublishBuildArtifacts@1
displayName: Publish build and test logs
Expand All @@ -190,5 +186,5 @@ jobs:
artifactName: ${{ parameters.name }} $(_config_short)
artifactType: container
- ${{ if eq(parameters.nightlyBuild, 'false') }}:
- script: ${{ parameters.buildScript }} -buildPackages
- script: ${{ parameters.buildScript }} -pack -ci
displayName: Build Packages
16 changes: 8 additions & 8 deletions build/vsts-ci.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
################################################################################
# ML.NET's official, signed build
# ML.NET's official, signed build
################################################################################

resources:
Expand Down Expand Up @@ -80,7 +80,7 @@ phases:
_TeamName: DotNetCore
queue:
name: DotNetCore-Build
demands:
demands:
- agent.os -equals Windows_NT
steps:

Expand All @@ -98,7 +98,7 @@ phases:
# Only build native assets to avoid conflicts.
- script: ./build.cmd -buildNative -$(BuildConfig) -buildArch=x86 -skipRIDAgnosticAssets
displayName: Build

- task: MSBuild@1
displayName: Sign Windows_x86 Binaries
inputs:
Expand Down Expand Up @@ -132,7 +132,7 @@ phases:
_TeamName: DotNetCore
queue:
name: DotNetCore-Build
demands:
demands:
- agent.os -equals Windows_NT
steps:

Expand All @@ -147,7 +147,7 @@ phases:
continueOnError: false
condition: and(succeeded(), in(variables._SignType, 'real', 'test'))

# Build both native and managed assets.
# Build both native and managed assets.
- script: ./build.cmd -$(BuildConfig)
displayName: Build

Expand All @@ -157,7 +157,7 @@ phases:
verbosity: 'Verbose'
alertWarningLevel: 'High'


- task: MSBuild@1
displayName: Sign Windows_x64 Binaries
inputs:
Expand Down Expand Up @@ -199,7 +199,7 @@ phases:
_MsdlSymbolServerPath: https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection
queue:
name: DotNetCore-Build
demands:
demands:
- agent.os -equals Windows_NT
steps:

Expand Down Expand Up @@ -228,7 +228,7 @@ phases:
inputs:
sourceFolder: $(Build.SourcesDirectory)/bin/obj/packages/PackageAssets
targetFolder: $(Build.SourcesDirectory)/bin/obj/packages

- script: ./build.cmd -buildPackages
displayName: Create Packages

Expand Down

0 comments on commit 0ab9dc4

Please sign in to comment.