From 7865d765a591b355cd1563b5bb0c72f66ff44613 Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Fri, 16 Sep 2022 13:02:04 -0700 Subject: [PATCH 1/3] Switch repo to use native AzDO container support This is a major refactor of the YAML used by the installer repo. The goal is to utilize the native container support that AzDO provides, rather than building containers on the fly and issuing commands using custom infrastructure. To do this, the YAML requires a bit of a refactor. The matrix strategy approach used by the repo to build a ton of different OS's does not work with containers, because a matrix strategy only changes the variables available to each build command. It cannot change the AzDO host environment. In order to resolve this, I refactored build.yml to take and use optional container names. In the process of doing this, I discovered a number of other things about the old YAML that just happened to 'accidentally' work and fixed them or did general cleanup. Including: - This construct, used in build.yml, does not work as you might expect. If the parameter is not declared as a boolean (or not declared at all), this evaluates to "does this parameter exist", not "is it true": ``` ${{ if parameters.pgoInstrument }}: ``` - I fully specified all the parameters and their types to avoid issues in the future. - Build pool selection was moved to build.yml - Removed some unused parameters. - There was a **very** subtle indentation change here: https://github.com/dotnet/installer/blob/762d2966ee649e9f8174747d528aa067eaa7c4f0/.vsts-ci.yml#L275-L281. This meant that this leg ran in both PR and official builds. I have no idea whether this was the intention or not, but I kept it this way and reorganized the file. **One change of note:** One of the upsides of the original matrix based approach is that job dependencies are simple. The jobs generated by the matrix are referred to in `dependsOn` lists only by the original job that contains the matrix. That keeps the dependsOn list small even if the number of jobs generated is large. Installer has a large set of independently addressable jobs now. Normally, we would solve this by using the arcade jobs template. The jobs template takes a set of jobs and automatically adds dependsOn for the publishing jobs. BUT, AzDO does not allow templates to be passed as parameters to other templates. This would mean that you couldn't use the build.yml template in conjunction with the jobs template and would have to list all the installer jobs as dependencies. This list would be hard to keep up to date. To avoid this, I used a new feature of publishing (`publishAssetsImmediately`) which uses the Publish To Build Asset Registry job to do the actual publishing call, and put it in a separate phase, then eliminated the post-build.yml call. This means that the publishing stage depends on all jobs in the build stage, and does not need to address them individually. Eliminating the post-build.yml stage may seem odd, but this is what actually happens when `publishAssetsImmediately` is set to true anyway. --- .vsts-ci.yml | 587 ++++++++++++++--------------- build.sh | 17 +- eng/build.yml | 181 ++++++--- eng/docker/alpine.3.15/Dockerfile | 34 -- eng/docker/centos/Dockerfile | 25 -- eng/docker/debian/Dockerfile | 48 --- eng/docker/fedora.36/Dockerfile | 30 -- eng/docker/ubuntu.18.04/Dockerfile | 34 -- eng/dockerrun.ps1 | 60 --- eng/dockerrun.sh | 179 --------- 10 files changed, 420 insertions(+), 775 deletions(-) delete mode 100644 eng/docker/alpine.3.15/Dockerfile delete mode 100644 eng/docker/centos/Dockerfile delete mode 100644 eng/docker/debian/Dockerfile delete mode 100644 eng/docker/fedora.36/Dockerfile delete mode 100644 eng/docker/ubuntu.18.04/Dockerfile delete mode 100644 eng/dockerrun.ps1 delete mode 100755 eng/dockerrun.sh diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 1e6c32e9e069..10fc7ccf5247 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -25,17 +25,6 @@ variables: - name: _PublishUsingPipelines value: true -# Default to running tests in PRs and public CI, but not in official builds -- name: _WindowsTestArg - value: '-test' -- name: _NonWindowsTestArg - value: '--test' -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - name: _WindowsTestArg - value: '' - - name: _NonWindowsTestArg - value: '' - - name: _InternalRuntimeDownloadArgs value: '' @@ -47,8 +36,9 @@ variables: /p:dotnetbuilds-internal-container-read-token-base64=$(dotnetbuilds-internal-container-read-token-base64) stages: -- stage: build +- stage: Build jobs: + # This job is for build retry configuration. - job: Publish_Build_Configuration pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: @@ -58,275 +48,300 @@ stages: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre steps: - - publish: $(Build.SourcesDirectory)\eng\BuildConfiguration - artifact: BuildConfiguration + - publish: $(Build.SourcesDirectory)\eng\buildConfiguration + artifact: buildConfiguration displayName: Publish Build Config - - template: /eng/build.yml - parameters: - agentOs: Windows_NT - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Public - demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre - timeoutInMinutes: 180 - strategy: - matrix: - # Public-only builds - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - Build_Release_x64: - _BuildConfig: Debug - _BuildArchitecture: x64 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '/p:PublishInternalAsset=true' - _TestArg: $(_WindowsTestArg) - # Internal-only builds - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - Build_Release_x64: - _BuildConfig: Release - _BuildArchitecture: x64 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '/p:PublishInternalAsset=true' - _TestArg: $(_WindowsTestArg) - Build_Release_x86: - _BuildConfig: Release - _BuildArchitecture: x86 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '' - _TestArg: $(_WindowsTestArg) - Build_Release_arm64: - _BuildConfig: Release - _BuildArchitecture: arm64 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '' - # Never run tests on arm64 - _TestArg: '' + + ## PR-only jobs + + - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + + ## Windows + + - template: /eng/build.yml + parameters: + agentOs: Windows_NT + jobName: Build_Debug_x64 + buildConfiguration: Debug + buildArchitecture: x64 + additionalBuildParameters: '/p:PublishInternalAsset=true' + runTests: true + + ## Linux + + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Ubuntu_18_04_Debug_x64 + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20220916154732-3c53da6' + buildConfiguration: Debug + buildArchitecture: x64 + linuxPortable: true + runTests: true + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Fedora_36_Debug_x64 + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36-20220912173100-a09384f' + buildConfiguration: Debug + buildArchitecture: x64 + linuxPortable: true + runTests: true + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_CentOS_7_Debug_x64 + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-20220912172913-d16db59' + buildConfiguration: Debug + buildArchitecture: x64 + linuxPortable: false + runTests: true + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Debian_Stretch_Debug_x64 + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:debian-stretch-20220912173009-94fc78a' + buildConfiguration: Debug + buildArchitecture: x64 + additionalBuildParameters: '/p:BuildSdkDeb=true' + linuxPortable: false + runTests: true + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Arm64_Debug + buildConfiguration: Debug + buildArchitecture: arm64 + runtimeIdentifier: 'linux-arm64' + linuxPortable: true + # Never run tests on arm64 + runTests: false + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Linux_musl_Debug_x64 + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.15-WithNode-20220916182008-f0ea7ba' + buildConfiguration: Debug + buildArchitecture: x64 + runtimeIdentifier: 'linux-musl-x64' + # Pass in HostOSName when running on alpine + additionalBuildParameters: '/p:HostOSName="linux-musl"' + linuxPortable: false + runTests: true + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_LinuxPortable_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + linuxPortable: true + runTests: true + + # MacOS + + - template: /eng/build.yml + parameters: + agentOs: Darwin + jobName: Build_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + runTests: true + ## Official/PGO instrumentation Builds + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + + ## Windows + + - template: /eng/build.yml + parameters: + agentOs: Windows_NT + jobName: Build_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + additionalBuildParameters: '/p:PublishInternalAsset=true' + runTests: false + - template: /eng/build.yml + parameters: + agentOs: Windows_NT + jobName: Build_Release_x86 + buildConfiguration: Release + buildArchitecture: x86 + runTests: false + - template: /eng/build.yml + parameters: + agentOs: Windows_NT + jobName: Build_Release_arm64 + buildConfiguration: Release + buildArchitecture: arm64 + runTests: false + + ## Linux + + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Arm_Release + buildConfiguration: Release + buildArchitecture: arm + runtimeIdentifier: 'linux-arm' + linuxPortable: true + runTests: false + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Arm64_Release + buildConfiguration: Release + buildArchitecture: arm64 + runtimeIdentifier: 'linux-arm64' + linuxPortable: true + runTests: false + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Linux_musl_Release_arm + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-20220916154619-56ef508' + buildConfiguration: Release + buildArchitecture: arm + runtimeIdentifier: 'linux-musl-arm' + additionalBuildParameters: '/p:OSName="linux-musl"' + linuxPortable: false + runTests: false + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Linux_musl_Release_arm64 + buildConfiguration: Release + buildArchitecture: arm64 + runtimeIdentifier: 'linux-musl-arm64' + additionalBuildParameters: '/p:OSName="linux-musl"' + linuxPortable: false + runTests: false + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Linux_musl_Release_x64 + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.15-WithNode-20220916182008-f0ea7ba' + buildConfiguration: Release + buildArchitecture: x64 + runtimeIdentifier: 'linux-musl-x64' + # Pass in HostOSName when running on alpine + additionalBuildParameters: '/p:OSName="linux-musl"' + linuxPortable: false + runTests: false + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Linux_Portable_Deb_Release_x64 + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-debpkg-20220916154732-cfdd435' + buildConfiguration: Release + buildArchitecture: x64 + # Do not publish zips and tarballs. The linux-x64 binaries are + # already published by Build_LinuxPortable_Release_x64 + additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:BuildSdkDeb=true' + linuxPortable: true + runTests: false + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Linux_Portable_Rpm_Release_x64 + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-rpmpkg-20220912172913-d0fa36f' + buildConfiguration: Release + buildArchitecture: x64 + # Do not publish zips and tarballs. The linux-x64 binaries are + # already published by Build_LinuxPortable_Release_x64 + additionalBuildParameters: '/p:PublishBinariesAndBadge=false' + linuxPortable: true + runTests: false + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Linux_Portable_Rpm_Release_Arm64 + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-rpmpkg-20220912172913-d0fa36f' + buildConfiguration: Release + buildArchitecture: arm64 + runtimeIdentifier: 'linux-arm64' + # Do not publish zips and tarballs. The linux-x64 binaries are + # already published by Build_LinuxPortable_Release_x64 + additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:CLIBUILD_SKIP_TESTS=true' + linuxPortable: true + runTests: false + - template: /eng/build.yml + parameters: + agentOs: Linux + jobName: Build_LinuxPortable_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + linuxPortable: true + runTests: false + + # MacOS + + - template: /eng/build.yml + parameters: + agentOs: Darwin + jobName: Build_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + runTests: false + - template: /eng/build.yml + parameters: + agentOs: Darwin + jobName: Build_Release_arm64 + runtimeIdentifier: 'osx-arm64' + buildConfiguration: Release + buildArchitecture: arm64 + runTests: false + + ## Windows PGO Instrumentation builds + + - template: /eng/build.yml + parameters: + agentOs: Windows_NT + pgoInstrument: true + jobName: Build_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + additionalBuildParameters: '/p:PublishInternalAsset=true' + runTests: false - template: /eng/build.yml parameters: agentOs: Windows_NT - pool: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre - timeoutInMinutes: 180 - strategy: - matrix: - Build_Release_x64: - _BuildConfig: Release - _BuildArchitecture: x64 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '/p:PublishInternalAsset=true' - # Never run tests on PGO bits - _TestArg: '' - Build_Release_x86: - _BuildConfig: Release - _BuildArchitecture: x86 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '' - _TestArg: '' - Build_Release_arm64: - _BuildConfig: Release - _BuildArchitecture: arm64 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '' - # Never run tests on arm64 - _TestArg: '' pgoInstrument: true + jobName: Build_Release_x86 + buildConfiguration: Release + buildArchitecture: x86 + runTests: false + - template: /eng/build.yml + parameters: + agentOs: Windows_NT + pgoInstrument: true + jobName: Build_Release_arm64 + buildConfiguration: Release + buildArchitecture: arm64 + runTests: false - - template: /eng/build.yml - parameters: - agentOs: Linux - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Public - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - timeoutInMinutes: 180 - strategy: - matrix: - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - Build_Ubuntu_18_04_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker ubuntu.18.04' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) - Build_Fedora_36_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker fedora.36' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) - Build_CentOS_7_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker centos' - _LinuxPortable: '' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) - Build_Debian_Stretch_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker debian' - _LinuxPortable: '' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _AdditionalBuildParameters: '/p:BuildSdkDeb=true' - _TestArg: $(_NonWindowsTestArg) - Build_Arm64_Debug: - _BuildConfig: Debug - _DockerParameter: '' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '--runtime-id linux-arm64' - _BuildArchitecture: 'arm64' - # Never run tests on arm64 - _TestArg: '' - Build_Linux_musl_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker alpine.3.15' - _LinuxPortable: '' - _RuntimeIdentifier: '--runtime-id linux-musl-x64' - _BuildArchitecture: 'x64' - # Pass in HostOSName when running on alpine - _AdditionalBuildParameters: '/p:HostOSName="linux-musl"' - _TestArg: $(_NonWindowsTestArg) - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - Build_Arm_Release: - _BuildConfig: Release - _DockerParameter: '' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '--runtime-id linux-arm' - _BuildArchitecture: 'arm' - # Never run tests on arm - _TestArg: '' - Build_Arm64_Release: - _BuildConfig: Release - _DockerParameter: '' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '--runtime-id linux-arm64' - _BuildArchitecture: 'arm64' - # Never run tests on arm64 - _TestArg: '' - Build_Linux_musl_Release_arm: - _BuildConfig: Release - # linux-musl-arm cross gen depends on glibc 2.27 (this OS has it) - _DockerParameter: '--docker ubuntu.18.04' - _LinuxPortable: '' - _RuntimeIdentifier: '--runtime-id linux-musl-arm' - _BuildArchitecture: 'arm' - _AdditionalBuildParameters: '/p:OSName="linux-musl"' - # Never run tests on arm - _TestArg: '' - Build_Linux_musl_Release_arm64: - _BuildConfig: Release - _DockerParameter: '' - _LinuxPortable: '' - _RuntimeIdentifier: '--runtime-id linux-musl-arm64' - _BuildArchitecture: 'arm64' - _AdditionalBuildParameters: '/p:OSName="linux-musl"' - # Never run tests on arm64 - _TestArg: '' - Build_Linux_musl_Release_x64: - _BuildConfig: Release - _DockerParameter: '--docker alpine.3.15' - _LinuxPortable: '' - _RuntimeIdentifier: '--runtime-id linux-musl-x64' - _BuildArchitecture: 'x64' - # Pass in HostOSName when running on alpine - _AdditionalBuildParameters: '/p:HostOSName="linux-musl"' - Build_Linux_Portable_Deb_Release_x64: - _BuildConfig: Release - _DockerParameter: '--docker ubuntu.18.04' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - # Do not publish zips and tarballs. The linux-x64 binaries are - # already published by Build_LinuxPortable_Release_x64 - _AdditionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:BuildSdkDeb=true' - _TestArg: $(_NonWindowsTestArg) - Build_Linux_Portable_Rpm_Release_x64: - _BuildConfig: Release - _DockerParameter: '--docker centos' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - # Do not publish zips and tarballs. The linux-x64 binaries are - # already published by Build_LinuxPortable_Release_x64 - _AdditionalBuildParameters: '/p:PublishBinariesAndBadge=false' - _TestArg: $(_NonWindowsTestArg) - Build_Linux_Portable_Rpm_Release_Arm64: - _BuildConfig: Release - _DockerParameter: '--docker centos' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '--runtime-id linux-arm64' - _BuildArchitecture: 'arm64' - # Do not publish zips and tarballs. The linux-x64 binaries are - # already published by Build_LinuxPortable_Release_x64 - _AdditionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:CLIBUILD_SKIP_TESTS=true' - # Never run tests on arm64 - _TestArg: '' - Build_LinuxPortable_Release_x64: - _BuildConfig: Release - _DockerParameter: '' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) + ## Linux PGO Instrumentation builds - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/build.yml parameters: agentOs: Linux - pool: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - timeoutInMinutes: 180 - strategy: - matrix: - Build_LinuxPortable_Release_x64: - _BuildConfig: Release - _DockerParameter: '' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: '' - Build_Release_arm64: - _BuildConfig: Release - _BuildArchitecture: arm64 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '' - # Never run tests on arm64 - _TestArg: '' pgoInstrument: true + jobName: Build_LinuxPortable_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + linuxPortable: true + runTests: false - - template: /eng/build.yml - parameters: - agentOs: Darwin - pool: - vmImage: 'macOS-latest' - timeoutInMinutes: 180 - strategy: - matrix: - Build_Release_x64: - _BuildConfig: Release - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - Build_Release_arm64: - _BuildConfig: Release - _RuntimeIdentifier: '--runtime-id osx-arm64' - _BuildArchitecture: 'arm64' - # Never run tests on arm64 - _TestArg: '' + - template: /eng/build.yml + parameters: + agentOs: Linux + pgoInstrument: true + jobName: Build_Release_arm64 + buildConfiguration: Release + buildArchitecture: arm64 + linuxPortable: true + runTests: false - template: /eng/common/templates/jobs/source-build.yml @@ -337,40 +352,16 @@ stages: dependsOn: Source_Build_Create_Tarball condition: eq(dependencies.Source_Build_Create_Tarball.outputs['Tarball_Build_Check._includeTarballBuild'], 'true') - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - stage: Publish + dependsOn: + - Build + jobs: - template: /eng/common/templates/job/publish-build-assets.yml parameters: - dependsOn: - - Windows_NT - - Linux - - Darwin - - Source_Build_Managed - - Source_Build_Create_Tarball publishUsingPipelines: true + publishAssetsImmediately: true pool: ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Windows.Amd64.VS2022 - - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: eng\common\templates\post-build\post-build.yml - parameters: - publishingInfraVersion: 3 - enableSymbolValidation: false - enableSigningValidation: false - enableNugetValidation: false - enableSourceLinkValidation: false - publishInstallersAndChecksums: true - SDLValidationParameters: - enable: false - params: ' -SourceToolsList @("policheck","credscan") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "dotnet-installer" - -TsaCodebaseName "dotnet-installer" - -TsaPublish $True' + demands: ImageOverride -equals Build.Windows.Amd64.VS2022 \ No newline at end of file diff --git a/build.sh b/build.sh index 7ff647c4060d..d65f55854d0f 100755 --- a/build.sh +++ b/build.sh @@ -29,11 +29,6 @@ args= while [[ $# > 0 ]]; do lowerI="$(echo $1 | awk '{print tolower($0)}')" case $lowerI in - --docker) - export BUILD_IN_DOCKER=1 - export DOCKER_IMAGENAME=$2 - shift - ;; --noprettyprint) export DOTNET_CORESDK_NOPRETTYPRINT=1 ;; @@ -44,14 +39,4 @@ while [[ $# > 0 ]]; do shift done -dockerbuild() -{ - BUILD_COMMAND=$DIR/run-build.sh $DIR/eng/dockerrun.sh --non-interactive "$@" -} - -# Check if we need to build in docker -if [ ! -z "$BUILD_IN_DOCKER" ]; then - dockerbuild $args -else - $DIR/run-build.sh $args -fi +$DIR/run-build.sh $args diff --git a/eng/build.yml b/eng/build.yml index c23bbb11b13f..a91073bc8ebe 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -1,65 +1,143 @@ parameters: # Agent OS identifier and used as job name - agentOs: '' +- name: agentOs + type: string - # Agent pool - pool: {} + # Job name +- name: jobName + type: string - # Additional variables - variables: {} - - # Build strategy - matrix - strategy: {} +# Container to run the build in, if any +- name: container + type: string + default: '' # Job timeout - timeoutInMinutes: 180 - - # Publish using pipelines - enablePublishUsingPipelines: true - -phases: +- name: timeoutInMinutes + type: number + default: 180 + +# Build configuration (Debug, Release) +- name: buildConfiguration + type: string + values: + - Debug + - Release + +# Build architecture +- name: buildArchitecture + type: string + values: + - arm + - arm64 + - x64 + - x86 + +# Linux portable. If true, passes portable switch to build +- name: linuxPortable + type: boolean + default: false + +# Runtime Identifier +- name: runtimeIdentifier + type: string + default: '' + +# UI lang +- name: dotnetCLIUILanguage + type: string + default: '' + +# Additional parameters +- name: additionalBuildParameters + type: string + default: '' + +# Run tests +- name: runTests + type: boolean + default: true + +# PGO instrumentation jobs +- name: pgoInstrument + type: boolean + default: false + +jobs: - template: /eng/common/templates/job/job.yml parameters: + # Set up the name of the job. ${{ if parameters.pgoInstrument }}: - name: PGO_${{ parameters.agentOs }} + name: PGO_${{ parameters.agentOs }}_${{ parameters.jobName }} ${{ if not(parameters.pgoInstrument) }}: - name: ${{ parameters.agentOs }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + name: ${{ parameters.agentOs }}_${{ parameters.jobName }} + + ## Set up the pool/machine info to be used based on the Agent OS ${{ if eq(parameters.agentOs, 'Windows_NT') }}: enableMicrobuild: true + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: NetCore-Svc-Public + demands: ImageOverride -equals build.windows.10.amd64.vs2019.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals build.windows.10.amd64.vs2019 + ${{ if eq(parameters.agentOs, 'Linux') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: NetCore-Svc-Public + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + container: ${{ parameters.container }} + ${{ if eq(parameters.agentOs, 'Darwin') }}: + pool: + vmImage: 'macOS-latest' + + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} enablePublishBuildAssets: true - # Don't use built-in arcade logic for publishing artifacts and test results, as we manually do this and give better names - #enablePublishBuildArtifacts: true - #enablePublishTestResults: true - enablePublishUsingPipelines: ${{parameters.enablePublishUsingPipelines}} + enablePublishUsingPipelines: true enableTelemetry: true helixRepo: dotnet/installer - pool: ${{ parameters.pool }} - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} workspace: clean: all +# Test parameters variables: - _PgoInstrument: '' - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - _PackArg: '-pack' + - ${{ if parameters.runTests }}: + - _TestArg: '-test' + - ${{ else }}: + - _TestArg: '' - ${{ if ne(parameters.agentOs, 'Windows_NT') }}: - _PackArg: '--pack' + - ${{ if parameters.runTests }}: + - _TestArg: '--test' + - ${{ else }}: + - _TestArg: '' + - ${{ if parameters.pgoInstrument }}: - _PgoInstrument: '/p:PgoInstrument=true' - _PackArg: '' + - ${{ if parameters.linuxPortable }}: + - _LinuxPortable: '--linux-portable' + + - ${{ if ne(parameters.runtimeIdentifier, '') }}: + - _RuntimeIdentifier: '--runtime-id ${{ parameters.runtimeIdentifier }}' + - ${{ else }}: + - _RuntimeIdentifier: '' + - _AgentOSName: ${{ parameters.agentOs }} - _TeamName: Roslyn-Project-System - _SignType: test - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)' - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-Symbol-Server-PATs - group: DotNet-HelixApi-Access - - group: DotNet-Blob-Feed - - _DotNetPublishToBlobFeed: true - _PushToVSFeed: true - _SignType: real - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) @@ -83,14 +161,14 @@ phases: - script: build.cmd $(_TestArg) $(_PackArg) -publish -ci -sign - -Configuration $(_BuildConfig) - -Architecture $(_BuildArchitecture) + -Configuration ${{ parameters.buildConfiguration }} + -Architecture ${{ parameters.buildArchitecture }} $(_BuildArgs) - $(_AdditionalBuildParameters) + ${{ parameters.additionalBuildParameters }} $(_InternalRuntimeDownloadArgs) displayName: Build env: - DOTNET_CLI_UI_LANGUAGE: $(_DOTNET_CLI_UI_LANGUAGE) + DOTNET_CLI_UI_LANGUAGE: ${{ parameters.dotnetCLIUILanguage }} - ${{ if ne(parameters.agentOs, 'Windows_NT') }}: - ${{ if ne(variables['System.TeamProject'], 'public') }}: @@ -106,13 +184,12 @@ phases: $(_TestArg) $(_PackArg) --publish --ci --noprettyprint - --configuration $(_BuildConfig) - $(_DockerParameter) - --architecture $(_BuildArchitecture) + --configuration ${{ parameters.buildConfiguration }} + --architecture ${{ parameters.buildArchitecture }} $(_LinuxPortable) $(_RuntimeIdentifier) $(_BuildArgs) - $(_AdditionalBuildParameters) + ${{ parameters.additionalBuildParameters }} $(_InternalRuntimeDownloadArgs) displayName: Build @@ -121,32 +198,34 @@ phases: $(_TestArg) --pack --publish --ci --noprettyprint - --configuration $(_BuildConfig) - --architecture $(_BuildArchitecture) + --configuration ${{ parameters.buildConfiguration }} + --architecture ${{ parameters.buildArchitecture }} $(_RuntimeIdentifier) $(_BuildArgs) - $(_AdditionalBuildParameters) + ${{ parameters.additionalBuildParameters }} $(_InternalRuntimeDownloadArgs) displayName: Build - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.agentOs, 'Windows_NT'), ne(variables['PostBuildSign'], 'true')) }}: - - task: NuGetCommand@2 - displayName: Push Visual Studio NuPkgs - inputs: - command: push - packagesToPush: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/NonShipping/VS.*.nupkg' - nuGetFeedType: external - publishFeedCredentials: 'DevDiv - VS package feed' - condition: and(succeeded(), ne(variables['PostBuildSign'], true), eq(variables['_PushToVSFeed'], 'true'), eq(variables['_DotNetPublishToBlobFeed'], 'true'), or(eq(variables['_BuildArchitecture'], 'x64'), eq(variables['_BuildArchitecture'], 'x86'))) + # In general, only push VS nupkgs if post-build signing is off, and this is not a PR + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables['PostBuildSign'], 'true')) }}: + # In addition, restrict to just Windows x86 or x64 builds (other windows builds do not produce VS redist packages.) + - ${{ if and(eq(parameters.agentOs, 'Windows_NT'), in(parameters.buildArchitecture, 'x64', 'x86')) }}: + - task: NuGetCommand@2 + displayName: Push Visual Studio NuPkgs + inputs: + command: push + packagesToPush: '$(Build.SourcesDirectory)/artifacts/packages/${{ parameters.buildConfiguration }}/NonShipping/VS.*.nupkg' + nuGetFeedType: external + publishFeedCredentials: 'DevDiv - VS package feed' - task: PublishTestResults@2 displayName: Publish Test Results inputs: testRunner: XUnit - testResultsFiles: 'artifacts/TestResults/$(_BuildConfig)/*.xml' + testResultsFiles: 'artifacts/TestResults/${{ parameters.buildConfiguration }}/*.xml' testRunTitle: '$(_AgentOSName)_$(Agent.JobName)' platform: '$(BuildPlatform)' - configuration: '$(_BuildConfig)' + configuration: '${{ parameters.buildConfiguration }}' condition: ne(variables['_TestArg'], '') - task: CopyFiles@2 @@ -154,8 +233,8 @@ phases: inputs: SourceFolder: '$(Build.SourcesDirectory)/artifacts' Contents: | - log/$(_BuildConfig)/**/* - TestResults/$(_BuildConfig)/**/* + log/${{ parameters.buildConfiguration }}/**/* + TestResults/${{ parameters.buildConfiguration }}/**/* TargetFolder: '$(Build.ArtifactStagingDirectory)' continueOnError: true condition: always() diff --git a/eng/docker/alpine.3.15/Dockerfile b/eng/docker/alpine.3.15/Dockerfile deleted file mode 100644 index 2d7e656981fe..000000000000 --- a/eng/docker/alpine.3.15/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -# Dockerfile that creates a container suitable to build dotnet-cli -FROM mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.15-20220610131707-f0ea7ba - -RUN apk update && apk upgrade && apk add --no-cache curl ncurses - -# This Dockerfile doesn't use the USER_ID, but the parameter needs to be declared to prevent docker -# from issuing a warning -ARG USER_ID=0 -RUN adduser code_executor -u ${USER_ID} -G root -D -RUN echo 'code_executor ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -# With the User Change, we need to change permissions on these directories -RUN chmod -R a+rwx /usr/local -RUN chmod -R a+rwx /home - -# Set working directory -ARG WORK_DIR -WORKDIR ${WORK_DIR} - -# Set up Azure Artifacts credential provider -# We download the installer and execute it using ${USER_ID} so that the installer -# put the NuGet plugins in the correct $HOME/.nuget folder. -RUN curl -O https://raw.githubusercontent.com/Microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh -RUN chmod +x ./installcredprovider.sh - -# Set user to the one we just created -USER ${USER_ID} - -RUN ./installcredprovider.sh diff --git a/eng/docker/centos/Dockerfile b/eng/docker/centos/Dockerfile deleted file mode 100644 index 1dfeaf8d270d..000000000000 --- a/eng/docker/centos/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -# Dockerfile that creates a container suitable to build dotnet-cli -# Via https://github.com/dotnet/versions/blob/main/build-info/docker/image-info.dotnet-dotnet-buildtools-prereqs-docker-main.json -FROM mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-rpmpkg-19d155e-20200221152150 - -# Setup User to match Host User, and give superuser permissions -ARG USER_ID=0 -RUN useradd -m code_executor -u ${USER_ID} -g root -RUN echo 'code_executor ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -# With the User Change, we need to change permissions on these directories -RUN chmod -R a+rwx /usr/local -RUN chmod -R a+rwx /home -RUN chmod -R 4755 /usr/bin/sudo - -# Set user to the one we just created -USER ${USER_ID} - -# Set working directory -ARG WORK_DIR -WORKDIR ${WORK_DIR} diff --git a/eng/docker/debian/Dockerfile b/eng/docker/debian/Dockerfile deleted file mode 100644 index ebafe9adb33c..000000000000 --- a/eng/docker/debian/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -# Dockerfile that creates a container suitable to build dotnet-cli -FROM mcr.microsoft.com/dotnet-buildtools/prereqs:debian-stretch-20211001171226-047508b - -# Install the deb packaging toolchain we need to build debs -RUN apt-get update \ - && apt-get -y install \ - debhelper \ - build-essential \ - devscripts \ - locales \ - && rm -rf /var/lib/apt/lists/* - -# liblldb is needed so deb package build does not throw missing library info errors -RUN apt-get update \ - && apt-get -y install liblldb-3.9 \ - && rm -rf /var/lib/apt/lists/* - -# Misc Dependencies for build -RUN rm -rf /var/lib/apt/lists/* && \ - apt-get update && \ - apt-get -qqy install \ - sudo && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN localedef -c -i en_US -f UTF-8 en_US.UTF-8 - -# Setup User to match Host User, and give superuser permissions -ARG USER_ID=0 -RUN useradd -m code_executor -u ${USER_ID} -g sudo -RUN echo 'code_executor ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -# With the User Change, we need to change permissions on these directories -RUN chmod -R a+rwx /usr/local -RUN chmod -R a+rwx /home -RUN chmod -R 755 /usr/lib/sudo - -# Set user to the one we just created -USER ${USER_ID} - -# Set working directory -ARG WORK_DIR -WORKDIR ${WORK_DIR} diff --git a/eng/docker/fedora.36/Dockerfile b/eng/docker/fedora.36/Dockerfile deleted file mode 100644 index d36d5c15ee6a..000000000000 --- a/eng/docker/fedora.36/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -# Dockerfile that creates a container suitable to build dotnet-cli -FROM mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36-20220716171953-531d246 - -RUN dnf install -y nss - -RUN dnf clean all - -# Override RID set by the dotnet-buildtools-prereqs docker image -ENV __PUBLISH_RID=fedora.36-x64 - -# Setup User to match Host User, and give superuser permissions -ARG USER_ID=0 -RUN useradd -m code_executor -u ${USER_ID} -g wheel -RUN echo 'code_executor ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -# With the User Change, we need to change permissions on these directories -RUN chmod -R a+rwx /usr/local -RUN chmod -R a+rwx /home - -# Set user to the one we just created -USER ${USER_ID} - -# Set working directory -ARG WORK_DIR -WORKDIR ${WORK_DIR} diff --git a/eng/docker/ubuntu.18.04/Dockerfile b/eng/docker/ubuntu.18.04/Dockerfile deleted file mode 100644 index 1f817500181f..000000000000 --- a/eng/docker/ubuntu.18.04/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -# Dockerfile that creates a container suitable to build dotnet-cli -FROM mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-f90bc20-20180320154721 - -RUN apt-get update && \ - apt-get -qqy install \ - curl \ - libcurl4 \ - devscripts \ - debhelper \ - sudo && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Setup User to match Host User, and give superuser permissions -ARG USER_ID=0 -RUN useradd -m code_executor -u ${USER_ID} -g sudo -RUN echo 'code_executor ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -# With the User Change, we need to change permissions on these directories -RUN chmod -R a+rwx /usr/local -RUN chmod -R a+rwx /home -RUN chmod -R 755 /usr/lib/sudo - -# Set user to the one we just created -USER ${USER_ID} - -# Set working directory -ARG WORK_DIR -WORKDIR ${WORK_DIR} diff --git a/eng/dockerrun.ps1 b/eng/dockerrun.ps1 deleted file mode 100644 index e3b117258855..000000000000 --- a/eng/dockerrun.ps1 +++ /dev/null @@ -1,60 +0,0 @@ -[CmdletBinding(PositionalBinding=$false)] -Param( - [bool] $noninteractive = $false, - [string] $dockerImageName, - [string] $dockerContainerTag = "dotnetcli-build", - [string] $dockerContainerName = "dotnetcli-build-container", - [Parameter(ValueFromRemainingArguments=$true)][String[]]$additionalArgs -) - -# sample command line: .\eng\dockerrun.ps1 -dockerImageName ubuntu.18.04 /p:DisableSourceLink=true --test --pack --publish - -Write-Host "Docker image name: $dockerImageName" -Write-Host "Additional args: $additionalArgs" - -. $PSScriptRoot\common\tools.ps1 - -# docker build -f old\scripts\docker\rhel\Dockerfile --build-arg USER_ID=1000 -t redhat . -# docker run -it -v c:\git\core-sdk-arcade:/opt/code redhat bash - -$dockerFile = Resolve-Path (Join-Path $RepoRoot "eng\docker\$dockerImageName") - -docker build --build-arg WORK_DIR=$RepoRoot --build-arg USER_ID=1000 -t "$dockerContainerTag" $dockerFile - -$interactiveFlag = "-i" -if ($noninteractive) -{ - $interactiveFlag = "" -} - -` # -e DOTNET_INSTALL_DIR=/opt/code/artifacts/docker/$dockerImageName/.dotnet ` - -docker run $interactiveFlag -t --rm --sig-proxy=true ` - --name "$dockerContainerName" ` - -v "${RepoRoot}:${RepoRoot}" ` - -e DOTNET_CORESDK_IGNORE_TAR_EXIT_CODE=1 ` - -e CHANNEL ` - -e DOTNET_BUILD_SKIP_CROSSGEN ` - -e PUBLISH_TO_AZURE_BLOB ` - -e NUGET_FEED_URL ` - -e NUGET_API_KEY ` - -e ARTIFACT_STORAGE_ACCOUNT ` - -e ARTIFACT_STORAGE_CONTAINER ` - -e CHECKSUM_STORAGE_ACCOUNT ` - -e CHECKSUM_STORAGE_CONTAINER ` - -e BLOBFEED_STORAGE_CONTAINER ` - -e CLIBUILD_SKIP_TESTS ` - -e COMMITCOUNT ` - -e DROPSUFFIX ` - -e RELEASESUFFIX ` - -e COREFXAZURECONTAINER ` - -e AZUREACCOUNTNAME ` - -e RELEASETOOLSGITURL ` - -e CORESETUPBLOBROOTURL ` - -e PB_ASSETROOTURL ` - -e PB_PACKAGEVERSIONPROPSURL ` - -e PB_PUBLISHBLOBFEEDURL ` - -e EXTERNALRESTORESOURCES ` - -e ARCADE_PARTITION="${dockerImageName}" ` - $dockerContainerTag ` - ${RepoRoot}/run-build.sh @additionalArgs \ No newline at end of file diff --git a/eng/dockerrun.sh b/eng/dockerrun.sh deleted file mode 100755 index 775ca367d009..000000000000 --- a/eng/dockerrun.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -set -e - -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - -cd "$DIR/.." - -INTERACTIVE="-i" - -while [[ $# > 0 ]]; do - key=$1 - - case $key in - --non-interactive) - INTERACTIVE= - ;; - -i|--image) - DOCKER_IMAGENAME=$2 - shift - ;; - -d|--dockerfile) - DOCKERFILE=$2 - shift - ;; - -h|-?|--help) - echo "Usage: $0 [-d|--dockerfile ] [-i|--image ] " - echo "" - echo "Options:" - echo " The path to the Dockerfile to use to create the build container" - echo " The name of an existing Dockerfile folder under eng/docker to use as the Dockerfile" - echo " The command to run once inside the container (repo root is mapped to DOCKER_HOST_SHARE_DIR; defaults to nothing, which runs the default shell)" - exit 0 - ;; - *) - break # the first non-switch we get ends parsing - ;; - esac - - shift -done - -if [ -z "$DOCKERFILE" ]; then - if [ -z "$DOCKER_IMAGENAME" ]; then - if [ "$(uname)" == "Darwin" ]; then - echo "Defaulting to 'ubuntu' image for Darwin" - export DOCKERFILE=eng/docker/ubuntu - elif [ "$(cat /etc/*-release | grep -cim1 ubuntu)" -eq 1 ]; then - echo "Detected current OS as Ubuntu, determining ubuntu version to use..." - if [ "$(cat /etc/*-release | grep -cim1 16.04)" -eq 1 ]; then - echo "using 'ubuntu.16.04' image" - export DOCKERFILE=eng/docker/ubuntu.16.04 - else - echo "using 'ubuntu' image" - export DOCKERFILE=eng/docker/ubuntu - fi - elif [ "$(cat /etc/*-release | grep -cim1 centos)" -eq 1 ]; then - echo "Detected current OS as CentOS, using 'centos' image" - export DOCKERFILE=eng/docker/centos - elif [ "$(cat /etc/*-release | grep -cim1 rhel)" -eq 1 ]; then - echo "Detected current OS as rhel, using 'rhel' image" - export DOCKERFILE=eng/docker/rhel - elif [ "$(cat /etc/*-release | grep -cim1 debian)" -eq 1 ]; then - echo "Detected current OS as Debian, using 'debian' image" - export DOCKERFILE=eng/docker/debian - elif [ "$(cat /etc/*-release | grep -cim1 fedora)" -eq 1 ]; then - echo "Detected current OS as Fedora, determining fedora version to use..." - if [ "$(cat /etc/*-release | grep -cim1 23)" -eq 1 ]; then - echo "using 'fedora.23' image" - export DOCKERFILE=eng/docker/fedora.23 - fi - else - echo "Unknown Linux Distro. Using 'ubuntu' image" - export DOCKERFILE=eng/docker/ubuntu - fi - else - echo "Using requested image: $DOCKER_IMAGENAME" - export DOCKERFILE="eng/docker/$DOCKER_IMAGENAME" - fi -fi - -[ -z "$DOTNET_BUILD_CONTAINER_TAG" ] && DOTNET_BUILD_CONTAINER_TAG="dotnetcli-build" -[ -z "$DOTNET_BUILD_CONTAINER_NAME" ] && DOTNET_BUILD_CONTAINER_NAME="dotnetcli-build-container" -[ -z "$DOCKER_HOST_SHARE_DIR" ] && DOCKER_HOST_SHARE_DIR=$(pwd) - -# Make container names CI-specific if we're running in CI -# Jenkins -[ ! -z "$BUILD_TAG" ] && DOTNET_BUILD_CONTAINER_NAME="$BUILD_TAG" -# VSO -[ ! -z "$BUILD_BUILDID" ] && DOTNET_BUILD_CONTAINER_NAME="$BUILD_BUILDID" - -function retry { - local max_count=$1 - shift - - local count=0 - until "$@"; do - exit=$? - wait=$((2 ** $count)) - count=$(($count + 1)) - if [[ ${count} -lt ${max_count} ]]; then - echo "Retry $count/$max_count returned $exit, wait $wait seconds..." - sleep ${wait} - else - echo "Retry $count/$max_count returned $exit." - return ${exit} - fi - done - return 0 -} - -# Build the docker container (will be fast if it is already built) -# with retry since docker pull has high failure rate -echo "Building Docker Container using Dockerfile: $DOCKERFILE" -retry 10 docker build --build-arg WORK_DIR=$DOCKER_HOST_SHARE_DIR --build-arg USER_ID=$(id -u) -t $DOTNET_BUILD_CONTAINER_TAG $DOCKERFILE 2>&1 - -# Run the build in the container -echo "Launching build in Docker Container" -echo "Running command: $BUILD_COMMAND" -echo "Using code from: $DOCKER_HOST_SHARE_DIR" -[ -z "$INTERACTIVE" ] || echo "Running Interactive" - -# Note: passwords/keys should not be passed in the environment -docker run $INTERACTIVE -t --rm --sig-proxy=true \ - --name $DOTNET_BUILD_CONTAINER_NAME \ - -v $DOCKER_HOST_SHARE_DIR:$DOCKER_HOST_SHARE_DIR \ - -e CHANNEL \ - -e DOTNET_BUILD_SKIP_CROSSGEN \ - -e PUBLISH_TO_AZURE_BLOB \ - -e NUGET_FEED_URL \ - -e NUGET_API_KEY \ - -e ARTIFACT_STORAGE_ACCOUNT \ - -e ARTIFACT_STORAGE_CONTAINER \ - -e CHECKSUM_STORAGE_ACCOUNT \ - -e CHECKSUM_STORAGE_CONTAINER \ - -e BLOBFEED_STORAGE_CONTAINER \ - -e CLIBUILD_SKIP_TESTS \ - -e COMMITCOUNT \ - -e DROPSUFFIX \ - -e RELEASESUFFIX \ - -e COREFXAZURECONTAINER \ - -e AZUREACCOUNTNAME \ - -e RELEASETOOLSGITURL \ - -e CORESETUPBLOBROOTURL \ - -e PB_ASSETROOTURL \ - -e PB_PACKAGEVERSIONPROPSURL \ - -e PB_PUBLISHBLOBFEEDURL \ - -e _PUBLISHBLOBFEEDURL \ - -e _ASSETROOTURL \ - -e _PACKAGEVERSIONPROPSURL \ - -e EXTERNALRESTORESOURCES \ - -e BUILD_REPOSITORY_URI \ - -e BUILD_REPOSITORY_NAME \ - -e BUILD_SOURCEBRANCH \ - -e BUILD_BUILDNUMBER \ - -e BUILD_BUILDID \ - -e BUILD_SOURCEVERSION \ - -e SYSTEM_TEAMPROJECT \ - -e POSTBUILDSIGN \ - -e SYSTEM_DEFINITIONID \ - -e SYSTEM_TEAMFOUNDATIONCOLLECTIONURI \ - -e AGENT_JOBNAME \ - -e AGENT_OS \ - -e VSS_NUGET_URI_PREFIXES \ - -e VSS_NUGET_ACCESSTOKEN \ - -e DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER=0 \ - -e NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED=true \ - $DOTNET_BUILD_CONTAINER_TAG \ - $BUILD_COMMAND "$@" From 44ea8999291f6bb889be37775c921acf96df8117 Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Tue, 20 Sep 2022 14:04:18 -0700 Subject: [PATCH 2/3] Feedback --- eng/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/build.yml b/eng/build.yml index a91073bc8ebe..946bea474200 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -77,18 +77,18 @@ jobs: enableMicrobuild: true pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Svc-Public + name: NetCore-Public demands: ImageOverride -equals build.windows.10.amd64.vs2019.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Svc-Internal + name: NetCore1ESPool-Internal demands: ImageOverride -equals build.windows.10.amd64.vs2019 ${{ if eq(parameters.agentOs, 'Linux') }}: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Svc-Public + name: NetCore-Public demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Svc-Internal + name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 container: ${{ parameters.container }} ${{ if eq(parameters.agentOs, 'Darwin') }}: From cd99b709a3ab80a695c38f3032e2266a80adac8d Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Tue, 27 Sep 2022 09:58:26 -0700 Subject: [PATCH 3/3] Fix indenting and HostOSName for Linux Musl --- .vsts-ci.yml | 2 +- eng/build.yml | 81 ++++++++++++++++++++++++++------------------------- 2 files changed, 43 insertions(+), 40 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 10fc7ccf5247..d0adb50075cb 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -226,7 +226,7 @@ stages: buildArchitecture: x64 runtimeIdentifier: 'linux-musl-x64' # Pass in HostOSName when running on alpine - additionalBuildParameters: '/p:OSName="linux-musl"' + additionalBuildParameters: '/p:HostOSName="linux-musl"' linuxPortable: false runTests: false - template: /eng/build.yml diff --git a/eng/build.yml b/eng/build.yml index 946bea474200..916e23617d5a 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -105,46 +105,49 @@ jobs: # Test parameters variables: - - _PgoInstrument: '' - - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - - _PackArg: '-pack' - - ${{ if parameters.runTests }}: - - _TestArg: '-test' - - ${{ else }}: - - _TestArg: '' - - ${{ if ne(parameters.agentOs, 'Windows_NT') }}: - - _PackArg: '--pack' - - ${{ if parameters.runTests }}: - - _TestArg: '--test' - - ${{ else }}: - - _TestArg: '' - - - ${{ if parameters.pgoInstrument }}: - - _PgoInstrument: '/p:PgoInstrument=true' - - _PackArg: '' - - - ${{ if parameters.linuxPortable }}: - - _LinuxPortable: '--linux-portable' - - - ${{ if ne(parameters.runtimeIdentifier, '') }}: - - _RuntimeIdentifier: '--runtime-id ${{ parameters.runtimeIdentifier }}' + - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: + - _PackArg: '-pack' + - ${{ if parameters.runTests }}: + - _TestArg: '-test' + - ${{ else }}: + - _TestArg: '' + - ${{ if ne(parameters.agentOs, 'Windows_NT') }}: + - _PackArg: '--pack' + - ${{ if parameters.runTests }}: + - _TestArg: '--test' - ${{ else }}: - - _RuntimeIdentifier: '' - - - _AgentOSName: ${{ parameters.agentOs }} - - _TeamName: Roslyn-Project-System - - _SignType: test - - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)' - - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-HelixApi-Access - - _PushToVSFeed: true - - _SignType: real - - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - /p:DotNetSignType=$(_SignType) - /p:TeamName=$(_TeamName) - /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) - $(_PgoInstrument) + - _TestArg: '' + + - ${{ if parameters.pgoInstrument }}: + - _PgoInstrument: '/p:PgoInstrument=true' + - _PackArg: '' + - ${{ else }}: + - _PgoInstrument: '' + + - ${{ if parameters.linuxPortable }}: + - _LinuxPortable: '--linux-portable' + - ${{ else }}: + - _LinuxPortable: '' + + - ${{ if ne(parameters.runtimeIdentifier, '') }}: + - _RuntimeIdentifier: '--runtime-id ${{ parameters.runtimeIdentifier }}' + - ${{ else }}: + - _RuntimeIdentifier: '' + + - _AgentOSName: ${{ parameters.agentOs }} + - _TeamName: Roslyn-Project-System + - _SignType: test + - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)' + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + - _PushToVSFeed: true + - _SignType: real + - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + /p:DotNetSignType=$(_SignType) + /p:TeamName=$(_TeamName) + /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) + $(_PgoInstrument) steps: - checkout: self