From c33f7d6ab18a6defe782775452da7e32cd5f67e1 Mon Sep 17 00:00:00 2001 From: Juan Sebastian Hoyos Ayala Date: Thu, 7 Mar 2024 19:36:12 -0800 Subject: [PATCH 1/6] Migrate to 1ES templates --- diagnostics.yml | 669 +++++++----------- eng/pipelines/build.yml | 523 +++++++------- eng/pipelines/global-variables.yml | 66 ++ eng/pipelines/pipeline-resources.yml | 49 +- eng/pipelines/prepare-release.yml | 30 +- .../publish-pipeline-artifact-shim.yml | 25 + eng/pipelines/template1es.yml | 31 + eng/pipelines/templateDispatch.yml | 13 + eng/pipelines/templatePublic.yml | 20 + 9 files changed, 741 insertions(+), 685 deletions(-) create mode 100644 eng/pipelines/global-variables.yml create mode 100644 eng/pipelines/publish-pipeline-artifact-shim.yml create mode 100644 eng/pipelines/template1es.yml create mode 100644 eng/pipelines/templateDispatch.yml create mode 100644 eng/pipelines/templatePublic.yml diff --git a/diagnostics.yml b/diagnostics.yml index a1ea543449..11372e39e5 100644 --- a/diagnostics.yml +++ b/diagnostics.yml @@ -6,7 +6,6 @@ parameters: values: - default - custom - - msrc-feed - dotnetclimsrc-feed - name: runtimeFeedToken displayName: Base 64 SAS Token for runtime installation @@ -15,7 +14,6 @@ parameters: values: - default - custom - - msrc-feed-sas-token-base64 - dotnetclimsrc-sas-token-base64 trigger: none @@ -33,50 +31,34 @@ pr: - LICENSE.TXT variables: - - name: _TeamName - value: DotNetCore - - name: _InternalBuildArgs - value: '' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: +- template : /eng/pipelines/global-variables.yml + parameters: + runtimeFeed: ${{ parameters.runtimeFeed }} + runtimeFeedToken: ${{ parameters.runtimeFeedToken }} +- name: _InternalBuildArgs + value: '' +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + # DotNet-Diagnostics-SDL-Params provides Tsa* variables for SDL checks. - name: _SignType value: real - # DotNet-Diagnostics-SDL-Params provides Tsa* variables for SDL checks. - - group: DotNet-Diagnostics-SDL-Params - name: _InternalBuildArgs value: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) /p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - - group: DotNet-MSRC-Storage - # Custom feed and token - - ${{ if eq(parameters.runtimeFeed, 'custom') }}: - - name: RuntimeFeedUrl - value: $(DotnetRuntimeDownloadFeed) - - ${{ if eq(parameters.runtimeFeedToken, 'custom') }}: - - name: RuntimeFeedBase64SasToken - value: $(DotnetRuntimeDownloadBase64SasToken) - # MSRC dotnet feed. Usually on orchestrated 2.1 releases. - - ${{ if eq(parameters.runtimeFeed, 'msrc-feed') }}: - - name: RuntimeFeedUrl - value: https://dotnetfeedmsrc.blob.core.windows.net - - ${{ if eq(parameters.runtimeFeedToken, 'msrc-feed-sas-token-base64') }}: - - name: RuntimeFeedBase64SasToken - value: $(dotnetfeedmsrc-read-sas-token-base64) - # dotnetclimsrc contains 3.1+ - - ${{ if eq(parameters.runtimeFeed, 'dotnetclimsrc-feed') }}: - - name: RuntimeFeedUrl - value: https://dotnetclimsrc.blob.core.windows.net/dotnet - - ${{ if eq(parameters.runtimeFeedToken, 'dotnetclimsrc-sas-token-base64') }}: - - name: RuntimeFeedBase64SasToken - value: $(dotnetclimsrc-read-sas-token-base64) extends: - template: /eng/pipelines/pipeline-resources.yml + template: /eng/pipelines/pipeline-resources.yml parameters: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + isOfficialBuild: false + ${{ else }}: + isOfficialBuild: true stages: - - stage: build - displayName: Build and Test Diagnostics - jobs: + - stage: build + displayName: Build and Test Diagnostics + jobs: ############################ # # @@ -84,408 +66,303 @@ extends: # # ############################ - - template: /eng/common/templates/job/source-build.yml - parameters: - platform: - name: Complete - buildScript: ./eng/common/build.sh + - template: ${{ variables.sourceBuildTemplate }} + parameters: + platform: + name: Complete + buildScript: ./eng/common/build.sh - ############################ - # # - # Build legs # - # # - ############################ + ############################ + # # + # Build legs # + # # + ############################ + + - template: /eng/pipelines/build.yml + parameters: + jobTemplate: ${{ variables.jobTemplate }} + name: Windows + osGroup: Windows_NT + buildConfigs: + - configuration: Debug + architecture: x64 + - configuration: Release + architecture: x64 + artifactUploadPath: bin + - configuration: Release + architecture: x86 + artifactUploadPath: bin/Windows_NT.x86.Release + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - configuration: Release + architecture: arm + artifactUploadPath: bin/Windows_NT.arm.Release + - configuration: Release + architecture: arm64 + artifactUploadPath: bin/Windows_NT.arm64.Release + + - template: /eng/pipelines/build.yml + parameters: + jobTemplate: ${{ variables.jobTemplate }} + osGroup: Linux + nativeBuildContainer: linux_x64 + buildOnly: true + buildConfigs: + - configuration: Release + architecture: x64 + artifactUploadPath: bin/linux.x64.Release + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - configuration: Debug + architecture: x64 + artifactUploadPath: bin/linux.x64.Debug + - template: /eng/pipelines/build.yml + parameters: + jobTemplate: ${{ variables.jobTemplate }} + name: Linux_musl + osGroup: Linux + osSuffix: -musl + nativeBuildContainer: linux_musl_x64 + buildOnly: true + buildConfigs: + - configuration: Release + architecture: x64 + artifactUploadPath: bin/linux.x64.Release + artifactTargetPath: bin/linux-musl.x64.Release + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - configuration: Debug + architecture: x64 + artifactUploadPath: bin/linux.x64.Debug + artifactTargetPath: bin/linux-musl.x64.Debug + + - template: /eng/pipelines/build.yml + parameters: + jobTemplate: ${{ variables.jobTemplate }} + osGroup: MacOS + buildConfigs: + - configuration: Release + architecture: x64 + artifactUploadPath: bin/osx.x64.Release + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - configuration: Debug + architecture: x64 + + - template: /eng/pipelines/build.yml + parameters: + jobTemplate: ${{ variables.jobTemplate }} + osGroup: MacOS + crossBuild: true + buildOnly: true + buildConfigs: + - configuration: Release + architecture: arm64 + artifactUploadPath: bin/osx.arm64.Release + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - configuration: Debug + architecture: arm64 + + - ${{ if ne(variables['System.TeamProject'], 'public') }}: - template: /eng/pipelines/build.yml parameters: - name: Windows - osGroup: Windows_NT - strategy: - matrix: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - _PublishArtifacts: bin - Build_Release_x86: - _BuildConfig: Release - _BuildArch: x86 - _PublishArtifacts: bin/Windows_NT.x86.Release - ${{ if ne(variables['System.TeamProject'], 'public') }}: - Build_Release_arm: - _BuildConfig: Release - _BuildArch: arm - _PublishArtifacts: bin/Windows_NT.arm.Release - Build_Release_arm64: - _BuildConfig: Release - _BuildArch: arm64 - _PublishArtifacts: bin/Windows_NT.arm64.Release + jobTemplate: ${{ variables.jobTemplate }} + osGroup: Linux + nativeBuildContainer: linux_arm + crossBuild: true + buildOnly: true + buildConfigs: + - configuration: Release + architecture: arm + artifactUploadPath: bin/linux.arm.Release - template: /eng/pipelines/build.yml parameters: - name: Linux_x64 + jobTemplate: ${{ variables.jobTemplate }} osGroup: Linux - nativeBuildContainer: linux_x64 + nativeBuildContainer: linux_arm64 + crossBuild: true buildOnly: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - _PublishArtifacts: bin/linux.x64.Release - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 - _PublishArtifacts: bin/linux.x64.Debug + buildConfigs: + - configuration: Release + architecture: arm64 + artifactUploadPath: bin/linux.arm64.Release - template: /eng/pipelines/build.yml parameters: + jobTemplate: ${{ variables.jobTemplate }} name: Linux_musl osGroup: Linux osSuffix: -musl - nativeBuildContainer: linux_musl_x64 + nativeBuildContainer: linux_musl_arm + crossBuild: true buildOnly: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - _PublishArtifacts: bin/linux.x64.Release - _ArtifactsTargetPath: bin/linux-musl.x64.Release - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 - _PublishArtifacts: bin/linux.x64.Debug - _ArtifactsTargetPath: bin/linux-musl.x64.Debug - - - template: /eng/pipelines/build.yml - parameters: - name: MacOS - osGroup: MacOS - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - _PublishArtifacts: bin/osx.x64.Release - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + buildConfigs: + - configuration: Release + architecture: arm + artifactUploadPath: bin/linux.arm.Release + artifactTargetPath: bin/linux-musl.arm.Release - template: /eng/pipelines/build.yml parameters: - name: MacOS_arm64 - osGroup: MacOS_cross + jobTemplate: ${{ variables.jobTemplate }} + name: Linux_musl + osGroup: Linux + osSuffix: -musl + nativeBuildContainer: linux_musl_arm64 crossBuild: true buildOnly: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 - _PublishArtifacts: bin/osx.arm64.Release - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: arm64 + buildConfigs: + - configuration: Release + architecture: arm64 + artifactUploadPath: bin/linux.arm64.Release + artifactTargetPath: bin/linux-musl.arm64.Release - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - template: /eng/pipelines/build.yml - parameters: - name: Linux_arm - osGroup: Linux - nativeBuildContainer: linux_arm - crossBuild: true - buildOnly: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm - _PublishArtifacts: bin/linux.arm.Release + ############################ + # # + # Test only legs # + # # + ############################ - - template: /eng/pipelines/build.yml - parameters: - name: Linux_arm64 - osGroup: Linux - nativeBuildContainer: linux_arm64 - crossBuild: true - buildOnly: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 - _PublishArtifacts: bin/linux.arm64.Release + - template: /eng/pipelines/build.yml + parameters: + jobTemplate: ${{ variables.jobTemplate }} + name: Ubuntu_20_04 + osGroup: Linux + container: test_ubuntu_20_04 + dependsOn: Linux + testOnly: true + buildConfigs: + - configuration: Release + architecture: x64 + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - configuration: Debug + architecture: x64 - - template: /eng/pipelines/build.yml - parameters: - name: Linux_musl_arm - osGroup: Linux - osSuffix: -musl - nativeBuildContainer: linux_musl_arm - crossBuild: true - buildOnly: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm - _PublishArtifacts: bin/linux.arm.Release - _ArtifactsTargetPath: bin/linux-musl.arm.Release - - - template: /eng/pipelines/build.yml - parameters: - name: Linux_musl_arm64 - osGroup: Linux - osSuffix: -musl - nativeBuildContainer: linux_musl_arm64 - crossBuild: true - buildOnly: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: arm64 - _PublishArtifacts: bin/linux.arm64.Release - _ArtifactsTargetPath: bin/linux-musl.arm64.Release - - ############################ - # # - # Test only legs # - # # - ############################ + - template: /eng/pipelines/build.yml + parameters: + jobTemplate: ${{ variables.jobTemplate }} + name: Alpine3_13 + osGroup: Linux + osSuffix: -musl + container: test_linux_musl_x64 + dependsOn: Linux_musl + testOnly: true + buildConfigs: + - configuration: Release + architecture: x64 + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - configuration: Debug + architecture: x64 + - ${{ if ne(variables['System.TeamProject'], 'public') }}: - template: /eng/pipelines/build.yml parameters: - name: Ubuntu_20_04 + jobTemplate: ${{ variables.jobTemplate }} + name: Debian_Bullseye osGroup: Linux - container: test_ubuntu_20_04 - dependsOn: Linux_x64 + container: test_debian_11_amd64 + dependsOn: Linux testOnly: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + buildConfigs: + - configuration: Release + architecture: x64 + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - configuration: Debug + architecture: x64 - template: /eng/pipelines/build.yml parameters: - name: Alpine3_13 + jobTemplate: ${{ variables.jobTemplate }} + name: Fedora_36 osGroup: Linux - osSuffix: -musl - container: test_linux_musl_x64 - dependsOn: Linux_musl + container: test_fedora_36 + dependsOn: Linux testOnly: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + buildConfigs: + - configuration: Release + architecture: x64 + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - configuration: Debug + architecture: x64 - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - template: /eng/pipelines/build.yml - parameters: - name: Debian_Bullseye - osGroup: Linux - container: test_debian_11_amd64 - dependsOn: Linux_x64 - testOnly: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - stage: package + displayName: Package, Sign, and Generate BAR Manifests + jobs: + - template: /eng/common/templates-official/job/job.yml + parameters: + name: Sign_Package_Publish + displayName: Packaging + condition: succeeded() + enablePublishUsingPipelines: true + enableMicrobuild: true + templateContext: + outputs: + - output: pipelineArtifact + artifact: BundledTools + path: '$(Build.SourcesDirectory)/artifacts/bundledtools' + displayName: 'Publish Bundled Tools' + condition: succeeded() + - output: pipelineArtifact + artifact: Logs_Packaging_Signing + path: '$(Build.SourcesDirectory)/artifacts/log' + displayName: 'Publish Signing and Packaging Logs' + condition: always() + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download release builds' + inputs: + patterns: | + Build_Linux_*_Release/bin/** + Build_Linux_musl_*_Release/bin/** + Build_Windows_*_Release/bin/** + Build_MacOS_*_Release/bin/** + downloadPath: '$(Build.ArtifactStagingDirectory)/__download__' + checkDownloadedFiles: true + cleanDestinationFolder: true - - template: /eng/pipelines/build.yml - parameters: - name: Fedora_36 - osGroup: Linux - container: test_fedora_36 - dependsOn: Linux_x64 - testOnly: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + - powershell: | + cd $(Build.ArtifactStagingDirectory)/__download__ - #- template: /eng/pipelines/build.yml - # parameters: - # name: OpenSuse_15_2 - # osGroup: Linux - # container: test_opensuse_15_2 - # dependsOn: Linux_x64 - # testOnly: true - # strategy: - # matrix: - # Build_Release: - # _BuildConfig: Release - # _BuildArch: x64 - # ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - # Build_Debug: - # _BuildConfig: Debug - # _BuildArch: x64 + $dirs = Get-ChildItem -Directory + if ($dirs.Count -ne 12) { + throw "Expected 12 release platforms to be present. 3 Linux glibc, 3 Linux musl, 4 Windows, and 2 macOS." + } - #- template: /eng/pipelines/build.yml - # parameters: - # name: Ubuntu_18_04 - # osGroup: Linux - # container: test_ubuntu_18_04 - # dependsOn: Linux_x64 - # testOnly: true - # strategy: - # matrix: - # Build_Release: - # _BuildConfig: Release - # _BuildArch: x64 - # ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - # Build_Debug: - # _BuildConfig: Debug - # _BuildArch: x64 + $targetDir = "$(Build.SourcesDirectory)/artifacts/bin" + if (!(Test-Path $targetDir)) { + New-Item $targetDir -ItemType Directory + } - - template: /eng/pipelines/build.yml - parameters: - name: Ubuntu_22_04 - osGroup: Linux - container: test_ubuntu_22_04 - dependsOn: Linux_x64 - testOnly: true - strategy: - matrix: - Build_Release: - _BuildConfig: Release - _BuildArch: x64 - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - Build_Debug: - _BuildConfig: Debug - _BuildArch: x64 + $dirs | ForEach-Object { + $sourceDir = "$($_.FullName)/bin" + Move-Item -Path $sourceDir/* -Destination $targetDir -Verbose + } + displayName: 'Setup layout for packaging' - # Download, sign, package and publish - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - template: /eng/common/templates/job/job.yml - parameters: - name: Sign_Package_Publish - displayName: Sign, Package, and Generate BAR Manifests - dependsOn: - - Windows - - MacOS - - MacOS_arm64 - - Linux_x64 - - Linux_musl - - Linux_arm - - Linux_arm64 - - Linux_musl_arm - - Linux_musl_arm64 + # Create nuget packages, sign binaries and publish to blob feed + - script: $(Build.SourcesDirectory)\eng\ci-prepare-artifacts.cmd $(_InternalBuildArgs) + displayName: Package, Sign, and Publish + continueOnError: false condition: succeeded() - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2022.amd64 - enablePublishUsingPipelines: true - enableMicrobuild: true - artifacts: - publish: - logs: - name: Logs_Packaging_Signing - steps: - - task: DownloadBuildArtifacts@0 - displayName: 'Download release builds' - inputs: - downloadPath: '$(Build.ArtifactStagingDirectory)/__download__' - artifactName: Build_Release - checkDownloadedFiles: true - - task: CopyFiles@2 - displayName: 'Binplace Product' - inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/__download__/Build_Release - targetFolder: '$(Build.SourcesDirectory)/artifacts/' - # Windows x64 download. Everything under "bin" is published for the Windows x64 build. - # Create nuget packages, sign binaries and publish to blob feed - - script: $(Build.SourcesDirectory)\eng\ci-prepare-artifacts.cmd $(_InternalBuildArgs) - displayName: Package, Sign, and Publish - continueOnError: false - condition: succeeded() - - # Publish package and log build artifacts - - task: PublishBuildArtifacts@1 - displayName: Publish Package Artifacts - inputs: - publishLocation: Container - pathtoPublish: '$(Build.SourcesDirectory)/artifacts/packages' - artifactName: Packages - continueOnError: true - condition: always() - - - task: PublishBuildArtifacts@1 - displayName: Publish Bundled Tools - inputs: - publishLocation: Container - pathtoPublish: '$(Build.SourcesDirectory)/artifacts/bundledtools' - artifactName: BundledTools - continueOnError: true - condition: always() - - - template: /eng/common/templates/job/publish-build-assets.yml - parameters: - configuration: Release - dependsOn: Sign_Package_Publish - publishUsingPipelines: true - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2022.amd64 - - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/common/templates/post-build/post-build.yml + - template: /eng/common/templates-official/job/publish-build-assets.yml parameters: - # This is to enable SDL runs part of Post-Build Validation Stage. - # as well as NuGet, SourceLink, and signing validation. - # The variables get imported from group dotnet-diagnostics-sdl-params - publishingInfraVersion: 3 - enableSourceLinkValidation: true - enableSigningValidation: false - enableSymbolValidation: false - enableNugetValidation: true - symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=false' - publishInstallersAndChecksums: true - SDLValidationParameters: - enable: true - continueOnError: true - params: ' -SourceToolsList @("policheck","credscan") - -ArtifactToolsList @("binskim") - -BinskimAdditionalRunConfigParams @("IgnorePdbLoadError < True","Recurse < True") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "diagnostics" - -TsaCodebaseName "diagnostics" - -TsaPublish $True' - artifactNames: - - 'Packages' + configuration: Release + dependsOn: Sign_Package_Publish + publishUsingPipelines: true + + - template: /eng/common/templates-official/post-build/post-build.yml@self + parameters: + # This is to enable SDL runs part of Post-Build Validation Stage. + # as well as NuGet, SourceLink, and signing validation. + # The variables get imported from group dotnet-diagnostics-sdl-params + validateDependsOn: package + publishingInfraVersion: 3 + enableSourceLinkValidation: true + enableSigningValidation: false + enableSymbolValidation: false + enableNugetValidation: true + symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=false' + publishInstallersAndChecksums: true - # This sets up the bits to do a Release. - - template: /eng/pipelines/prepare-release.yml + # This sets up the bits to do a Release. + - template: /eng/pipelines/prepare-release.yml diff --git a/eng/pipelines/build.yml b/eng/pipelines/build.yml index e9756625f3..beea048178 100644 --- a/eng/pipelines/build.yml +++ b/eng/pipelines/build.yml @@ -1,272 +1,299 @@ parameters: # Job name - name: '' - # Agent OS (Windows_NT, Linux, MacOS, FreeBSD) - osGroup: Windows_NT - # Optional: OS suffix like -musl - osSuffix: '' +- name: name + type: string + default: '' + +- name: jobTemplate + type: string + default: /eng/common/templates-official/job/job.yml@self + values: + - /eng/common/templates-official/job/job.yml@self + - /eng/common/templates/job/job.yml@self + +- name: osGroup + type: string + default: Windows_NT + values: + - Windows_NT + - Linux + - MacOS + - FreeBSD + +- name: osSuffix + type: string + default: '' + # Additional variables - variables: {} - # Build strategy - matrix - strategy: '' +- name: variables + type: object + default: {} + # Optional: Job timeout - timeoutInMinutes: 180 + +- name: timeoutInMinutes + type: number + default: 180 + + # Optional: Build container to use +- name: templateContext + type: object + default: {} + +# Build configs. An object with the following properties: configuration, architecture. +# Optionally it can also contain artifactUploadPath, artifactTargetPath +- name: buildConfigs + type: object + default: {} + # Optional: native build container resource name - nativeBuildContainer: '' +- name: nativeBuildContainer + type: string + default: '' + # Optional: container resource name - container: '' +- name: container + type: string + default: '' + # Optional: build only job if true - buildOnly: false +- name: buildOnly + type: boolean + default: false + # Optional: test only job if true - testOnly: false - # Optional: architecture cross build if true - crossBuild: false - # Depends on - dependsOn: '' - isCodeQLRun: false +- name: testOnly + type: boolean + default: false + +# Optional: architecture cross build if true +- name: crossBuild + type: boolean + default: false + +- name: dependsOn + type: string + default: '' + +- name: isCodeQLRun + type: boolean + default: false jobs: -- template: /eng/common/templates/job/job.yml - parameters: - name: ${{ parameters.name }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - enableMicrobuild: true - enableTelemetry: true - helixRepo: dotnet/diagnostics - runAsPublic: ${{ parameters.isCodeQLRun }} - - pool: - # Public Linux Build Pool - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(variables['System.TeamProject'], 'public')) }}: - name: NetCore-Public - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - - # Official Build Linux Pool - ${{ if and(eq(parameters.osGroup, 'Linux'), ne(variables['System.TeamProject'], 'public')) }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - - # FreeBSD builds only in the internal project - ${{ if and(eq(parameters.osGroup, 'FreeBSD'), ne(variables['System.TeamProject'], 'public')) }}: - name: dnceng-freebsd-internal - - # Build OSX Pool (we don't have on-prem OSX BuildPool) - ${{ if in(parameters.osGroup, 'MacOS', 'MacOS_cross') }}: - vmImage: macOS-latest - - # Official Build Windows Pool - ${{ if and(eq(parameters.osGroup, 'Windows_NT'), ne(variables['System.TeamProject'], 'public')) }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2022.amd64 - - # Public Windows Build Pool - ${{ if and(eq(parameters.osGroup, 'Windows_NT'), eq(variables['System.TeamProject'], 'public')) }}: - name: NetCore-Public - demands: ImageOverride -equals windows.vs2022.amd64.open - - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - workspace: - clean: all - - variables: - - ${{ insert }}: ${{ parameters.variables }} - - _PhaseName : ${{ parameters.name }} - - _HelixType: build/product - - _HelixBuildConfig: $(_BuildConfig) - - _Pipeline_StreamDumpDir: $(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/streams - - - _BuildDisplayName: 'Build / Test' - - _ExtraBuildParams: '' - - _TestArgs: '-test' - - _Cross: '' - - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - _buildScript: $(Build.SourcesDirectory)\build.cmd - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - _buildScript: $(Build.SourcesDirectory)/build.sh - - - ${{ if eq(parameters.testOnly, 'true') }}: - - _TestArgs: '-test -skipnative' - - _BuildDisplayName: Test - - - ${{ if or(eq(parameters.buildOnly, 'true'), eq(parameters.isCodeQLRun, 'true')) }}: - - _TestArgs: '' - - - ${{ if eq(parameters.isCodeQLRun, 'true') }}: - - name: Codeql.Enabled - value: True - - name: Codeql.Cadence - value: 0 - - name: Codeql.TSAEnabled - value: True - - name: Codeql.BuildIdentifier - value: $(System.JobDisplayName) - - name: Codeql.Language - value: csharp,cpp - - # For testing msrc's and service releases. The RuntimeSourceVersion is either "default" or the service release version to test - - _InternalInstallArgs: '' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.isCodeQLRun, 'false')) }}: - - _InternalInstallArgs: - -dotnetruntimeversion '$(DotnetRuntimeVersion)' - -dotnetruntimedownloadversion '$(DotnetRuntimeDownloadVersion)' - -runtimesourcefeed '$(RuntimeFeedUrl)' - -runtimesourcefeedkey '$(RuntimeFeedBase64SasToken)' - - # Only enable publishing in non-public, non PR scenarios. - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - _HelixSource: official/dotnet/arcade/$(Build.SourceBranch) - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - - _HelixSource: pr/dotnet/arcade/$(Build.SourceBranch) - - # This is only required for cross builds. - - ${{ if eq(parameters.crossBuild, true) }}: - - _Cross: -cross - - # If there is a native build container, build managed in the host vm/container and native in the nativeBuildContainer - - ${{ if ne(parameters.nativeBuildContainer, '') }}: - - _ExtraBuildParams: -skipnative - - _BuildDisplayName: 'Build Managed' - - # Only add the cross build option if a combined build/test managed/native build (i.e. MacOS arm64) - - ${{ if eq(parameters.nativeBuildContainer, '') }}: - - _ExtraBuildParams: $(_Cross) - - steps: - - ${{ if eq(parameters.osGroup, 'Linux') }}: +- ${{ each config in parameters.buildConfigs }}: + - template: ${{ parameters.jobTemplate }} + parameters: + name: ${{ coalesce(parameters.name, parameters.osGroup) }}_${{ config.architecture }}_${{ config.configuration }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + enableMicrobuild: true + runAsPublic: ${{ parameters.isCodeQLRun }} + templateContext: ${{ parameters.templateContext }} + + pool: + # Public Linux Build Pool + ${{ if eq(parameters.osGroup, 'Linux') }}: + name: $(BuildPool) + demands: ImageOverride -equals $(LinuxImage) + os: linux + + ${{ elseif eq(parameters.osGroup, 'Windows_NT') }}: + name: $(BuildPool) + demands: ImageOverride -equals $(WindowsImage) + os: windows + + ${{ elseif in(parameters.osGroup, 'MacOS') }}: + name: Azure Pipelines + vmImage: $(macOSImage) + os: macOS + + ${{ elseif and(eq(parameters.osGroup, 'FreeBSD'), ne(variables['System.TeamProject'], 'public')) }}: + name: dnceng-freebsd-internal + os: freebsd + + ${{ else }}: + 'Unable to identify pool for required build': error + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.strategy, '') }}: + 'error, we can no longer support the strategy feature in the new pipeline system. Please remove the strategy from the job template.' + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }}_${{ config.architecture }}_${{ config.configuration }} + + workspace: + clean: all + + variables: + - ${{ insert }}: ${{ parameters.variables }} + + - _PhaseName: ${{ coalesce(parameters.name, parameters.osGroup) }}_${{ config.architecture }}_${{ config.configuration }} + - _Pipeline_StreamDumpDir: $(Build.SourcesDirectory)/artifacts/tmp/${{ config.configuration }}/streams + + - _ExtraBuildParams: '' + - _TestArgs: '-test' + - _Cross: '' + + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - _buildScript: $(Build.SourcesDirectory)\build.cmd + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + - _buildScript: $(Build.SourcesDirectory)/build.sh + + - ${{ if and(eq(parameters.testOnly, 'true'), eq(parameters.buildOnly, 'true')) }}: + 'error, testOnly and buildOnly cannot be true at the same time': error + - ${{ if eq(parameters.testOnly, 'true') }}: - - task: DownloadBuildArtifacts@0 + - _TestArgs: '-test -skipnative' + + - ${{ if or(eq(parameters.buildOnly, 'true'), eq(parameters.isCodeQLRun, 'true')) }}: + - _TestArgs: '' + + # For testing msrc's and service releases. The RuntimeSourceVersion is either "default" or the service release version to test + - _InternalInstallArgs: '' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.isCodeQLRun, 'false')) }}: + - _InternalInstallArgs: + -dotnetruntimeversion '$(DotnetRuntimeVersion)' + -dotnetruntimedownloadversion '$(DotnetRuntimeDownloadVersion)' + -runtimesourcefeed '$(RuntimeFeedUrl)' + -runtimesourcefeedkey '$(RuntimeFeedBase64SasToken)' + + # This is only required for cross builds. + - ${{ if eq(parameters.crossBuild, true) }}: + - _Cross: -cross + + # If there is a native build container, build managed in the host vm/container and native in the nativeBuildContainer + - ${{ if ne(parameters.nativeBuildContainer, '') }}: + - _ExtraBuildParams: -skipnative + + # TODO: remove this once we move to crossroot builds for all containers. We are currently blocked + # on mariner cross containers not supporting glibc versions supported in 6.0. + # Only add the cross build option if a combined build/test managed/native build (i.e. MacOS arm64) + - ${{ if eq(parameters.nativeBuildContainer, '') }}: + - _ExtraBuildParams: $(_Cross) + + steps: + - ${{ if eq(parameters.testOnly, true) }}: + - ${{ if ne(parameters.osGroup, 'Linux') }}: + - 'error, testOnly is only supported on Linux': error + - task: DownloadPipelineArtifact@2 displayName: 'Download Build Artifacts' inputs: - downloadPath: '$(Build.ArtifactStagingDirectory)/__download__' - downloadType: specific + targetPath: '$(Build.ArtifactStagingDirectory)/__download__' itemPattern: | - Build_$(_BuildConfig)/bin/linux${{ parameters.osSuffix }}.$(_BuildArch).$(_BuildConfig)/** + Build_${{ parameters.dependsOn }}_${{ config.architecture }}_${{ config.configuration }}/bin/** checkDownloadedFiles: true - task: CopyFiles@2 - displayName: 'Binplace Product' + displayName: 'Binplace Downloaded Product' inputs: - sourceFolder: $(Build.ArtifactStagingDirectory)/__download__/Build_$(_BuildConfig)/bin/linux${{ parameters.osSuffix }}.$(_BuildArch).$(_BuildConfig) - targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/linux.$(_BuildArch).$(_BuildConfig)' - - - ${{ if eq(parameters.isCodeQLRun, 'true') }}: - - task: CodeQL3000Init@0 - displayName: CodeQL Initialize - - - script: $(_buildScript) - -ci - -configuration $(_BuildConfig) - -architecture $(_BuildArch) - $(_ExtraBuildParams) - $(_TestArgs) - /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - $(_InternalInstallArgs) - displayName: $(_BuildDisplayName) - condition: succeeded() - - - ${{ if ne(parameters.nativeBuildContainer, '') }}: + sourceFolder: $(Build.ArtifactStagingDirectory)/__download__/Build_${{ parameters.dependsOn }}_${{ config.architecture }}_${{ config.configuration }}/bin/linux${{ parameters.osSuffix }}.${{ config.architecture }}.${{ config.configuration }} + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/linux.${{ config.architecture }}.${{ config.configuration }}' + + - ${{ if eq(parameters.isCodeQLRun, 'true') }}: + - task: CodeQL3000Init@0 + displayName: CodeQL Initialize + - script: $(_buildScript) -ci - -configuration $(_BuildConfig) - -architecture $(_BuildArch) - -skipmanaged - $(_Cross) + -configuration ${{ config.configuration }} + -architecture ${{ config.architecture }} + $(_ExtraBuildParams) + $(_TestArgs) /p:OfficialBuildId=$(BUILD.BUILDNUMBER) $(_InternalInstallArgs) - displayName: Build Native - target: ${{ parameters.nativeBuildContainer }} - - - ${{ if eq(parameters.isCodeQLRun, 'true') }}: - - task: CodeQL3000Finalize@0 - displayName: CodeQL Finalize - - - task: CopyFiles@2 - displayName: Gather binaries for publish to special artifacts path - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/$(_PublishArtifacts)' - Contents: '**' - TargetFolder: $(Build.ArtifactStagingDirectory)/artifacts/$(_ArtifactsTargetPath) - condition: and(ne(variables['_PublishArtifacts'], ''), ne(variables['_ArtifactsTargetPath'], '')) - - - task: CopyFiles@2 - displayName: Gather binaries for publish to artifacts - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/$(_PublishArtifacts)' - Contents: '**' - TargetFolder: $(Build.ArtifactStagingDirectory)/artifacts/$(_PublishArtifacts) - condition: and(ne(variables['_PublishArtifacts'], ''), eq(variables['_ArtifactsTargetPath'], '')) - - - task: PublishBuildArtifacts@1 - displayName: Publish Build Artifacts - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - artifactName: Build_$(_BuildConfig) - condition: ne(variables['_PublishArtifacts'], '') - - - task: PublishBuildArtifacts@1 - displayName: Publish Artifacts on failure - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/bin' - PublishLocation: Container - ArtifactName: Artifacts_$(_PhaseName)_$(_BuildArch)_$(_BuildConfig) - continueOnError: true - condition: failed() - - - task: PublishBuildArtifacts@1 - displayName: Publish Dump Artifacts on failure - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/dumps' - PublishLocation: Container - ArtifactName: Dumps_$(_PhaseName)_$(_BuildArch)_$(_BuildConfig) - continueOnError: true - condition: failed() - - - task: PublishBuildArtifacts@1 - displayName: Publish Stream Artifacts on failure - inputs: - PathtoPublish: $(_Pipeline_StreamDumpDir) - PublishLocation: Container - ArtifactName: Streams_$(_PhaseName)_$(_BuildArch)_$(_BuildConfig) - continueOnError: true - condition: failed() - - - task: CopyFiles@2 - displayName: Gather Logs - inputs: - sourceFolder: '$(Build.SourcesDirectory)/artifacts' - contents: '?(log|TestResults)/**' - targetFolder: '$(Build.StagingDirectory)/BuildLogs' - continueOnError: true - condition: always() - - - task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.StagingDirectory)/BuildLogs' - PublishLocation: Container - ArtifactName: Logs_$(_PhaseName)_$(_BuildArch)_$(_BuildConfig) - continueOnError: true - condition: always() - - - ${{ if and(eq(parameters.buildOnly, 'false'), eq(parameters.isCodeQLRun, 'false')) }}: - # Publish test results to Azure Pipelines - - task: PublishTestResults@2 + ${{ if eq(parameters.testOnly, 'true') }}: + displayName: Test + ${{ elseif eq(parameters.buildOnly, 'true') }}: + displayName: Build + ${{ elseif ne(parameters.nativeBuildContainer, '') }}: + displayName: Build Managed + ${{ else }}: + displayName: Build / Test + condition: succeeded() + + - ${{ if ne(parameters.nativeBuildContainer, '') }}: + - script: $(_buildScript) + -ci + -configuration ${{ config.configuration }} + -architecture ${{ config.architecture }} + -skipmanaged + $(_Cross) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + $(_InternalInstallArgs) + displayName: Build Native + target: ${{ parameters.nativeBuildContainer }} + + - ${{ if eq(parameters.isCodeQLRun, 'true') }}: + - task: CodeQL3000Finalize@0 + displayName: CodeQL Finalize + + - ${{ if ne(config.artifactUploadPath, '') }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/${{ config.artifactUploadPath }}' + Contents: '**' + TargetFolder: $(Build.ArtifactStagingDirectory)/artifacts/${{ coalesce(config.artifactTargetPath, config.artifactUploadPath) }} + + - template: /eng/pipelines/publish-pipeline-artifact-shim.yml@self + parameters: + displayName: Publish Build Artifacts + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: Build_$(_PhaseName) + + - task: CopyFiles@2 + displayName: Gather diagnostic artifacts on failure inputs: - testResultsFormat: xUnit - testResultsFiles: '**/*UnitTests*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults' - failTaskOnFailedTests: true - testRunTitle: 'Tests $(_PhaseName) $(_BuildArch) $(_BuildConfig)' - publishRunAttachments: true - mergeTestResults: true - buildConfiguration: ${{ parameters.name }} + SourceFolder: '$(Build.SourcesDirectory)/artifacts' + TargetFolder: $(Build.ArtifactStagingDirectory)/diagnostics + Contents: | + bin/** + log/** + TestResults/** + tmp/${{ config.configuration }}/dumps/** + tmp/${{ config.configuration }}/streams/** + continueOnError: true + condition: failed() + + - template: /eng/pipelines/publish-pipeline-artifact-shim.yml@self + parameters: + displayName: Publish diagnostic artifacts on failure + inputs: + targetPath: $(Build.ArtifactStagingDirectory)/diagnostics + artifactName: DiagnosticArtifacts_$(_PhaseName) + continueOnError: true + condition: failed() + + - task: CopyFiles@2 + displayName: Gather Logs + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '?(log|TestResults)/**' + targetFolder: '$(Build.StagingDirectory)/BuildLogs' continueOnError: true condition: always() + + - template: /eng/pipelines/publish-pipeline-artifact-shim.yml@self + parameters: + displayName: Publish Logs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: Logs_$(_PhaseName) + continueOnError: true + condition: always() + + - ${{ if and(eq(parameters.buildOnly, 'false'), eq(parameters.isCodeQLRun, 'false')) }}: + # Publish test results to Azure Pipelines + - task: PublishTestResults@2 + inputs: + testResultsFormat: xUnit + testResultsFiles: '**/*UnitTests*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults' + failTaskOnFailedTests: true + testRunTitle: 'Tests $(_PhaseName)' + publishRunAttachments: true + mergeTestResults: true + buildConfiguration: ${{ parameters.name }} + continueOnError: true + condition: always() diff --git a/eng/pipelines/global-variables.yml b/eng/pipelines/global-variables.yml new file mode 100644 index 0000000000..3ea66e870e --- /dev/null +++ b/eng/pipelines/global-variables.yml @@ -0,0 +1,66 @@ +parameters: + runtimeFeed: default + runtimeFeedToken: default + isCodeQLRun: false + +variables: +- name: _TeamName + value: DotNetCore + +# # Internal build pools. +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - group: DotNet-Diagnostics-SDL-Params + - group: DotNet-MSRC-Storage + - template: /eng/common/templates/variables/pool-providers.yml@self + - name: BuildPool + value: $(DncEngInternalBuildPool) + - name: WindowsImage + value: 1es-windows-2022-pt + - name: LinuxImage + value: 1es-ubuntu-2204-pt + - name: macOSImage + value: macOS-latest + - name: sourceBuildTemplate + value: /eng/common/templates-official/job/source-build.yml@self + - name: jobTemplate + value: /eng/common/templates-official/job/job.yml@self + - ${{ if eq(parameters.isCodeQLRun, 'true') }}: + - name: Codeql.Enabled + value: True + - name: Codeql.Cadence + value: 0 + - name: Codeql.TSAEnabled + value: True + - name: Codeql.BuildIdentifier + value: $(System.JobDisplayName) + - name: Codeql.Language + value: csharp,cpp +# Public build settings +- ${{ else }}: + - template: /eng/common/templates/variables/pool-providers.yml@self + - name: BuildPool + value: $(DncEngPublicBuildPool) + - name: sourceBuildTemplate + value: /eng/common/templates/job/source-build.yml@self + - name: jobTemplate + value: /eng/common/templates/job/job.yml@self + - name: WindowsImage + value: windows.vs2022.amd64.open + - name: LinuxImage + value: Build.Ubuntu.2204.Amd64.Open + - name: macOSImage + value: macOS-latest + +# Custom feed and token +- ${{ if eq(parameters.runtimeFeed, 'custom') }}: + - name: RuntimeFeedUrl + value: $(DotnetRuntimeDownloadFeed) +- ${{ if eq(parameters.runtimeFeedToken, 'custom') }}: + - name: RuntimeFeedBase64SasToken + value: $(DotnetRuntimeDownloadBase64SasToken) +- ${{ if eq(parameters.runtimeFeed, 'dotnetclimsrc-feed') }}: + - name: RuntimeFeedUrl + value: https://dotnetclimsrc.blob.core.windows.net/dotnet +- ${{ if eq(parameters.runtimeFeedToken, 'dotnetclimsrc-sas-token-base64') }}: + - name: RuntimeFeedBase64SasToken + value: $(dotnetclimsrc-read-sas-token-base64) diff --git a/eng/pipelines/pipeline-resources.yml b/eng/pipelines/pipeline-resources.yml index 0b3a5b51ed..dd384e25cf 100644 --- a/eng/pipelines/pipeline-resources.yml +++ b/eng/pipelines/pipeline-resources.yml @@ -1,62 +1,71 @@ parameters: - - name: stages - type: stageList +- name: stages + type: stageList +- name: isOfficialBuild + type: boolean -resources: - containers: - - container: linux_x64 +extends: + template: templateDispatch.yml + parameters: + ${{ if parameters.isOfficialBuild }}: + templatePath: template1es.yml + ${{ else }}: + templatePath: templatePublic.yml + + stages: ${{ parameters.stages }} + + containers: + linux_x64: image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7 - - container: linux_arm + linux_arm: image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross env: ROOTFS_DIR: /crossrootfs/arm - - container: linux_arm64 + linux_arm64: image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64 env: ROOTFS_DIR: /crossrootfs/arm64 - - container: linux_musl_x64 + linux_musl_x64: image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.13-WithNode - - container: linux_musl_arm + linux_musl_arm: image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm-alpine env: ROOTFS_DIR: /crossrootfs/arm - - container: linux_musl_arm64 + linux_musl_arm64: image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-alpine env: ROOTFS_DIR: /crossrootfs/arm64 - - container: test_linux_x64 + test_linux_x64: image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7 - - container: test_linux_musl_x64 + test_linux_musl_x64: image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.13-WithNode options: --cap-add=SYS_PTRACE - - container: test_debian_11_amd64 + test_debian_11_amd64: image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 options: '--env PYTHONPATH=/usr/bin/python3.9' - - container: test_fedora_36 + test_fedora_36: image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36 options: --cap-add=SYS_PTRACE - - container: test_opensuse_15_2 + test_opensuse_15_2: image: mcr.microsoft.com/dotnet-buildtools/prereqs:opensuse-15.2-helix-amd64 - - container: test_ubuntu_18_04 + test_ubuntu_18_04: image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04 - - container: test_ubuntu_20_04 + test_ubuntu_20_04: image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04 options: '--env PYTHONPATH=/usr/bin/python3.8' - - container: test_ubuntu_22_04 + test_ubuntu_22_04: image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 options: '--env PYTHONPATH=/usr/bin/python3.10' - -stages: ${{ parameters.stages }} diff --git a/eng/pipelines/prepare-release.yml b/eng/pipelines/prepare-release.yml index 73bf0499c0..fbf306648e 100644 --- a/eng/pipelines/prepare-release.yml +++ b/eng/pipelines/prepare-release.yml @@ -1,19 +1,17 @@ stages: - stage: PrepareReleaseStage displayName: Release Preparation - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: - dependsOn: - - publish_using_darc jobs: - job: PrepareReleaseJob displayName: Prepare release with Darc - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Public - demands: ImageOverride -equals windows.vs2022.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2022.amd64 + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'],'refs/heads/juhoyosa/1es-migration'))) }}: + templateContext: + outputs: + - output: pipelineArtifact + path: '$(System.ArtifactsDirectory)\ReleaseStaging' + artifact: DiagnosticsRelease + displayName: 'Publish Release Drop' + condition: succeeded() variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: - group: DotNet-Diagnostics-Storage @@ -24,12 +22,7 @@ stages: - script: '$(Build.Repository.LocalPath)\dotnet.cmd build $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release /bl' workingDirectory: '$(System.ArtifactsDirectory)' displayName: 'Build Manifest generation and asset publishing tool' - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '$(System.ArtifactsDirectory)' - publishLocation: 'pipeline' - artifact: 'DiagnosticsReleaseToolBin' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'],'refs/heads/juhoyosa/1es-migration'))) }}: - task: UseDotNet@2 displayName: 'Use .NET Core runtime 6.x' inputs: @@ -66,8 +59,3 @@ stages: -v True workingDirectory: '$(Build.Repository.LocalPath)\' displayName: 'Manifest generation and asset publishing' - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '$(System.ArtifactsDirectory)\ReleaseStaging' - publishLocation: 'pipeline' - artifact: 'DiagnosticsRelease' diff --git a/eng/pipelines/publish-pipeline-artifact-shim.yml b/eng/pipelines/publish-pipeline-artifact-shim.yml new file mode 100644 index 0000000000..a54ad7b11c --- /dev/null +++ b/eng/pipelines/publish-pipeline-artifact-shim.yml @@ -0,0 +1,25 @@ +parameters: + inputs: {} + displayName: 'Publish Pipeline Artifact' + condition: succeeded() + continueOnError: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + # If we are using this in a project other than the public one, use the 1es task. + - task: 1es.PublishPipelineArtifact@1 + inputs: + targetPath: ${{ parameters.inputs.targetPath }} + artifactName: ${{ parameters.inputs.artifactName }} + condition: ${{ parameters.condition }} + displayName: ${{ parameters.displayName }} + continueOnError: ${{ parameters.continueOnError }} +- ${{ else }}: + # Otherwise, use the public task. + - task: PublishPipelineArtifact@1 + inputs: + targetPath: ${{ parameters.inputs.targetPath }} + artifactName: ${{ parameters.inputs.artifactName }} + condition: ${{ parameters.condition }} + displayName: ${{ parameters.displayName }} + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/pipelines/template1es.yml b/eng/pipelines/template1es.yml new file mode 100644 index 0000000000..f95fab27fd --- /dev/null +++ b/eng/pipelines/template1es.yml @@ -0,0 +1,31 @@ + + +parameters: + - name: templatePath + type: string + default: no + - name: stages + type: stageList + - name: containers + type: object + + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: netcore1espool-internal + image: 1es-windows-2022-pt + os: windows + + containers: + ${{ parameters.containers }} + + stages: ${{ parameters.stages }} \ No newline at end of file diff --git a/eng/pipelines/templateDispatch.yml b/eng/pipelines/templateDispatch.yml new file mode 100644 index 0000000000..1860af47ae --- /dev/null +++ b/eng/pipelines/templateDispatch.yml @@ -0,0 +1,13 @@ +parameters: + - name: templatePath + type: string + - name: stages + type: stageList + - name: containers + type: object + +extends: + template: ${{ parameters.templatePath }} + parameters: + stages: ${{ parameters.stages }} + containers: ${{ parameters.containers }} \ No newline at end of file diff --git a/eng/pipelines/templatePublic.yml b/eng/pipelines/templatePublic.yml new file mode 100644 index 0000000000..8e80c802b2 --- /dev/null +++ b/eng/pipelines/templatePublic.yml @@ -0,0 +1,20 @@ + +parameters: + - name: templatePath + type: string + default: no + - name: stages + type: stageList + - name: containers + type: object + +resources: + containers: + - ${{ each container_pair in parameters.containers }}: + - ${{ if container_pair.value.image }}: + - container: ${{ container_pair.key }} + ${{ each pair in container_pair.value }}: + ${{ if notIn(pair.key, 'tenantId', 'identityType', 'registry') }}: + ${{ pair.key }}: ${{ pair.value }} + +stages: ${{ parameters.stages }} From 65feb765e70ba1588fa9d4ee5faecffad461ae41 Mon Sep 17 00:00:00 2001 From: Mike McLaughlin Date: Thu, 28 Mar 2024 17:55:40 -0700 Subject: [PATCH 2/6] Wrong internal pool-providers.yml --- eng/pipelines/global-variables.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/global-variables.yml b/eng/pipelines/global-variables.yml index 3ea66e870e..bf6f32e27c 100644 --- a/eng/pipelines/global-variables.yml +++ b/eng/pipelines/global-variables.yml @@ -11,7 +11,7 @@ variables: - ${{ if ne(variables['System.TeamProject'], 'public') }}: - group: DotNet-Diagnostics-SDL-Params - group: DotNet-MSRC-Storage - - template: /eng/common/templates/variables/pool-providers.yml@self + - template: /eng/common/templates-official/variables/pool-providers.yml@self - name: BuildPool value: $(DncEngInternalBuildPool) - name: WindowsImage From 5fb5c5ee1bb970c3059de52d279e5f2345d1080c Mon Sep 17 00:00:00 2001 From: Mike McLaughlin Date: Fri, 29 Mar 2024 06:08:12 -0700 Subject: [PATCH 3/6] Code review feedback --- diagnostics.yml | 14 +++++++------- eng/pipelines/build.yml | 1 - eng/pipelines/prepare-release.yml | 6 +++--- eng/pipelines/template1es.yml | 3 +-- eng/pipelines/templatePublic.yml | 1 - 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/diagnostics.yml b/diagnostics.yml index 11372e39e5..3788f0f642 100644 --- a/diagnostics.yml +++ b/diagnostics.yml @@ -96,9 +96,9 @@ extends: - configuration: Release architecture: arm artifactUploadPath: bin/Windows_NT.arm.Release - - configuration: Release - architecture: arm64 - artifactUploadPath: bin/Windows_NT.arm64.Release + - configuration: Release + architecture: arm64 + artifactUploadPath: bin/Windows_NT.arm64.Release - template: /eng/pipelines/build.yml parameters: @@ -289,7 +289,7 @@ extends: jobs: - template: /eng/common/templates-official/job/job.yml parameters: - name: Sign_Package_Publish + name: Package_Sign_Publish displayName: Packaging condition: succeeded() enablePublishUsingPipelines: true @@ -347,7 +347,7 @@ extends: - template: /eng/common/templates-official/job/publish-build-assets.yml parameters: configuration: Release - dependsOn: Sign_Package_Publish + dependsOn: Package_Sign_Publish publishUsingPipelines: true - template: /eng/common/templates-official/post-build/post-build.yml@self @@ -364,5 +364,5 @@ extends: symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=false' publishInstallersAndChecksums: true - # This sets up the bits to do a Release. - - template: /eng/pipelines/prepare-release.yml + # This sets up the bits to do a Release. + - template: /eng/pipelines/prepare-release.yml diff --git a/eng/pipelines/build.yml b/eng/pipelines/build.yml index beea048178..19277ee8d5 100644 --- a/eng/pipelines/build.yml +++ b/eng/pipelines/build.yml @@ -30,7 +30,6 @@ parameters: default: {} # Optional: Job timeout - - name: timeoutInMinutes type: number default: 180 diff --git a/eng/pipelines/prepare-release.yml b/eng/pipelines/prepare-release.yml index fbf306648e..5ea837cc9a 100644 --- a/eng/pipelines/prepare-release.yml +++ b/eng/pipelines/prepare-release.yml @@ -4,7 +4,7 @@ stages: jobs: - job: PrepareReleaseJob displayName: Prepare release with Darc - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'],'refs/heads/juhoyosa/1es-migration'))) }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/') }}: templateContext: outputs: - output: pipelineArtifact @@ -22,7 +22,7 @@ stages: - script: '$(Build.Repository.LocalPath)\dotnet.cmd build $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release /bl' workingDirectory: '$(System.ArtifactsDirectory)' displayName: 'Build Manifest generation and asset publishing tool' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'],'refs/heads/juhoyosa/1es-migration'))) }}: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: - task: UseDotNet@2 displayName: 'Use .NET Core runtime 6.x' inputs: @@ -45,7 +45,7 @@ stages: -ReleaseVersion "$(Build.BuildNumber)" workingDirectory: '$(Build.Repository.LocalPath)' - script: >- - dotnet.cmd run --project $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release + $(Build.Repository.LocalPath)\dotnet.cmd run --project $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release -- prepare-release --input-drop-path "$(System.ArtifactsDirectory)\ReleaseTarget" diff --git a/eng/pipelines/template1es.yml b/eng/pipelines/template1es.yml index f95fab27fd..b53ccb0bee 100644 --- a/eng/pipelines/template1es.yml +++ b/eng/pipelines/template1es.yml @@ -1,5 +1,4 @@ - parameters: - name: templatePath type: string @@ -28,4 +27,4 @@ extends: containers: ${{ parameters.containers }} - stages: ${{ parameters.stages }} \ No newline at end of file + stages: ${{ parameters.stages }} diff --git a/eng/pipelines/templatePublic.yml b/eng/pipelines/templatePublic.yml index 8e80c802b2..cca1b3f0d2 100644 --- a/eng/pipelines/templatePublic.yml +++ b/eng/pipelines/templatePublic.yml @@ -1,4 +1,3 @@ - parameters: - name: templatePath type: string From 5777e3e4f8e02cff368a5a2a335e904b8a83b192 Mon Sep 17 00:00:00 2001 From: Mike McLaughlin Date: Fri, 29 Mar 2024 06:10:10 -0700 Subject: [PATCH 4/6] Fix prepare-release.yml --- eng/pipelines/prepare-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/prepare-release.yml b/eng/pipelines/prepare-release.yml index 5ea837cc9a..3a0d97b4ca 100644 --- a/eng/pipelines/prepare-release.yml +++ b/eng/pipelines/prepare-release.yml @@ -4,7 +4,7 @@ stages: jobs: - job: PrepareReleaseJob displayName: Prepare release with Darc - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: templateContext: outputs: - output: pipelineArtifact From 96bf2792e7833079ba4b04e2db9c0c8c34fcb7e2 Mon Sep 17 00:00:00 2001 From: Mike McLaughlin Date: Fri, 29 Mar 2024 10:23:00 -0700 Subject: [PATCH 5/6] Code review feedback --- eng/pipelines/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/build.yml b/eng/pipelines/build.yml index 19277ee8d5..bd0f6289a0 100644 --- a/eng/pipelines/build.yml +++ b/eng/pipelines/build.yml @@ -110,7 +110,7 @@ jobs: os: freebsd ${{ else }}: - 'Unable to identify pool for required build': error + 'error, Unable to identify pool for required build': error ${{ if ne(parameters.container, '') }}: container: ${{ parameters.container }} @@ -260,7 +260,7 @@ jobs: displayName: Publish diagnostic artifacts on failure inputs: targetPath: $(Build.ArtifactStagingDirectory)/diagnostics - artifactName: DiagnosticArtifacts_$(_PhaseName) + artifactName: DiagnosticArtifacts_$(_PhaseName)_Attempt$(System.JobAttempt) continueOnError: true condition: failed() @@ -278,7 +278,7 @@ jobs: displayName: Publish Logs inputs: targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: Logs_$(_PhaseName) + artifactName: Logs_$(_PhaseName)_Attempt$(System.JobAttempt) continueOnError: true condition: always() From 595ac10da1f0398563edb32acf6d2804cbbe935b Mon Sep 17 00:00:00 2001 From: Mike McLaughlin Date: Fri, 29 Mar 2024 10:32:42 -0700 Subject: [PATCH 6/6] Cloean up _InternalBuildArgs variable --- diagnostics.yml | 14 +------------- eng/pipelines/global-variables.yml | 6 ++++++ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/diagnostics.yml b/diagnostics.yml index 3788f0f642..24133cab4d 100644 --- a/diagnostics.yml +++ b/diagnostics.yml @@ -35,18 +35,6 @@ variables: parameters: runtimeFeed: ${{ parameters.runtimeFeed }} runtimeFeedToken: ${{ parameters.runtimeFeedToken }} -- name: _InternalBuildArgs - value: '' -- ${{ if ne(variables['System.TeamProject'], 'public') }}: - - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - # DotNet-Diagnostics-SDL-Params provides Tsa* variables for SDL checks. - - name: _SignType - value: real - - name: _InternalBuildArgs - value: /p:DotNetSignType=$(_SignType) - /p:TeamName=$(_TeamName) - /p:DotNetPublishUsingPipelines=true - /p:OfficialBuildId=$(BUILD.BUILDNUMBER) extends: template: /eng/pipelines/pipeline-resources.yml @@ -339,7 +327,7 @@ extends: displayName: 'Setup layout for packaging' # Create nuget packages, sign binaries and publish to blob feed - - script: $(Build.SourcesDirectory)\eng\ci-prepare-artifacts.cmd $(_InternalBuildArgs) + - script: $(Build.SourcesDirectory)\eng\ci-prepare-artifacts.cmd /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) /p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER) displayName: Package, Sign, and Publish continueOnError: false condition: succeeded() diff --git a/eng/pipelines/global-variables.yml b/eng/pipelines/global-variables.yml index bf6f32e27c..8511708432 100644 --- a/eng/pipelines/global-variables.yml +++ b/eng/pipelines/global-variables.yml @@ -7,6 +7,9 @@ variables: - name: _TeamName value: DotNetCore +- name: _SignType + value: '' + # # Internal build pools. - ${{ if ne(variables['System.TeamProject'], 'public') }}: - group: DotNet-Diagnostics-SDL-Params @@ -35,6 +38,9 @@ variables: value: $(System.JobDisplayName) - name: Codeql.Language value: csharp,cpp + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - name: _SignType + value: real # Public build settings - ${{ else }}: - template: /eng/common/templates/variables/pool-providers.yml@self