From e81bf4eaa377a94f56f98fb6272e8bdd74ca09c9 Mon Sep 17 00:00:00 2001 From: dotnet-docker-bot <60522487+dotnet-docker-bot@users.noreply.github.com> Date: Fri, 23 Aug 2024 12:26:18 -0700 Subject: [PATCH] [main] Update common Docker engineering infrastructure with latest (#1143) Co-authored-by: Logan Bussell --- .../templates/jobs/cg-build-projects.yml | 36 +++++++++++++++++++ .../templates/steps/annotate-eol-digests.yml | 31 ++++++++++------ .../steps/test-images-linux-client.yml | 16 ++++----- .../steps/test-images-windows-client.yml | 9 ++--- .../templates/variables/docker-images.yml | 2 +- 5 files changed, 68 insertions(+), 26 deletions(-) diff --git a/eng/common/templates/jobs/cg-build-projects.yml b/eng/common/templates/jobs/cg-build-projects.yml index 91134b459..976a06783 100644 --- a/eng/common/templates/jobs/cg-build-projects.yml +++ b/eng/common/templates/jobs/cg-build-projects.yml @@ -1,6 +1,13 @@ # This job builds all projects in the repository. It is intended to be used for CG purposes. # The 1ES CG step does not scan artifacts that are built within Dockerfiles therefore they # need to be built outside of Dockerfiles. +parameters: +# Setting cgDryRun will run CG but not submit the results +- name: cgDryRun + type: boolean + default: false + displayName: CG Dry Run + jobs: - job: BuildProjects displayName: Build Projects @@ -15,3 +22,32 @@ jobs: - script: > find . -name '*.csproj' | grep $(cgBuildGrepArgs) | xargs -n 1 /usr/share/.dotnet/dotnet build displayName: Build Projects + + # Component Detection is only automatically run on production branches. + # To run Component Detection on non-production branches, the task must be manually injected. + - ${{ if eq(parameters.cgDryRun, true) }}: + - powershell: | + Write-Host "##vso[build.updatebuildnumber]$env:BUILD_BUILDNUMBER (Dry run)" + Write-Host "##vso[build.addbuildtag]dry-run" + + if ("$(officialBranches)".Split(',').Contains("$(Build.SourceBranch)")) + { + Write-Host "##vso[task.logissue type=error]Cannot run a CG dry-run build from an official branch ($(officialBranches))." + Write-Host "##vso[task.logissue type=error]Run the pipeline again from a different branch to avoid registering scan results." + exit 1 + } + displayName: Update Build Number + - task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (manually injected) + inputs: + # Running CG with `whatIf: true` or `scanType: LogOnly` outputs a list of detected components, but doesn't show + # which components would trigger an alert on a production build. + # As long as the build isn't ran on a tracked branch (typically main or nightly), it's OK to submit components + # to CG for analysis. Only the results for tracked branches matter. + scanType: Register + whatIf: false + alertWarningLevel: Low + failOnAlert: false + ignoreDirectories: $(Build.SourcesDirectory)/versions + showAlertLink: true + timeoutInMinutes: 10 diff --git a/eng/common/templates/steps/annotate-eol-digests.yml b/eng/common/templates/steps/annotate-eol-digests.yml index f58fb4d06..92c9199eb 100644 --- a/eng/common/templates/steps/annotate-eol-digests.yml +++ b/eng/common/templates/steps/annotate-eol-digests.yml @@ -1,23 +1,32 @@ parameters: - internalProjectName: null - force: false dataFile: null steps: - - script: | - optionalArgs="" - if [ "${{ lower(parameters.force) }}" == "true" ]; then - optionalArgs="$optionalArgs --force" - fi - echo "##vso[task.setvariable variable=optionalArgs]$optionalArgs" - displayName: Set Optional Args + - script: mkdir -p $(Build.ArtifactStagingDirectory)/annotation-digests + displayName: Create Annotation Digests Directory - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: name: AnnotateEOLImages displayName: Annotate EOL Images serviceConnection: $(publish.serviceConnectionName) - internalProjectName: ${{ parameters.internalProjectName }} + internalProjectName: internal args: > annotateEolDigests /repo/${{ parameters.dataFile }} $(acr.server) - $(optionalArgs) + $(publishRepoPrefix) + $(artifactsPath)/annotation-digests/annotation-digests.txt + - template: /eng/common/templates/steps/publish-artifact.yml@self + parameters: + path: $(Build.ArtifactStagingDirectory)/annotation-digests + artifactName: annotation-digests-$(System.JobAttempt) + displayName: Publish Annotation Digests List + internalProjectName: internal + publicProjectName: public + - template: /eng/common/templates/steps/run-imagebuilder.yml@self + parameters: + displayName: Wait for Annotation Ingestion + serviceConnection: $(marStatus.serviceConnectionName) + internalProjectName: internal + args: > + waitForMarAnnotationIngestion + $(artifactsPath)/annotation-digests/annotation-digests.txt diff --git a/eng/common/templates/steps/test-images-linux-client.yml b/eng/common/templates/steps/test-images-linux-client.yml index 61090bbf1..6a747016c 100644 --- a/eng/common/templates/steps/test-images-linux-client.yml +++ b/eng/common/templates/steps/test-images-linux-client.yml @@ -17,18 +17,18 @@ steps: - script: | echo "##vso[task.setvariable variable=testRunner.container]testrunner-$(Build.BuildId)-$(System.JobId)" - optionalTestArgs="" + additionalTestArgs="$ADDITIONALTESTARGS" if [ "${{ parameters.preBuildValidation }}" == "true" ]; then - optionalTestArgs="$optionalTestArgs -TestCategories pre-build" + additionalTestArgs="$additionalTestArgs -TestCategories pre-build" else if [ "${{ variables['System.TeamProject'] }}" == "${{ parameters.internalProjectName }}" ] && [ "${{ variables['Build.Reason'] }}" != "PullRequest" ]; then - optionalTestArgs="$optionalTestArgs -PullImages -Registry $(acr-staging.server) -RepoPrefix $(stagingRepoPrefix) -ImageInfoPath $(artifactsPath)/image-info.json" - fi - if [ "$REPOTESTARGS" != "" ]; then - optionalTestArgs="$optionalTestArgs $REPOTESTARGS" + additionalTestArgs="$additionalTestArgs -PullImages -Registry $(acr-staging.server) -RepoPrefix $(stagingRepoPrefix) -ImageInfoPath $(artifactsPath)/image-info.json" + if [ "$TESTCATEGORIESOVERRIDE" != "" ]; then + additionalTestArgs="$additionalTestArgs -TestCategories $TESTCATEGORIESOVERRIDE" + fi fi fi - echo "##vso[task.setvariable variable=optionalTestArgs]$optionalTestArgs" + echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" displayName: Set Test Variables condition: and(succeeded(), ${{ parameters.condition }}) - script: > @@ -69,7 +69,7 @@ steps: -Paths $(imageBuilderPathsArrayInitStr) -OSVersions $(osVersionsArrayInitStr) -Architecture '$(architecture)' - $(optionalTestArgs)" + $(additionalTestArgs)" displayName: Test Images condition: and(succeeded(), ${{ parameters.condition }}) - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: diff --git a/eng/common/templates/steps/test-images-windows-client.yml b/eng/common/templates/steps/test-images-windows-client.yml index 265ab421b..04b099d52 100644 --- a/eng/common/templates/steps/test-images-windows-client.yml +++ b/eng/common/templates/steps/test-images-windows-client.yml @@ -23,12 +23,9 @@ steps: - ${{ parameters.customInitSteps }} - powershell: | if ("${{ variables['System.TeamProject'] }}" -eq "${{ parameters.internalProjectName }}" -and "${{ variables['Build.Reason'] }}" -ne "PullRequest") { - $optionalTestArgs="$optionalTestArgs -PullImages -Registry ${env:ACR-STAGING_SERVER} -RepoPrefix $env:STAGINGREPOPREFIX -ImageInfoPath $(artifactsPath)/image-info.json" + $additionalTestArgs="$env:ADDITIONALTESTARGS -PullImages -Registry ${env:ACR-STAGING_SERVER} -RepoPrefix $env:STAGINGREPOPREFIX -ImageInfoPath $(artifactsPath)/image-info.json" } - if ($env:REPOTESTARGS) { - $optionalTestArgs += " $env:REPOTESTARGS" - } - echo "##vso[task.setvariable variable=optionalTestArgs]$optionalTestArgs" + echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" displayName: Set Test Variables condition: and(succeeded(), ${{ parameters.condition }}) - powershell: Get-ChildItem -Path tests -r | Where {$_.Extension -match "trx"} | Remove-Item @@ -46,7 +43,7 @@ steps: $(testScriptPath) -Paths $(imageBuilderPathsArrayInitStr) -OSVersions $(osVersionsArrayInitStr) - $(optionalTestArgs) + $(additionalTestArgs) displayName: Test Images condition: and(succeeded(), ${{ parameters.condition }}) - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: diff --git a/eng/common/templates/variables/docker-images.yml b/eng/common/templates/variables/docker-images.yml index 016d1f63a..028c5f193 100644 --- a/eng/common/templates/variables/docker-images.yml +++ b/eng/common/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2491493 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2516503 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner2.0-docker-testrunner