[main] Update common Docker engineering infrastructure with latest (#…

…1143)

Co-authored-by: Logan Bussell <[email protected]>

eng/common/templates/jobs/cg-build-projects.yml

@@ -1,6 +1,13 @@
 # This job builds all projects in the repository. It is intended to be used for CG purposes.
 # The 1ES CG step does not scan artifacts that are built within Dockerfiles therefore they
 # need to be built outside of Dockerfiles.
+parameters:
+# Setting cgDryRun will run CG but not submit the results
+- name: cgDryRun
+  type: boolean
+  default: false
+  displayName: CG Dry Run
+
 jobs:
 - job: BuildProjects
   displayName: Build Projects
@@ -15,3 +22,32 @@ jobs:
   - script: >
       find . -name '*.csproj' | grep $(cgBuildGrepArgs) | xargs -n 1 /usr/share/.dotnet/dotnet build
     displayName: Build Projects
+
+    # Component Detection is only automatically run on production branches.
+    # To run Component Detection on non-production branches, the task must be manually injected.
+  - ${{ if eq(parameters.cgDryRun, true) }}:
+    - powershell: |
+        Write-Host "##vso[build.updatebuildnumber]$env:BUILD_BUILDNUMBER (Dry run)"
+        Write-Host "##vso[build.addbuildtag]dry-run"
+        
+        if ("$(officialBranches)".Split(',').Contains("$(Build.SourceBranch)"))
+        {
+          Write-Host "##vso[task.logissue type=error]Cannot run a CG dry-run build from an official branch ($(officialBranches))."
+          Write-Host "##vso[task.logissue type=error]Run the pipeline again from a different branch to avoid registering scan results."
+          exit 1
+        }
+      displayName: Update Build Number
+    - task: ComponentGovernanceComponentDetection@0
+      displayName: Component Detection (manually injected)
+      inputs:
+        # Running CG with `whatIf: true` or `scanType: LogOnly` outputs a list of detected components, but doesn't show
+        # which components would trigger an alert on a production build.
+        # As long as the build isn't ran on a tracked branch (typically main or nightly), it's OK to submit components
+        # to CG for analysis. Only the results for tracked branches matter.
+        scanType: Register
+        whatIf: false
+        alertWarningLevel: Low
+        failOnAlert: false
+        ignoreDirectories: $(Build.SourcesDirectory)/versions
+        showAlertLink: true
+        timeoutInMinutes: 10

eng/common/templates/steps/annotate-eol-digests.yml

@@ -1,23 +1,32 @@
 parameters:
-  internalProjectName: null
-  force: false
   dataFile: null
 steps:
-  - script: |
-      optionalArgs=""
-      if [ "${{ lower(parameters.force) }}" == "true" ]; then
-        optionalArgs="$optionalArgs --force"
-      fi
-      echo "##vso[task.setvariable variable=optionalArgs]$optionalArgs"
-    displayName: Set Optional Args
+  - script: mkdir -p $(Build.ArtifactStagingDirectory)/annotation-digests
+    displayName: Create Annotation Digests Directory
   - template: /eng/common/templates/steps/run-imagebuilder.yml@self
     parameters:
       name: AnnotateEOLImages
       displayName: Annotate EOL Images
       serviceConnection: $(publish.serviceConnectionName)
-      internalProjectName: ${{ parameters.internalProjectName }}
+      internalProjectName: internal
       args: >
         annotateEolDigests
         /repo/${{ parameters.dataFile }}
         $(acr.server)
-        $(optionalArgs)
+        $(publishRepoPrefix)
+        $(artifactsPath)/annotation-digests/annotation-digests.txt
+  - template: /eng/common/templates/steps/publish-artifact.yml@self
+    parameters:
+      path: $(Build.ArtifactStagingDirectory)/annotation-digests
+      artifactName: annotation-digests-$(System.JobAttempt)
+      displayName: Publish Annotation Digests List
+      internalProjectName: internal
+      publicProjectName: public
+  - template: /eng/common/templates/steps/run-imagebuilder.yml@self
+    parameters:
+      displayName: Wait for Annotation Ingestion
+      serviceConnection: $(marStatus.serviceConnectionName)
+      internalProjectName: internal
+      args: >
+        waitForMarAnnotationIngestion
+        $(artifactsPath)/annotation-digests/annotation-digests.txt

eng/common/templates/steps/test-images-linux-client.yml

@@ -17,18 +17,18 @@ steps:
 - script: |
     echo "##vso[task.setvariable variable=testRunner.container]testrunner-$(Build.BuildId)-$(System.JobId)"
 
-    optionalTestArgs=""
+    additionalTestArgs="$ADDITIONALTESTARGS"
     if [ "${{ parameters.preBuildValidation }}" == "true" ]; then
-      optionalTestArgs="$optionalTestArgs -TestCategories pre-build"
+      additionalTestArgs="$additionalTestArgs -TestCategories pre-build"
     else
       if [ "${{ variables['System.TeamProject'] }}" == "${{ parameters.internalProjectName }}" ] && [ "${{ variables['Build.Reason'] }}" != "PullRequest" ]; then
-        optionalTestArgs="$optionalTestArgs -PullImages -Registry $(acr-staging.server) -RepoPrefix $(stagingRepoPrefix) -ImageInfoPath $(artifactsPath)/image-info.json"
-      fi
-      if [ "$REPOTESTARGS" != "" ]; then
-        optionalTestArgs="$optionalTestArgs $REPOTESTARGS"
+        additionalTestArgs="$additionalTestArgs -PullImages -Registry $(acr-staging.server) -RepoPrefix $(stagingRepoPrefix) -ImageInfoPath $(artifactsPath)/image-info.json"
+        if [ "$TESTCATEGORIESOVERRIDE" != "" ]; then
+          additionalTestArgs="$additionalTestArgs -TestCategories $TESTCATEGORIESOVERRIDE"
+        fi
       fi
     fi
-    echo "##vso[task.setvariable variable=optionalTestArgs]$optionalTestArgs"
+    echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs"
   displayName: Set Test Variables
   condition: and(succeeded(), ${{ parameters.condition }})
 - script: >
@@ -69,7 +69,7 @@ steps:
     -Paths $(imageBuilderPathsArrayInitStr)
     -OSVersions $(osVersionsArrayInitStr)
     -Architecture '$(architecture)'
-    $(optionalTestArgs)"
+    $(additionalTestArgs)"
   displayName: Test Images
   condition: and(succeeded(), ${{ parameters.condition }})
 - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:

eng/common/templates/steps/test-images-windows-client.yml

@@ -23,12 +23,9 @@ steps:
 - ${{ parameters.customInitSteps }}
 - powershell: |
     if ("${{ variables['System.TeamProject'] }}" -eq "${{ parameters.internalProjectName }}" -and "${{ variables['Build.Reason'] }}" -ne "PullRequest") {
-      $optionalTestArgs="$optionalTestArgs -PullImages -Registry ${env:ACR-STAGING_SERVER} -RepoPrefix $env:STAGINGREPOPREFIX -ImageInfoPath $(artifactsPath)/image-info.json"
+      $additionalTestArgs="$env:ADDITIONALTESTARGS -PullImages -Registry ${env:ACR-STAGING_SERVER} -RepoPrefix $env:STAGINGREPOPREFIX -ImageInfoPath $(artifactsPath)/image-info.json"
     }
-    if ($env:REPOTESTARGS) {
-      $optionalTestArgs += " $env:REPOTESTARGS"
-    }
-    echo "##vso[task.setvariable variable=optionalTestArgs]$optionalTestArgs"
+    echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs"
   displayName: Set Test Variables
   condition: and(succeeded(), ${{ parameters.condition }})
 - powershell: Get-ChildItem -Path tests -r | Where {$_.Extension -match "trx"} | Remove-Item
@@ -46,7 +43,7 @@ steps:
     $(testScriptPath)
     -Paths $(imageBuilderPathsArrayInitStr)
     -OSVersions $(osVersionsArrayInitStr)
-    $(optionalTestArgs)
+    $(additionalTestArgs)
   displayName: Test Images
   condition: and(succeeded(), ${{ parameters.condition }})
 - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:

eng/common/templates/variables/docker-images.yml

@@ -1,5 +1,5 @@
 variables:
-  imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2491493
+  imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2516503
   imageNames.imageBuilder: $(imageNames.imageBuilderName)
   imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId)
   imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner2.0-docker-testrunner