Sync eng/common directory with azure-sdk-tools for PR 9281 (Azure#31589)

* package-properties are now populated with matrix configurations from their ci.yml if present * create new code path for generate-job-matrix.yml which combines Create-JobMatrix and the "distribute-packages-to-matrix" action to generate dynamic matrices for PRs --------- Co-authored-by: Scott Beddall <[email protected]>
azure-sdk · Oct 30, 2024 · 6f7114a · 6f7114a
1 parent ad4d271
commit 6f7114a
Show file tree

Hide file tree

Showing 4 changed files with 340 additions and 113 deletions.
diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml
@@ -42,6 +42,12 @@ parameters:
 - name: PreGenerationSteps
   type: stepList
   default: []
+- name: EnablePRGeneration
+  type: boolean
+  default: false
+- name: PRMatrixSetting
+  type: string
+  default: 'ArtifactPackageNames'
 # Mappings to OS name required at template compile time by 1es pipeline templates
 - name: Pools
   type: object
@@ -84,57 +90,87 @@ jobs:
 
     - ${{ parameters.PreGenerationSteps }}
 
-    - ${{ each config in parameters.MatrixConfigs }}:
+    - ${{ if eq(parameters.EnablePRGeneration, false) }}:
+      - ${{ each config in parameters.MatrixConfigs }}:
+        - ${{ each pool in parameters.Pools }}:
+          - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+            - task: Powershell@2
+              inputs:
+                pwsh: true
+                filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+                arguments: >
+                  -ConfigPath ${{ config.Path }}
+                  -Selection ${{ config.Selection }}
+                  -DisplayNameFilter '$(displayNameFilter)'
+                  -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
+                  -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
+                  -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+              displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
+              name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
+          - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+            - task: Powershell@2
+              inputs:
+                pwsh: true
+                filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+                arguments: >
+                  -ConfigPath ${{ config.Path }}
+                  -Selection ${{ config.Selection }}
+                  -DisplayNameFilter '$(displayNameFilter)'
+                  -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
+                  -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
+              displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
+              name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
+
+      # This else being set also currently assumes that the $(Build.ArtifactStagingDirectory)/PackageInfo folder is populated by PreGenerationSteps.
+      # Not currently not hardcoded, so not doing the needful and populating this folder before we hit this step will result in generation errors.
+    - ${{ else }}:
       - ${{ each pool in parameters.Pools }}:
-        - ${{ if eq(config.GenerateVMJobs, 'true') }}:
-          - task: Powershell@2
-            inputs:
-              pwsh: true
-              filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
-              arguments: >
-                -ConfigPath ${{ config.Path }}
-                -Selection ${{ config.Selection }}
-                -DisplayNameFilter '$(displayNameFilter)'
-                -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
-                -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
-                -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
-            displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
-            name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
+        - pwsh: |
+            # dump the conglomerated CI matrix
+            '${{ convertToJson(parameters.MatrixConfigs) }}' | Set-Content matrix.json
 
-        - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
-          - task: Powershell@2
-            inputs:
-              pwsh: true
-              filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
-              arguments: >
-                -ConfigPath ${{ config.Path }}
-                -Selection ${{ config.Selection }}
-                -DisplayNameFilter '$(displayNameFilter)'
-                -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
-                -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
-            displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
-            name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
+            ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 `
+              -PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo `
+              -PRMatrixFile matrix.json `
+              -PRMatrixSetting ${{ parameters.PRMatrixSetting }} `
+              -DisplayNameFilter '$(displayNameFilter)' `
+              -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' `
+              -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
+          displayName: Create ${{ pool.name }} PR Matrix
+          name: vm_job_matrix_pr_${{ pool.name }}
 
-- ${{ each config in parameters.MatrixConfigs }}:
-  - ${{ each pool in parameters.Pools }}:
-    - ${{ if eq(config.GenerateVMJobs, 'true') }}:
-      - template: ${{ parameters.JobTemplatePath }}
-        parameters:
-          UsePlatformContainer: false
-          OSName: ${{ pool.os }}
-          Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
-          DependsOn: ${{ parameters.GenerateJobName }}
-          CloudConfig: ${{ parameters.CloudConfig }}
-          ${{ each param in parameters.AdditionalParameters }}:
-            ${{ param.key }}: ${{ param.value }}
+- ${{ if eq(parameters.EnablePRGeneration, false) }}:
+  - ${{ each config in parameters.MatrixConfigs }}:
+    - ${{ each pool in parameters.Pools }}:
+      - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+        - template: ${{ parameters.JobTemplatePath }}
+          parameters:
+            UsePlatformContainer: false
+            OSName: ${{ pool.os }}
+            Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
+            DependsOn: ${{ parameters.GenerateJobName }}
+            CloudConfig: ${{ parameters.CloudConfig }}
+            ${{ each param in parameters.AdditionalParameters }}:
+              ${{ param.key }}: ${{ param.value }}
 
-    - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
-      - template: ${{ parameters.JobTemplatePath }}
-        parameters:
-          UsePlatformContainer: true
-          OSName: ${{ pool.os }}
-          Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
-          DependsOn: ${{ parameters.GenerateJobName }}
-          CloudConfig: ${{ parameters.CloudConfig }}
-          ${{ each param in parameters.AdditionalParameters }}:
-            ${{ param.key }}: ${{ param.value }}
+      - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+        - template: ${{ parameters.JobTemplatePath }}
+          parameters:
+            UsePlatformContainer: true
+            OSName: ${{ pool.os }}
+            Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
+            DependsOn: ${{ parameters.GenerateJobName }}
+            CloudConfig: ${{ parameters.CloudConfig }}
+            ${{ each param in parameters.AdditionalParameters }}:
+              ${{ param.key }}: ${{ param.value }}
+- ${{ else }}:
+  - ${{ each pool in parameters.Pools }}:
+    - template: ${{ parameters.JobTemplatePath }}
+      parameters:
+        UsePlatformContainer: false
+        OSName: ${{ pool.os }}
+        Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_pr_${{ pool.name }}.matrix']
+        DependsOn: ${{ parameters.GenerateJobName }}
+        CloudConfig: ${{ parameters.CloudConfig }}
+        ${{ each param in parameters.AdditionalParameters }}:
+          ${{ param.key }}: ${{ param.value }}
diff --git a/eng/common/scripts/Helpers/Package-Helpers.ps1 b/eng/common/scripts/Helpers/Package-Helpers.ps1
@@ -170,10 +170,86 @@ function GetValueSafelyFrom-Yaml {
         $current = $current[$key]
       }
       else {
-        Write-Host "The '$key' part of the path $($Keys -join "/") doesn't exist or is null."
         return $null
       }
   }
 
   return [object]$current
-}
+}
+
+function Get-ObjectKey {
+  param (
+    [Parameter(Mandatory = $true)]
+    [object]$Object
+  )
+
+  if (-not $Object) {
+    return "unset"
+  }
+
+  if ($Object -is [hashtable] -or $Object -is [System.Collections.Specialized.OrderedDictionary]) {
+    $sortedEntries = $Object.GetEnumerator() | Sort-Object Name
+    $hashString = ($sortedEntries | ForEach-Object { "$($_.Key)=$($_.Value)" }) -join ";"
+    return $hashString.GetHashCode()
+  }
+
+  elseif ($Object -is [PSCustomObject]) {
+    $sortedProperties = $Object.PSObject.Properties | Sort-Object Name
+    $propertyString = ($sortedProperties | ForEach-Object { "$($_.Name)=$($_.Value)" }) -join ";"
+    return $propertyString.GetHashCode()
+  }
+
+  elseif ($Object -is [array]) {
+    $arrayString = ($Object | ForEach-Object { Get-ObjectKey $_ }) -join ";"
+    return $arrayString.GetHashCode()
+  }
+
+  else {
+    return $Object.GetHashCode()
+  }
+}
+
+function Group-ByObjectKey {
+  param (
+    [Parameter(Mandatory)]
+    [array]$Items,
+
+    [Parameter(Mandatory)]
+    [string]$GroupByProperty
+  )
+
+  $groupedDictionary = @{}
+
+  foreach ($item in $Items) {
+    $key = Get-ObjectKey $item."$GroupByProperty"
+
+    if (-not $groupedDictionary.ContainsKey($key)) {
+      $groupedDictionary[$key] = @()
+    }
+
+    # Add the current item to the array for this key
+    $groupedDictionary[$key] += $item
+  }
+
+  return $groupedDictionary
+}
+
+function Split-ArrayIntoBatches {
+  param (
+    [Parameter(Mandatory = $true)]
+    [Object[]]$InputArray,
+
+    [Parameter(Mandatory = $true)]
+    [int]$BatchSize
+  )
+
+  $batches = @()
+
+  for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) {
+    $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)]
+
+    $batches += , $batch
+  }
+
+  return , $batches
+}