Add job matrix generation scripts (#1429)

This PR is a port of functionality that is currently duplicated across the net/java/python repositories. The intent was to settle on an implementation before moving it to the /eng/common/scripts directory. After merge to this location, I'll update the net/java/python and js repos to point to the `eng/common/scripts` location and remove the scripts from `eng/scripts`.

Here is the PR text used against the other repos for reference:

This adds scripts, docs and samples supporting dynamic, cross-product matrix generation for azure pipeline jobs.
It aims to replicate the [cross-product matrix functionality in github actions](https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#example-running-with-more-than-one-version-of-nodejs),
but also adds some additional features like sparse matrix generation, cross-product includes and excludes, parameter grouping and matrix filters.

This functionality is made possible by the ability for the azure pipelines yaml to take a [dynamic variable as an input
for a job matrix definition](https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration) (see the code sample at the bottom of the linked section).

See the README.md file for more details on the config file syntax and usage, as well as implementation details.

The tests (`test-matrix-functions.tests.ps1`) contain a lot of detail on expected data structures at various processing stages. The `-`test-matrix-functions.ps1` file could perhaps be split up or use some more organization, so let me know if it's hard to navigate.

Example:
```
{
  "displayNames": {
    "true": "TestFromSource"
  },
  "matrix": {
    "Agent": {
      "ubuntu-18.04": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" },
      "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" },
      "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" }
    },
    "JavaTestVersion": [ "1.8", "1.11" ],
    "AZURE_TEST_HTTP_CLIENTS": [ "okhttp", "netty" ]
  },
  "include": [
    {
      "Agent": {
          "ubuntu-18.04": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" }
      },
      "JavaTestVersion": "1.11",
      "AZURE_TEST_HTTP_CLIENTS": "netty",
      "TestFromSource": true
    }
  ]
}
```

Sparse matrix job generation in a pipeline: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=705622&view=results

![image](https://user-images.githubusercontent.com/1020379/106040177-151e1f80-60a8-11eb-823c-2af96b5e84aa.png)

Related discussion: microsoft/azure-pipelines-yaml#20

eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml

@@ -0,0 +1,73 @@
+parameters:
+- name: AdditionalParameters
+  type: object
+- name: CloudConfig
+  type: object
+  default: {}
+- name: MatrixConfigs
+  type: object
+  default: []
+- name: MatrixFilters
+  type: object
+  default: []
+- name: JobTemplatePath
+  type: string
+
+jobs:
+- job: generate_matrix
+  variables:
+    displayNameFilter: $[ coalesce(variables.jobMatrixFilter, '.*') ]
+  pool:
+    name: Azure Pipelines
+    vmImage: ubuntu-18.04
+  displayName: Generate Job Matrix
+  steps:
+    - ${{ each config in parameters.MatrixConfigs }}:
+      - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+        - task: Powershell@2
+          inputs:
+            pwsh: true
+            filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+            arguments: >
+              -ConfigPath ${{ config.Path }}
+              -Selection ${{ config.Selection }}
+              -DisplayNameFilter "$(displayNameFilter)"
+              -Filters "${{ join('","', parameters.MatrixFilters) }}","container=^$","SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}"
+              -NonSparseParameters "${{ join('","', config.NonSparseParameters) }}"
+          displayName: Generate VM Job Matrix ${{ config.Name }}
+          name: generate_vm_job_matrix_${{ config.Name }}
+
+      - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+        - task: Powershell@2
+          inputs:
+            pwsh: true
+            filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
+            arguments: >
+              -ConfigPath ${{ config.Path }}
+              -Selection ${{ config.Selection }}
+              -DisplayNameFilter "$(displayNameFilter)"
+              -Filters "${{ join('","', parameters.MatrixFilters) }}", "container=.*", "SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}"
+              -NonSparseParameters "${{ join('","', config.NonSparseParameters) }}"
+          displayName: Generate Container Job Matrix
+          name: generate_container_job_matrix_${{ config.Name }}
+
+- ${{ each config in parameters.MatrixConfigs }}:
+  - ${{ if eq(config.GenerateVMJobs, 'true') }}:
+    - template: ${{ parameters.JobTemplatePath }}
+      parameters:
+        UsePlatformContainer: false
+        Matrix: dependencies.generate_matrix.outputs['generate_vm_job_matrix_${{ config.Name }}.matrix']
+        DependsOn: generate_matrix
+        CloudConfig: ${{ parameters.CloudConfig }}
+        ${{ each param in parameters.AdditionalParameters }}:
+          ${{ param.key }}: ${{ param.value }}
+
+  - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
+    - template: ${{ parameters.JobTemplatePath }}
+      parameters:
+        UsePlatformContainer: true
+        Matrix: dependencies.generate_matrix.outputs['generate_container_job_matrix_${{ config.Name }}.matrix']
+        DependsOn: generate_matrix
+        CloudConfig: ${{ parameters.CloudConfig }}
+        ${{ each param in parameters.AdditionalParameters }}:
+          ${{ param.key }}: ${{ param.value }}

eng/common/scripts/job-matrix/Create-JobMatrix.ps1

@@ -0,0 +1,35 @@
+<#
+    .SYNOPSIS
+        Generates a JSON object representing an Azure Pipelines Job Matrix.
+        See https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#parallelexec
+
+    .EXAMPLE
+    ./eng/common/scripts/Create-JobMatrix $context
+#>
+
+[CmdletBinding()]
+param (
+    [Parameter(Mandatory=$True)][string] $ConfigPath,
+    [Parameter(Mandatory=$True)][string] $Selection,
+    [Parameter(Mandatory=$False)][string] $DisplayNameFilter,
+    [Parameter(Mandatory=$False)][array] $Filters,
+    [Parameter(Mandatory=$False)][array] $NonSparseParameters
+)
+
+. $PSScriptRoot/job-matrix-functions.ps1
+
+$config = GetMatrixConfigFromJson (Get-Content $ConfigPath)
+# Strip empty string filters in order to be able to use azure pipelines yaml join()
+$Filters = $Filters | Where-Object { $_ }
+
+[array]$matrix = GenerateMatrix `
+    -config $config `
+    -selectFromMatrixType $Selection `
+    -displayNameFilter $DisplayNameFilter `
+    -filters $Filters `
+    -nonSparseParameters $NonSparseParameters
+
+$serialized = SerializePipelineMatrix $matrix
+
+Write-Output $serialized.pretty
+Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)"