From cc42f23c54f304e6f42c2f8c85c1cb3528eb15cf Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 26 Mar 2021 11:05:59 -0400 Subject: [PATCH 1/2] Fix issue where 2d index arrays with length 1 get flattened by powershell --- .../job-matrix/job-matrix-functions.ps1 | 4 +-- ...ob-matrix-functions.modification.tests.ps1 | 33 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index adcdaf4bd963..dc7e19684abd 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -437,7 +437,7 @@ function GenerateSparseMatrix( $matrix = GenerateFullMatrix $parameters $displayNamesLookup $sparseMatrix = @() - $indexes = GetSparseMatrixIndexes $dimensions + [array]$indexes = GetSparseMatrixIndexes $dimensions foreach ($idx in $indexes) { $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions } @@ -469,7 +469,7 @@ function GetSparseMatrixIndexes([Array]$dimensions) $indexes += ,$idx } - return $indexes + return ,$indexes } function GenerateFullMatrix( diff --git a/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 b/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 index 676968f9b7a6..f8fcc527dba5 100644 --- a/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 +++ b/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 @@ -129,6 +129,39 @@ Describe "Platform Matrix Import" -Tag "import" { CompareMatrices $matrix $expected } + It "Should import a matrix and combine with length=1 vectors" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "TestField1": "test1", + "TestField2": "test2" + }, + "exclude": [ { "Baz": "importedBaz" } ] +} +'@ + + $expectedMatrix = @' +[ + { + "parameters": { "TestField1": "test1", "TestField2": "test2", "Foo": "foo1", "Bar": "bar1" }, + "name": "test1_test2_foo1_bar1" + }, + { + "parameters": { "TestField1": "test1", "TestField2": "test2", "Foo": "foo2", "Bar": "bar2" }, + "name": "test1_test2_foo2_bar2" + } +] +'@ + + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" + $expected = $expectedMatrix | ConvertFrom-Json -AsHashtable + + $matrix.Length | Should -Be 2 + CompareMatrices $matrix $expected + } + It "Should generate a matrix with nonSparseParameters and an imported sparse matrix" { $matrixJson = @' { From 96da2325ed4276876d0e77de3ae917c70998eca4 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 26 Mar 2021 12:04:36 -0400 Subject: [PATCH 2/2] Pass NonSparseParameters to imported matrix generation --- eng/common/scripts/job-matrix/README.md | 4 ++ .../job-matrix/job-matrix-functions.ps1 | 10 +++-- ...ob-matrix-functions.modification.tests.ps1 | 40 +++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/eng/common/scripts/job-matrix/README.md b/eng/common/scripts/job-matrix/README.md index 13bf6cf43990..e4d0464a941f 100644 --- a/eng/common/scripts/job-matrix/README.md +++ b/eng/common/scripts/job-matrix/README.md @@ -171,6 +171,8 @@ Importing can be useful, for example, in cases where there is a shared base matr once for each instance of a language version. Importing does not support overriding duplicate parameters. To achieve this, use the [Replace](#replace-values) argument instead. +The `Selection` and `NonSparseParameters` parameters are respected when generating an imported matrix. + The processing order is as follows: Given a matrix and import matrix like below: @@ -494,6 +496,8 @@ Given a matrix like below with `JavaTestVersion` marked as a non-sparse paramete A matrix with 6 entries will be generated: A sparse matrix of Agent, AZURE_TEST_HTTP_CLIENTS and ArmTemplateParameters (3 total entries) will be multipled by the two `JavaTestVersion` parameters `1.8` and `1.11`. +NOTE: NonSparseParameters are also applied when generating an imported matrix. + #### Under the hood The script generates an N-dimensional matrix with dimensions equal to the parameter array lengths. For example, diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index dc7e19684abd..25ff756950d3 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -94,7 +94,8 @@ function GenerateMatrix( [Array]$replace = @(), [Array]$nonSparseParameters = @() ) { - $matrixParameters, $importedMatrix, $combinedDisplayNameLookup = ProcessImport $config.matrixParameters $selectFromMatrixType $config.displayNamesLookup + $matrixParameters, $importedMatrix, $combinedDisplayNameLookup = ` + ProcessImport $config.matrixParameters $selectFromMatrixType $nonSparseParameters $config.displayNamesLookup if ($selectFromMatrixType -eq "sparse") { $matrix = GenerateSparseMatrix $matrixParameters $config.displayNamesLookup $nonSparseParameters } elseif ($selectFromMatrixType -eq "all") { @@ -340,7 +341,7 @@ function ProcessReplace return $replaceMatrix } -function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Hashtable]$displayNamesLookup) +function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$nonSparseParameters, [Hashtable]$displayNamesLookup) { $importPath = "" $matrix = $matrix | ForEach-Object { @@ -355,7 +356,10 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Hashtabl } $importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath) - $importedMatrix = GenerateMatrix $importedMatrixConfig $selection + $importedMatrix = GenerateMatrix ` + -config $importedMatrixConfig ` + -selectFromMatrixType $selection ` + -nonSparseParameters $nonSparseParameters $combinedDisplayNameLookup = $importedMatrixConfig.displayNamesLookup foreach ($lookup in $displayNamesLookup.GetEnumerator()) { diff --git a/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 b/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 index f8fcc527dba5..08979caaaf54 100644 --- a/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 +++ b/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 @@ -83,6 +83,46 @@ Describe "Platform Matrix nonSparse" -Tag "nonsparse" { $matrix = GenerateMatrix $config "sparse" -nonSparseParameters "testField3","testField4" $matrix.Length | Should -Be 8 } + + It "Should apply nonSparseParameters to an imported matrix" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "TestField1": "test1" + }, + "exclude": [ { "Baz": "importedBaz" } ] +} +'@ + + $expectedMatrix = @' +[ + { + "parameters": { "TestField1": "test1", "Foo": "foo1", "Bar": "bar1" }, + "name": "test1_foo1_bar1" + }, + { + "parameters": { "TestField1": "test1", "Foo": "foo1", "Bar": "bar2" }, + "name": "test1_foo1_bar2" + }, + { + "parameters": { "TestField1": "test1", "Foo": "foo2", "Bar": "bar1" }, + "name": "test1_foo2_bar1" + }, + { + "parameters": { "TestField1": "test1", "Foo": "foo2", "Bar": "bar2" }, + "name": "test1_foo2_bar2" + } +] +'@ + + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" -nonSparseParameters "Foo" + $expected = $expectedMatrix | ConvertFrom-Json -AsHashtable + + $matrix.Length | Should -Be 4 + CompareMatrices $matrix $expected + } } Describe "Platform Matrix Import" -Tag "import" {