From c05116ea3477b354fd8fb004507361f55faa5a29 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Fri, 9 Jul 2021 10:43:58 -0700 Subject: [PATCH 1/3] Update tests to only check extensionconfig creation --- .../test/extensions/public/AzureDefender.Tests.ps1 | 2 +- .../extensions/public/AzureMLKubernetes.Tests.ps1 | 6 +++--- testing/test/extensions/public/AzureMonitor.Tests.ps1 | 6 ++---- .../test/extensions/public/OpenServiceMesh.Tests.ps1 | 6 ++---- testing/test/helper/Helper.ps1 | 11 +++++++++++ 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/testing/test/extensions/public/AzureDefender.Tests.ps1 b/testing/test/extensions/public/AzureDefender.Tests.ps1 index 1ef498b67e6..e60d443f620 100644 --- a/testing/test/extensions/public/AzureDefender.Tests.ps1 +++ b/testing/test/extensions/public/AzureDefender.Tests.ps1 @@ -23,7 +23,7 @@ Describe 'Azure Defender Testing' { do { # Only check the extension config, not the pod since this doesn't bring up pods - if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) { + if (Has-ExtensionData $extensionName) { break } Start-Sleep -Seconds 10 diff --git a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 index 20b6a802b73..44598499166 100644 --- a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 +++ b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 @@ -24,7 +24,7 @@ Describe 'AzureML Kubernetes Testing' { $n = 0 do { - if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) { + if (Has-ExtensionData $extensionName) { break } Start-Sleep -Seconds 20 @@ -120,7 +120,7 @@ Describe 'AzureML Kubernetes Testing' { $n = 0 do { - if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) { + if (Has-ExtensionData $extensionName) { break } Start-Sleep -Seconds 20 @@ -167,7 +167,7 @@ Describe 'AzureML Kubernetes Testing' { $n = 0 do { - if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) { + if (Has-ExtensionData $extensionName) { break } Start-Sleep -Seconds 20 diff --git a/testing/test/extensions/public/AzureMonitor.Tests.ps1 b/testing/test/extensions/public/AzureMonitor.Tests.ps1 index a34d8160728..a78ec6ba980 100644 --- a/testing/test/extensions/public/AzureMonitor.Tests.ps1 +++ b/testing/test/extensions/public/AzureMonitor.Tests.ps1 @@ -23,10 +23,8 @@ Describe 'Azure Monitor Testing' { $n = 0 do { - if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) { - if (Get-PodStatus $extensionAgentName -Namespace $extensionAgentNamespace -eq $POD_RUNNING) { - break - } + if (Has-ExtensionData $extensionName) { + break } Start-Sleep -Seconds 10 $n += 1 diff --git a/testing/test/extensions/public/OpenServiceMesh.Tests.ps1 b/testing/test/extensions/public/OpenServiceMesh.Tests.ps1 index 893d3c3e468..a637ee48302 100644 --- a/testing/test/extensions/public/OpenServiceMesh.Tests.ps1 +++ b/testing/test/extensions/public/OpenServiceMesh.Tests.ps1 @@ -27,10 +27,8 @@ Describe 'Azure OpenServiceMesh Testing' { $n = 0 do { - if (Get-ExtensionStatus $extensionName -eq $SUCCESS_MESSAGE) { - if (Get-PodStatus $extensionAgentName -Namespace $extensionAgentNamespace -eq $POD_RUNNING) { - break - } + if (Has-ExtensionData $extensionName) { + break } Start-Sleep -Seconds 10 $n += 1 diff --git a/testing/test/helper/Helper.ps1 b/testing/test/helper/Helper.ps1 index db76c41cff4..7bb11146ab2 100644 --- a/testing/test/helper/Helper.ps1 +++ b/testing/test/helper/Helper.ps1 @@ -7,6 +7,17 @@ function Get-ExtensionData { return $output.items | Where-Object { $_.metadata.name -eq $extensionName } } +function Has-ExtensionData { + param( + [string]$extensionName + ) + $extensionData = Get-ExtensionData $extensionName + if ($extensionData) { + return $true + } + return $false +} + function Get-ExtensionStatus { param( [string]$extensionName From 87310634c46a3741485cc858dc697c9df52013f8 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Fri, 9 Jul 2021 12:28:40 -0700 Subject: [PATCH 2/3] Single set of CRUD for AzureML --- .../public/AzureMLKubernetes.Tests.ps1 | 199 +++++++++--------- 1 file changed, 100 insertions(+), 99 deletions(-) diff --git a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 index 44598499166..4625ff0016a 100644 --- a/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 +++ b/testing/test/extensions/public/AzureMLKubernetes.Tests.ps1 @@ -10,8 +10,10 @@ Describe 'AzureML Kubernetes Testing' { . $PSScriptRoot/../../helper/Helper.ps1 } - It 'Creates the extension and checks that it onboards correctly with training enabled' { - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableTraining=true" -ErrorVariable badOut + It 'Creates the extension and checks that it onboards correctly with inference and SSL enabled' { + $sslKeyPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_key.pem" + $sslCertPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_cert.pem" + Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net inferenceLoadBalancerHA=False --config-protected sslKeyPemFile=$sslKeyPemFile sslCertPemFile=$sslCertPemFile" -ErrorVariable badOut $badOut | Should -BeNullOrEmpty $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut @@ -37,12 +39,6 @@ Describe 'AzureML Kubernetes Testing' { $relayResourceID | Should -Not -BeNullOrEmpty } - It "Performs a show on the extension" { - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty - $output | Should -Not -BeNullOrEmpty - } - It "Runs an update on the extension on the cluster" { Set-ItResult -Skipped -Because "Update is not a valid scenario for now" az k8s-extension update --cluster-name $ENVCONFIG.arcClusterName -g $ENVCONFIG.resourceGroup --cluster-type connectedClusters --name $extensionName --auto-upgrade-minor-version false @@ -70,6 +66,13 @@ Describe 'AzureML Kubernetes Testing' { $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS } + It "Performs a show on the extension" { + $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut + $badOut | Should -BeNullOrEmpty + $output | Should -Not -BeNullOrEmpty + } + + It "Lists the extensions on the cluster" { $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut $badOut | Should -BeNullOrEmpty @@ -79,7 +82,7 @@ Describe 'AzureML Kubernetes Testing' { $extensionExists | Should -Not -BeNullOrEmpty } - It "Deletes the extension from the cluster" { + It "Deletes the extension from the cluster with inference enabled" { # cleanup the relay and servicebus $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey $serviceBusResourceID = Get-ExtensionConfigurationSettings $extensionName $serviceBusResourceIDKey @@ -96,7 +99,7 @@ Describe 'AzureML Kubernetes Testing' { $badOut | Should -Not -BeNullOrEmpty $output | Should -BeNullOrEmpty } - + It "Performs another list after the delete" { $output = Invoke-Expression "az $Env:K8sExtensionName list -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters" -ErrorVariable badOut $badOut | Should -BeNullOrEmpty @@ -106,95 +109,93 @@ Describe 'AzureML Kubernetes Testing' { $extensionExists | Should -BeNullOrEmpty } - It 'Creates the extension and checks that it onboards correctly with inference enabled' { - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net allowInsecureConnections=True inferenceLoadBalancerHA=false" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty - - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty - - $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion - $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue - - # Loop and retry until the extension installs - $n = 0 - do - { - if (Has-ExtensionData $extensionName) { - break - } - Start-Sleep -Seconds 20 - $n += 1 - } while ($n -le $MAX_RETRY_ATTEMPTS) - $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + # It 'Creates the extension and checks that it onboards correctly with training enabled' { + # Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableTraining=true" -ErrorVariable badOut + # $badOut | Should -BeNullOrEmpty + + # $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut + # $badOut | Should -BeNullOrEmpty + + # $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion + # $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue + + # # Loop and retry until the extension installs + # $n = 0 + # do + # { + # if (Has-ExtensionData $extensionName) { + # break + # } + # Start-Sleep -Seconds 20 + # $n += 1 + # } while ($n -le $MAX_RETRY_ATTEMPTS) + # $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS - # check if relay is populated - $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey - $relayResourceID | Should -Not -BeNullOrEmpty - } - - It "Deletes the extension from the cluster with inference enabled" { - # cleanup the relay and servicebus - $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey - $serviceBusResourceID = Get-ExtensionConfigurationSettings $extensionName $serviceBusResourceIDKey - $relayNamespaceName = $relayResourceID.split("/")[8] - $serviceBusNamespaceName = $serviceBusResourceID.split("/")[8] - az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName - az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName - - $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty - - # Extension should not be found on the cluster - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -Not -BeNullOrEmpty - $output | Should -BeNullOrEmpty - } - - It 'Creates the extension and checks that it onboards correctly with inference and SSL enabled' { - $sslKeyPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_key.pem" - $sslCertPemFile = Join-Path (Join-Path (Join-Path (Split-Path $PSScriptRoot -Parent) "data") "azure_ml") "test_cert.pem" - Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net inferenceLoadBalancerHA=False --config-protected sslKeyPemFile=$sslKeyPemFile sslCertPemFile=$sslCertPemFile" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty - - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty - - $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion - $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue - - # Loop and retry until the extension installs - $n = 0 - do - { - if (Has-ExtensionData $extensionName) { - break - } - Start-Sleep -Seconds 20 - $n += 1 - } while ($n -le $MAX_RETRY_ATTEMPTS) - $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS + # # check if relay is populated + # $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey + # $relayResourceID | Should -Not -BeNullOrEmpty + # } + + # It "Deletes the extension from the cluster" { + # # cleanup the relay and servicebus + # $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey + # $serviceBusResourceID = Get-ExtensionConfigurationSettings $extensionName $serviceBusResourceIDKey + # $relayNamespaceName = $relayResourceID.split("/")[8] + # $serviceBusNamespaceName = $serviceBusResourceID.split("/")[8] + # az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName + # az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName + + # $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut + # $badOut | Should -BeNullOrEmpty + + # # Extension should not be found on the cluster + # $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut + # $badOut | Should -Not -BeNullOrEmpty + # $output | Should -BeNullOrEmpty + # } + + # It 'Creates the extension and checks that it onboards correctly with inference enabled' { + # Invoke-Expression "az $Env:K8sExtensionName create -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters --extension-type $extensionType -n $extensionName --release-train staging --config enableInference=true identity.proxy.remoteEnabled=True identity.proxy.remoteHost=https://master.experiments.azureml-test.net allowInsecureConnections=True inferenceLoadBalancerHA=false" -ErrorVariable badOut + # $badOut | Should -BeNullOrEmpty + + # $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut + # $badOut | Should -BeNullOrEmpty + + # $isAutoUpgradeMinorVersion = ($output | ConvertFrom-Json).autoUpgradeMinorVersion + # $isAutoUpgradeMinorVersion.ToString() -eq "True" | Should -BeTrue + + # # Loop and retry until the extension installs + # $n = 0 + # do + # { + # if (Has-ExtensionData $extensionName) { + # break + # } + # Start-Sleep -Seconds 20 + # $n += 1 + # } while ($n -le $MAX_RETRY_ATTEMPTS) + # $n | Should -BeLessOrEqual $MAX_RETRY_ATTEMPTS - # check if relay is populated - $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey - $relayResourceID | Should -Not -BeNullOrEmpty - } - - It "Deletes the extension from the cluster with inference enabled" { - # cleanup the relay and servicebus - $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey - $serviceBusResourceID = Get-ExtensionConfigurationSettings $extensionName $serviceBusResourceIDKey - $relayNamespaceName = $relayResourceID.split("/")[8] - $serviceBusNamespaceName = $serviceBusResourceID.split("/")[8] - az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName - az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName - - $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -BeNullOrEmpty - - # Extension should not be found on the cluster - $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut - $badOut | Should -Not -BeNullOrEmpty - $output | Should -BeNullOrEmpty - } + # # check if relay is populated + # $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey + # $relayResourceID | Should -Not -BeNullOrEmpty + # } + + # It "Deletes the extension from the cluster with inference enabled" { + # # cleanup the relay and servicebus + # $relayResourceID = Get-ExtensionConfigurationSettings $extensionName $relayResourceIDKey + # $serviceBusResourceID = Get-ExtensionConfigurationSettings $extensionName $serviceBusResourceIDKey + # $relayNamespaceName = $relayResourceID.split("/")[8] + # $serviceBusNamespaceName = $serviceBusResourceID.split("/")[8] + # az relay namespace delete --resource-group $ENVCONFIG.resourceGroup --name $relayNamespaceName + # az servicebus namespace delete --resource-group $ENVCONFIG.resourceGroup --name $serviceBusNamespaceName + + # $output = Invoke-Expression "az $Env:K8sExtensionName delete -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut + # $badOut | Should -BeNullOrEmpty + + # # Extension should not be found on the cluster + # $output = Invoke-Expression "az $Env:K8sExtensionName show -c $($ENVCONFIG.arcClusterName) -g $($ENVCONFIG.resourceGroup) --cluster-type connectedClusters -n $extensionName" -ErrorVariable badOut + # $badOut | Should -Not -BeNullOrEmpty + # $output | Should -BeNullOrEmpty + # } } From 6796a5e2241690bb2fea2fa07328365a2467c4a3 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Fri, 9 Jul 2021 15:42:35 -0700 Subject: [PATCH 3/3] Debug logs for connectedk8s --- testing/Cleanup.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testing/Cleanup.ps1 b/testing/Cleanup.ps1 index 9957a044241..dc1d1d3d908 100644 --- a/testing/Cleanup.ps1 +++ b/testing/Cleanup.ps1 @@ -12,6 +12,11 @@ az account set --subscription $ENVCONFIG.subscriptionId $Env:KUBECONFIG="$PSScriptRoot/tmp/KUBECONFIG" Write-Host "Removing the connectedk8s arc agents from the cluster..." az connectedk8s delete -g $ENVCONFIG.resourceGroup -n $ENVCONFIG.arcClusterName +if (!$?) +{ + kubectl get pods -A + Exit 1 +} # Skip deleting the AKS Cluster if this is CI if (!$CI) {