diff --git a/.pipeline/e2e-test-pipeline.yml b/.pipeline/e2e-test-pipeline.yml new file mode 100644 index 0000000..925481b --- /dev/null +++ b/.pipeline/e2e-test-pipeline.yml @@ -0,0 +1,35 @@ +# E2E pipeline for AKS Iot + +trigger: none + +resources: + repositories: + - repository: AksEdge + type: github + endpoint: AksEdge-GithubConnection + name: Azure/AKS-Edge + ref: $(Build.SourceBranch) + + +# Reference the MSI pipelines + +parameters: +- name: TestGroup + displayName: Test Group to execute + type: string + default: LinuxOffline + values: + - LinuxOffline + - LinuxOnline + - LinuxAndWindows + - All +jobs: + - template: templates/matrix-generator.yml + parameters: + TestGroup: ${{ parameters.TestGroup }} + - ${{ if or( eq(parameters.TestGroup, 'All'), eq(parameters.TestGroup, 'LinuxOffline')) }}: + - template: templates/e2e-linuxoffline.yml + - ${{ if or( eq(parameters.TestGroup, 'All'), eq(parameters.TestGroup, 'LinuxOnline')) }}: + - template: templates/e2e-linuxonline.yml + - ${{ if or( eq(parameters.TestGroup, 'All'), eq(parameters.TestGroup, 'LinuxAndWindows')) }}: + - template: templates/e2e-linuxandwindows.yml diff --git a/.pipeline/templates/e2e-linuxandwindows.yml b/.pipeline/templates/e2e-linuxandwindows.yml new file mode 100644 index 0000000..fb592ad --- /dev/null +++ b/.pipeline/templates/e2e-linuxandwindows.yml @@ -0,0 +1,103 @@ +jobs: + - job: aidescript_e2e_test_linuxandwindows + dependsOn: + - mtrxgenerator + displayName: Run E2E Linux And Windows Offline Test Suite + pool: 1es-aksiot-windows-x64-ltsc-2021-test-pool + timeoutInMinutes: 60 + condition: eq(dependencies.mtrxgenerator.outputs['mtrx.LinuxAndWindows'], 'true') + + steps: + - checkout: AksEdge + path: self + clean: true + fetchDepth: 1 + + - powershell: | + $CheckInstaller = Get-WmiObject -Class Win32_Product | where Name -match "AKS Edge Essentials - (K8s|K3s)" + $Module = Get-Module -ListAvailable -Name AksEdge + if ($CheckInstaller -ne $Null -Or $Module -ne $Null) + { + Write-Error "AksEdge is already installed on this agent" + Exit 1 + } + Write-Host "AksEdge is not installed on this agent" + + $freememInMB = ((Get-CimInstance -Class Win32_OperatingSystem).FreePhysicalMemory / 1024) + $freememInMBRounded = [Math]::Round($freememInMB) + if ($freememInMbRounded -lt 4096) + { + Write-Error "The host does not have enough resources to install and run AksEdge" + Exit 1 + } + Write-Host "The host has $freememInMBRounded free memory, AksEdge can be installed on it" + + $PSConfiguration = Get-ExecutionPolicy + if ($PSConfiguration -ne "Bypass" -and $PSConfiguration -ne "Unrestricted") + { + Write-Error "The host current powershell configuration is $PSConfiguration, expected configuration is Bypass or Unrestricted" + Exit 1 + } + Write-Host "The host current powershell configuration is $PSConfiguration" + + $SSHCheck = (Get-WindowsCapability -Online | Where-Object Name -like 'OpenSSH.Client*').State + $HyperVHyperVisor = (Get-WindowsOptionalFeature -FeatureName Microsoft-Hyper-V-Hypervisor -Online).State + $HyperV = (Get-WindowsOptionalFeature -FeatureName Microsoft-Hyper-V -Online).State + $HyperVMngPowershell = (Get-WindowsOptionalFeature -FeatureName Microsoft-Hyper-V-Management-PowerShell -Online).State + if ($SSHCheck -ne "Installed" -Or $HyperVHyperVisor -ne "Enabled" -Or $HyperV -ne "Enabled" -Or $HyperVMngPowershell -ne "Enabled") + { + Write-Error "Not all software modules are installed, the software modules that are needed are: OpenSSH.Client, Microsoft-Hyper-V-Hypervisor, Microsoft-Hyper-V, Microsoft-Hyper-V-Management-PowerShell." + Exit 1 + } + Write-Host "All the software modules are installed: OpenSSH.Client, Microsoft-Hyper-V-Hypervisor, Microsoft-Hyper-V, Microsoft-Hyper-V-Management-PowerShell." + displayName: 'Validating agent state' + + - powershell: | + Install-PackageProvider -Name NuGet -MinimumVersion '2.8.5.201' -Force + $modules = Get-Module -ListAvailable + Write-Host "Modules available are" + Write-Host $modules + $psgallery = Get-PSRepository | Where-Object { $_.Name -like "PSGallery" } + if ($psgallery.InstallationPolicy -ine "Trusted") { + # Do this always as by default PSGallery is untrusted. + # See alternate means to force install rather than making this trusted. + Write-Host "Setting PSGallery as Trusted Source" + Set-PSRepository -Name "PSGallery" -InstallationPolicy Trusted + } + else { Write-Host "PSGallery is trusted" -ForegroundColor Green } + displayName: 'Enabling PS Reposiotory access to install required module' + + - powershell: | + $LogFile = 'e2e_junit.xml' + $TestPath = "$(Agent.BuildDirectory)\self\tests" + $WindowsConfig = [PSCustomObject]@{ + "CpuCount"= 4 + "MemoryInMB"= 4096 + "DataSizeInGB"= 20 + } + + $AksEdgeConfigFilePath = "$(Agent.BuildDirectory)\self\tools\aksedge-config.json" + $AksEdgeConfig = Get-Content -Raw $AksEdgeConfigFilePath | ConvertFrom-Json + Add-Member -InputObject $AksEdgeConfig.Machines[0] -MemberType NoteProperty -Name WindowsNode -Value $WindowsConfig + $AksEdgeConfig | ConvertTo-Json | Out-File $AksEdgeConfigFilePath + + $JsonConfigFilePath = "$(Agent.BuildDirectory)\self\tools\aide-userconfig.json" + + $JsonTestParameters = Get-Content -Raw $JsonConfigFilePath | ConvertFrom-Json + $JsonTestParameters.AksEdgeConfigFile = $AksEdgeConfigFilePath + + $JsonTestParameters | ConvertTo-Json -Depth 4 | Out-File $JsonConfigFilePath + + Write-Host "Executing tests from $TestPath, Results at $LogFile" + Get-ChildItem -Path $TestPath + & c:\windows\system32\windowspowershell\v1.0\powershell.exe -File "$TestPath\e2e.ps1" -LogFile $LogFile -JsonConfigFilePath $JsonConfigFilePath -IncludeGroup BasicLinuxAndWindowsNodeOffline + Copy-Item -Path $LogFile -Destination "$(Build.ArtifactStagingDirectory)\e2e_junit.xml" + Copy-Item -Path $TestPath\Logs-*.zip -Destination "$(Build.ArtifactStagingDirectory)\$E2E-Logs.zip" + displayName: 'Run e2e.ps1' + + - task: PublishTestResults@2 + inputs: + testResultsFormat: JUnit + testResultsFiles: '$(Build.ArtifactStagingDirectory)/e2e_junit.xml' + testRunTitle: 'AideScripts E2E Test Pass' + displayName: 'Publish E2E Test Results' \ No newline at end of file diff --git a/.pipeline/templates/e2e-linuxoffline.yml b/.pipeline/templates/e2e-linuxoffline.yml new file mode 100644 index 0000000..71becb7 --- /dev/null +++ b/.pipeline/templates/e2e-linuxoffline.yml @@ -0,0 +1,93 @@ +jobs: + - job: aidescript_e2e_test_linuxoffline + dependsOn: + - mtrxgenerator + displayName: Run E2E Linux Offline Test Suite + pool: 1es-aksiot-windows-x64-ltsc-2021-test-pool + timeoutInMinutes: 60 + condition: eq(dependencies.mtrxgenerator.outputs['mtrx.LinuxNodeOffline'], 'true') + + + steps: + - checkout: AksEdge + path: self + clean: true + fetchDepth: 1 + + - powershell: | + $CheckInstaller = Get-WmiObject -Class Win32_Product | where Name -match "AKS Edge Essentials - (K8s|K3s)" + $Module = Get-Module -ListAvailable -Name AksEdge + if ($CheckInstaller -ne $Null -Or $Module -ne $Null) + { + Write-Error "AksEdge is already installed on this agent" + Exit 1 + } + Write-Host "AksEdge is not installed on this agent" + + $freememInMB = ((Get-CimInstance -Class Win32_OperatingSystem).FreePhysicalMemory / 1024) + $freememInMBRounded = [Math]::Round($freememInMB) + if ($freememInMbRounded -lt 4096) + { + Write-Error "The host does not have enough resources to install and run AksEdge" + Exit 1 + } + Write-Host "The host has $freememInMBRounded free memory, AksEdge can be installed on it" + + $PSConfiguration = Get-ExecutionPolicy + if ($PSConfiguration -ne "Bypass" -and $PSConfiguration -ne "Unrestricted") + { + Write-Error "The host current powershell configuration is $PSConfiguration, expected configuration is Bypass or Unrestricted" + Exit 1 + } + Write-Host "The host current powershell configuration is $PSConfiguration" + + $SSHCheck = (Get-WindowsCapability -Online | Where-Object Name -like 'OpenSSH.Client*').State + $HyperVHyperVisor = (Get-WindowsOptionalFeature -FeatureName Microsoft-Hyper-V-Hypervisor -Online).State + $HyperV = (Get-WindowsOptionalFeature -FeatureName Microsoft-Hyper-V -Online).State + $HyperVMngPowershell = (Get-WindowsOptionalFeature -FeatureName Microsoft-Hyper-V-Management-PowerShell -Online).State + if ($SSHCheck -ne "Installed" -Or $HyperVHyperVisor -ne "Enabled" -Or $HyperV -ne "Enabled" -Or $HyperVMngPowershell -ne "Enabled") + { + Write-Error "Not all software modules are installed, the software modules that are needed are: OpenSSH.Client, Microsoft-Hyper-V-Hypervisor, Microsoft-Hyper-V, Microsoft-Hyper-V-Management-PowerShell." + Exit 1 + } + Write-Host "All the software modules are installed: OpenSSH.Client, Microsoft-Hyper-V-Hypervisor, Microsoft-Hyper-V, Microsoft-Hyper-V-Management-PowerShell." + displayName: 'Validating agent state' + + - powershell: | + Install-PackageProvider -Name NuGet -MinimumVersion '2.8.5.201' -Force + $modules = Get-Module -ListAvailable + Write-Host "Modules available are" + Write-Host $modules + $psgallery = Get-PSRepository | Where-Object { $_.Name -like "PSGallery" } + if ($psgallery.InstallationPolicy -ine "Trusted") { + # Do this always as by default PSGallery is untrusted. + # See alternate means to force install rather than making this trusted. + Write-Host "Setting PSGallery as Trusted Source" + Set-PSRepository -Name "PSGallery" -InstallationPolicy Trusted + } + else { Write-Host "PSGallery is trusted" -ForegroundColor Green } + displayName: 'Enabling PS Reposiotory access to install required module' + + - powershell: | + $LogFile = 'e2e_junit.xml' + $TestPath = "$(Agent.BuildDirectory)\self\tests" + + $JsonConfigFilePath = "$(Agent.BuildDirectory)\self\tools\aide-userconfig.json" + $JsonTestParameters = Get-Content -Raw $JsonConfigFilePath | ConvertFrom-Json + $JsonTestParameters.AksEdgeConfigFile = "$(Agent.BuildDirectory)\self\tools\aksedge-config.json" + + $JsonTestParameters | ConvertTo-Json | Out-File $JsonConfigFilePath + + Write-Host "Executing tests from $TestPath, Results at $LogFile" + Get-ChildItem -Path $TestPath + & c:\windows\system32\windowspowershell\v1.0\powershell.exe -File "$TestPath\e2e.ps1" -LogFile $LogFile -JsonConfigFilePath $JsonConfigFilePath -IncludeGroup BasicLinuxNodeOffline + Copy-Item -Path $LogFile -Destination "$(Build.ArtifactStagingDirectory)\e2e_junit.xml" + Copy-Item -Path $TestPath\Logs-*.zip -Destination "$(Build.ArtifactStagingDirectory)\$E2E-Logs.zip" + displayName: 'Run e2e.ps1' + + - task: PublishTestResults@2 + inputs: + testResultsFormat: JUnit + testResultsFiles: '$(Build.ArtifactStagingDirectory)/e2e_junit.xml' + testRunTitle: 'AideScripts E2E Test Pass' + displayName: 'Publish E2E Test Results' \ No newline at end of file diff --git a/azure-pipelines.yml b/.pipeline/templates/e2e-linuxonline.yml similarity index 89% rename from azure-pipelines.yml rename to .pipeline/templates/e2e-linuxonline.yml index 2cdcab1..c5ff158 100644 --- a/azure-pipelines.yml +++ b/.pipeline/templates/e2e-linuxonline.yml @@ -1,18 +1,12 @@ -trigger: none - -resources: - repositories: - - repository: AksEdge - type: github - endpoint: AksEdge-GithubConnection - name: Azure/AKS-Edge - ref: $(Build.SourceBranch) - jobs: - - job: aidescript_e2e_test - displayName: Run E2E Aide Test Script + - job: aidescript_e2e_test_linuxonline + displayName: Run E2E Linux Online Test Suite + dependsOn: + - mtrxgenerator pool: 1es-aksiot-windows-x64-ltsc-2021-test-pool timeoutInMinutes: 60 + condition: eq(dependencies.mtrxgenerator.outputs['mtrx.LinuxNodeOnline'], 'true') + steps: - checkout: AksEdge @@ -91,15 +85,16 @@ jobs: } 'CustomLocationOID' = '51dfe1e8-70c6-4de5-a08e-e18aff23d815' } - $JsonTestParameters = Get-Content -Raw "$(Agent.BuildDirectory)\self\tools\aide-userconfig.json" | ConvertFrom-Json + $JsonConfigFilePath = "$(Agent.BuildDirectory)\self\tools\aide-userconfig.json" + $JsonTestParameters = Get-Content -Raw $JsonConfigFilePath | ConvertFrom-Json $JsonTestParameters.AksEdgeConfigFile = "$(Agent.BuildDirectory)\self\tools\aksedge-config.json" $JsonTestParameters.Azure = New-Object psobject -Property $AzureConfig - $JsonTestParameters | ConvertTo-Json | Out-File "$(Agent.BuildDirectory)\self\tools\aide-userconfig.json" + $JsonTestParameters | ConvertTo-Json | Out-File "$JsonConfigFilePath" Write-Host "Executing tests from $TestPath, Results at $LogFile" Get-ChildItem -Path $TestPath - & c:\windows\system32\windowspowershell\v1.0\powershell.exe -File "$TestPath\e2e.ps1" -All -LogFile $LogFile + & c:\windows\system32\windowspowershell\v1.0\powershell.exe -File "$TestPath\e2e.ps1" -LogFile $LogFile -JsonConfigFilePath $JsonConfigFilePath -IncludeGroup BasicLinuxNodeOnline Copy-Item -Path $LogFile -Destination "$(Build.ArtifactStagingDirectory)\e2e_junit.xml" Copy-Item -Path $TestPath\Logs-*.zip -Destination "$(Build.ArtifactStagingDirectory)\$E2E-Logs.zip" displayName: 'Run e2e.ps1' diff --git a/.pipeline/templates/matrix-generator.yml b/.pipeline/templates/matrix-generator.yml new file mode 100644 index 0000000..724f7f1 --- /dev/null +++ b/.pipeline/templates/matrix-generator.yml @@ -0,0 +1,45 @@ +# Template for the matrix generator + +parameters: +- name: TestGroup + displayName: Test Group to execute + +jobs: +- job: mtrxgenerator + pool: + vmImage: 'ubuntu-latest' + steps: + - checkout: self + clean: true + fetchDepth: 1 + - bash: | + # Use the json file to get the values of params passed to the MSI pipeline + # Install jq + sudo apt-get update + sudo apt-get install jq -y + + LinuxNodeOnline='false' + LinuxNodeOffline='false' + LinuxAndWindows='false' + + if [ "${{ parameters.TestGroup }}" == "All" ]; then + LinuxNodeOnline='true' + LinuxNodeOffline='true' + LinuxAndWindows='true' + elif [ "${{ parameters.TestGroup }}" == "LinuxOffline" ]; then + LinuxNodeOffline='true' + elif [ "${{ parameters.TestGroup }}" == "LinuxOnline" ]; then + LinuxNodeOnline='true' + elif [ "${{ parameters.TestGroup }}" == "LinuxAndWindows" ]; then + LinuxAndWindows='true' + fi + + echo "LinuxOffline: $LinuxNodeOffline" + echo "LinuxOnline: $LinuxNodeOnline" + echo "LinuxAndWindows: $LinuxAndWindows" + + echo "##vso[task.setVariable variable=LinuxNodeOffline;isOutput=true]$LinuxNodeOffline" + echo "##vso[task.setVariable variable=LinuxNodeOnline;isOutput=true]$LinuxNodeOnline" + echo "##vso[task.setVariable variable=LinuxAndWindows;isOutput=true]$LinuxAndWindows" + + name: mtrx diff --git a/tests/E2E/e2e_basiclinuxandwindowsoffline_test.ps1 b/tests/E2E/e2e_basiclinuxandwindowsoffline_test.ps1 new file mode 100644 index 0000000..c4fc020 --- /dev/null +++ b/tests/E2E/e2e_basiclinuxandwindowsoffline_test.ps1 @@ -0,0 +1,138 @@ + +Import-Module "$PSScriptRoot\utils.ps1" + +function Setup-BasicLinuxAndWindowsNodeOffline { + param( + # Test Parameters + [String] + $JsonTestParameters, + + [HashTable] + # Optional parameters from the commandline + $TestVar + ) + + # Get Aide UserConfig + + $retval = Start-AideWorkflow -jsonString $JsonTestParameters + + if($retval) { + Write-Host "Deployment Successful" + } else { + throw "Deployment Failed" + } +} + +function Cleanup-BasicLinuxAndWindowsNodeOffline { + param( + # Test Parameters + [String] + $JsonTestParameters, + + [HashTable] + # Optional parameters from the commandline + $TestVar + ) + + $retval = Remove-AideDeployment + + if($retval) { + Write-Host "Cleanup Successful" + } else { + throw "Cleanup Failed" + } +} + +function E2etest-BasicLinuxAndWindowsNodeOffline-TestOfflineClusterNodesReady { + param( + + # Test Parameters + [String] + $JsonTestParameters, + + [HashTable] + # Optional parameters from the commandline + $TestVar + ) + + Write-Host "Running kubectl on node" + + # Assuming the cluster is ready after this is done, let's prove whether it's good or not + Get-AksEdgeKubeConfig -Confirm:$false + + $output = & 'c:\program files\AksEdge\kubectl\kubectl.exe' get nodes + Assert-Equal $LastExitCode 0 + Write-Host "kubectl output:`n$output" + + $result = $($output -split '\r?\n' -replace '\s+', ';' | ConvertFrom-Csv -Delimiter ';') + + Write-Host "Kubernetes nodes STATUS:" + foreach ( $NODE in $result ) + { + Write-Host "NAME: $($NODE.NAME) STATUS: $($NODE.STATUS)" + } + foreach ( $NODE in $result ) + { + Assert-Equal $NODE.STATUS 'Ready' + } +} + +function E2etest-BasicLinuxAndWindowsNodeOffline-TestOfflineClusterPodsReady +{ + param + ( + [String] + # Test Parameter + $JsonTestParameters, + + [HashTable] + # Optional parameters from the commandline + $TestVar + ) + + Write-Host "Running kubectl" + + Get-AksEdgeKubeConfig -Confirm:$false + $kubectloutput = & 'c:\program files\AksEdge\kubectl\kubectl.exe' get pods --all-namespaces + $result = $($kubectloutput -split '\r?\n' -replace '\s+', ';' | ConvertFrom-Csv -Delimiter ';') + Write-Host "`n Kube pods output: $kubectloutput" + + Write-Host "Kubernetes pods STATUS:" + foreach ( $POD in $result ) + { + Write-Host "NAME: $($POD.NAME) READY: $($POD.READY) STATUS: $($POD.STATUS)" + } + + # Verify if we get any pods output from kubectl + $condition = [string]::IsNullOrEmpty($result.NAME) + Assert-Equal $condition $false + + foreach ( $POD in $result ) + { + # Verify if all the pods are Ready + $ReadyValues = $POD.READY.Split("/") + Assert-Equal $ReadyValues[0] $ReadyValues[1] + } +} + +function E2eTest-BasicLinuxAndWindowsNodeOffline-WindowsVmIp4Address +{ + param + ( + [String] + # Test Parameter + $JsonTestParameters, + + [HashTable] + # Optional parameters from the commandline + $TestVar + ) + $windowsVmIp = Get-WindowsVmIpAddress + + $output = Invoke-WindowsSSH "powershell.exe -command `"(Get-NetIPAddress).IpAddress`"" + $result = $output.Contains($windowsVmIp) + Assert-Equal $result $True + Write-Host "VM IPV4 ip address is: `n$windowsVmIp" +} + + diff --git a/tests/E2E/utils.ps1 b/tests/E2E/utils.ps1 new file mode 100644 index 0000000..501f865 --- /dev/null +++ b/tests/E2E/utils.ps1 @@ -0,0 +1,55 @@ +function Get-WindowsVmIpAddress +{ + $env:WSSD_CONFIG_PATH="c:\programdata\aksedge\protected\.wssd\cloudconfig" + $WindowsVmTag="eb28e4f7-6522-4c33-a531-cfedf24b08e6" + $IdLine = & 'C:\Program Files\aksedge\nodectl' network vnic list --query "[?tags.keys(@).contains(@,'$WindowsVmTag')]" | Select-String -Pattern "ipaddress:" + $vmIp = ($IdLine -split ":")[1].Trim() + + return $vmIP +} + +function Invoke-WindowsSSH +{ + param ( + [Parameter(Mandatory)] + [String] $command + ) + + $vmIP = Get-WindowsVmIpAddress + + try + { + $sshPrivKey = New-SshPrivateKey + + & ssh.exe -o LogLevel=ERROR -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectionAttempts=10 -o ConnectTimeout=30 -o PasswordAuthentication=no -i "$sshPrivKey" "aksedge-user@$vmIp" $command + } + finally + { + Remove-Item -Path $sshPrivKey -Force -ErrorAction SilentlyContinue + } +} + +function New-SshPrivateKey +{ + + $SshPrivateKey = $([io.Path]::Combine("C:\ProgramData\AksEdge\protected\.sshkey", "id_ecdsa")) + if(!(Test-Path -Path $SshPrivateKey)) + { + Throw $("'$SshPrivateKey' is not found") + } + + $TempFile = New-TemporaryFile + Copy-Item $SshPrivateKey $TempFile + + $CurrentUser = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name + $NewOwner = New-Object -TypeName System.Security.Principal.NTAccount -ArgumentList $CurrentUser + + $acl = Get-Acl $TempFile + $acl.SetOwner($NewOwner) + $acl.SetAccessRuleProtection($true, $false) + $rule = New-Object System.Security.AccessControl.FileSystemAccessRule($NewOwner, "FullControl", "allow") + $acl.AddAccessRule($rule) + $acl | Set-Acl + + return $TempFile +} diff --git a/tests/e2e.ps1 b/tests/e2e.ps1 index b42bed1..7d24d1d 100644 --- a/tests/e2e.ps1 +++ b/tests/e2e.ps1 @@ -34,6 +34,10 @@ param( # For the test run, run all tests except those matching these tests $ExcludeTest, + [String] + #Path of the JSON Configuration file + $JsonConfigFilePath, + [HashTable] # Pass variables into tests $TestVar, @@ -78,7 +82,7 @@ function Assert-Equal $AideModulePath = "$PSScriptRoot\..\tools" -$JsonTestParameters = Get-Content -Raw $AideModulePath\aide-userconfig.json +$JsonTestParameters = Get-Content -Raw $JsonConfigFilePath $aksedgeShell = (Get-ChildItem -Path "$AideModulePath" -Filter AksEdgeShell.ps1 -Recurse).FullName . $aksedgeShell @@ -95,6 +99,7 @@ $aksedgeShell = (Get-ChildItem -Path "$AideModulePath" -Filter AksEdgeShell.ps1 . "$PSScriptRoot\E2E\e2e_basiclinuxoffline_test.ps1" . "$PSScriptRoot\E2E\e2e_basiclinuxonline_test.ps1" +. "$PSScriptRoot\E2E\e2e_basiclinuxandwindowsoffline_test.ps1" # Put all commands into $CommandTree, which is a map(GroupName => Array(FunctionNames)) $AllCommands = Get-Command -Verb 'E2eTest'