From f0b65b245c5d43c5249edfe0067e62eee5e316a5 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 4 Jun 2020 17:18:05 -0700 Subject: [PATCH 1/6] dualstack changes for windows vmss --- parts/k8s/kubeletstart.ps1 | 9 +++- parts/k8s/kubeproxystart.ps1 | 7 ++- parts/k8s/kuberneteswindowssetup.ps1 | 5 +- parts/k8s/windowsazurecnifunc.ps1 | 49 +++++++++++++++--- parts/k8s/windowsnodereset.ps1 | 2 + pkg/api/vlabs/validate.go | 4 +- pkg/engine/templates_generated.go | 73 +++++++++++++++++++++++---- pkg/engine/virtualmachinescalesets.go | 8 +-- 8 files changed, 131 insertions(+), 26 deletions(-) diff --git a/parts/k8s/kubeletstart.ps1 b/parts/k8s/kubeletstart.ps1 index fe2e949a68..2c39874e19 100644 --- a/parts/k8s/kubeletstart.ps1 +++ b/parts/k8s/kubeletstart.ps1 @@ -214,7 +214,14 @@ if ($global:NetworkPlugin -eq "azure") { if ((Test-Path $cnilock)) { Remove-Item $cnilock } - + $cnijson = [io.path]::Combine("$KubeDir", "azure-vnet-ipamv6.json") + if ((Test-Path $cnijson)) { + Remove-Item $cnijson + } + $cnilock = [io.path]::Combine("$KubeDir", "azure-vnet-ipamv6.json.lock") + if ((Test-Path $cnilock)) { + Remove-Item $cnilock + } $cnijson = [io.path]::Combine("$KubeDir", "azure-vnet.json") if ((Test-Path $cnijson)) { Remove-Item $cnijson diff --git a/parts/k8s/kubeproxystart.ps1 b/parts/k8s/kubeproxystart.ps1 index ab6d94c24e..6e6c4a778c 100644 --- a/parts/k8s/kubeproxystart.ps1 +++ b/parts/k8s/kubeproxystart.ps1 @@ -24,4 +24,9 @@ Import-Module $global:HNSModule # and https://github.com/kubernetes/kubernetes/pull/78612 for <= 1.15 Get-HnsPolicyList | Remove-HnsPolicyList -.$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --hostname-override=$env:computername --kubeconfig=$KubeDir\config +if (("--feature-gates=IPv6DualStack=true" | ? { $Global:ClusterConfiguration.Kubernetes.Kubelet.ConfigArgs -match $_ }) -ne $null) { + .$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --feature-gates=IPv6DualStack=true --hostname-override=$env:computername --kubeconfig=$KubeDir\config +} +else { + .$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --hostname-override=$env:computername --kubeconfig=$KubeDir\config +} \ No newline at end of file diff --git a/parts/k8s/kuberneteswindowssetup.ps1 b/parts/k8s/kuberneteswindowssetup.ps1 index c9eee84a2b..59c6f419e1 100644 --- a/parts/k8s/kuberneteswindowssetup.ps1 +++ b/parts/k8s/kuberneteswindowssetup.ps1 @@ -357,7 +357,8 @@ try -KubeServiceCIDR $global:KubeServiceCIDR ` -VNetCIDR $global:VNetCIDR ` {{- /* Azure Stack has discrete Azure CNI config requirements */}} - -IsAzureStack {{if IsAzureStackCloud}}$true{{else}}$false{{end}} + -IsAzureStack {{if IsAzureStackCloud}}$true{{else}}$false{{end}} ` + -IsDualStackEnabled {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` @@ -382,7 +383,7 @@ try } } - New-ExternalHnsNetwork + New-ExternalHnsNetwork -IsDualStackEnabled {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} Install-KubernetesServices ` -KubeDir $global:KubeDir diff --git a/parts/k8s/windowsazurecnifunc.ps1 b/parts/k8s/windowsazurecnifunc.ps1 index 2f2f77468b..ee4eff9e19 100644 --- a/parts/k8s/windowsazurecnifunc.ps1 +++ b/parts/k8s/windowsazurecnifunc.ps1 @@ -69,10 +69,18 @@ Set-AzureCNIConfig [Parameter(Mandatory=$true)][string] $VNetCIDR, [Parameter(Mandatory=$true)][bool] - $IsAzureStack + $IsAzureStack, + [Parameter(Mandatory=$true)][bool] + $IsDualStackEnabled ) # Fill in DNS information for kubernetes. - $exceptionAddresses = @($KubeClusterCIDR, $MasterSubnet, $VNetCIDR) + if ($IsDualStackEnabled){ + $subnetToPass = $KubeClusterCIDR -split "," + $exceptionAddresses = @($subnetToPass[0], $MasterSubnet, $VNetCIDR) + } + else { + $exceptionAddresses = @($KubeClusterCIDR, $MasterSubnet, $VNetCIDR) + } $fileName = [Io.path]::Combine("$AzureCNIConfDir", "10-azure.conflist") $configJson = Get-Content $fileName | ConvertFrom-Json @@ -93,7 +101,25 @@ Set-AzureCNIConfig $configJson.plugins.AdditionalArgs[0].Value.ExceptionList = $exceptionAddresses } - $configJson.plugins.AdditionalArgs[1].Value.DestinationPrefix = $KubeServiceCIDR + if ($IsDualStackEnabled){ + $configJson.plugins[0]|Add-Member -Name "ipv6Mode" -Value "ipv6nat" -MemberType NoteProperty + $serviceCidr = $KubeServiceCIDR -split "," + $configJson.plugins[0].AdditionalArgs[1].Value.DestinationPrefix = $serviceCidr[0] + $valueObj = [PSCustomObject]@{ + Type = 'ROUTE' + DestinationPrefix = $serviceCidr[1] + NeedEncap = $True + } + + $jsonContent = [PSCustomObject]@{ + Name = 'EndpointPolicy' + Value = $valueObj + } + $configJson.plugins[0].AdditionalArgs += $jsonContent + } + else { + $configJson.plugins[0].AdditionalArgs[1].Value.DestinationPrefix = $KubeServiceCIDR + } if ($IsAzureStack) { Add-Member -InputObject $configJson.plugins[0].ipam -MemberType NoteProperty -Name "environment" -Value "mas" @@ -255,7 +281,13 @@ function GenerateAzureStackCNIConfig Set-ItemProperty -Path $azureCNIConfigFile -Name IsReadOnly -Value $true } -function New-ExternalHnsNetwork { +function New-ExternalHnsNetwork +{ + param ( + [Parameter(Mandatory=$true)][bool] + $IsDualStackEnabled + ) + Write-Log "Creating new HNS network `"ext`"" $externalNetwork = "ext" $na = @(Get-NetAdapter -Physical) @@ -272,9 +304,14 @@ function New-ExternalHnsNetwork { $stopWatch = New-Object System.Diagnostics.Stopwatch $stopWatch.Start() - # Fixme : use a smallest range possible, that will not collide with any pod space - New-HNSNetwork -Type $global:NetworkMode -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -AdapterName $adapterName -Name $externalNetwork -Verbose + # Fixme : use a smallest range possible, that will not collide with any pod space + if ($IsDualStackEnabled) { + New-HNSNetwork -Type $global:NetworkMode -AddressPrefix @("192.168.255.0/30","192:168:255::0/127") -Gateway @("192.168.255.1","192:168:255::1") -AdapterName $adapterName -Name $externalNetwork -Verbose + } + else { + New-HNSNetwork -Type $global:NetworkMode -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -AdapterName $adapterName -Name $externalNetwork -Verbose + } # Wait for the switch to be created and the ip address to be assigned. for ($i = 0; $i -lt 60; $i++) { $mgmtIPAfterNetworkCreate = Get-NetIPAddress $managementIP -ErrorAction SilentlyContinue diff --git a/parts/k8s/windowsnodereset.ps1 b/parts/k8s/windowsnodereset.ps1 index a4170d53a4..91ecc1a614 100644 --- a/parts/k8s/windowsnodereset.ps1 +++ b/parts/k8s/windowsnodereset.ps1 @@ -66,6 +66,8 @@ if ($hnsNetwork) { "c:\k\azure-vnet.json.lock", "c:\k\azure-vnet-ipam.json", "c:\k\azure-vnet-ipam.json.lock" + "c:\k\azure-vnet-ipamv6.json", + "c:\k\azure-vnet-ipamv6.json.lock" ) foreach ($file in $filesToRemove) { diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 8a2c5cf758..df1e548a76 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -480,9 +480,9 @@ func (a *Properties) validateAgentPoolProfiles(isUpdate bool) error { } // validate os type is linux if dual stack feature is enabled - if a.FeatureFlags.IsIPv6DualStackEnabled() || a.FeatureFlags.IsIPv6OnlyEnabled() { + if a.FeatureFlags.IsIPv6OnlyEnabled() { if agentPoolProfile.OSType == Windows { - return errors.Errorf("Dual stack and single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfile.Name, agentPoolProfile.OSType) + return errors.Errorf("single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfile.Name, agentPoolProfile.OSType) } } diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index 6ce7f9c35e..4ff8da06c6 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -22175,7 +22175,14 @@ if ($global:NetworkPlugin -eq "azure") { if ((Test-Path $cnilock)) { Remove-Item $cnilock } - + $cnijson = [io.path]::Combine("$KubeDir", "azure-vnet-ipamv6.json") + if ((Test-Path $cnijson)) { + Remove-Item $cnijson + } + $cnilock = [io.path]::Combine("$KubeDir", "azure-vnet-ipamv6.json.lock") + if ((Test-Path $cnilock)) { + Remove-Item $cnilock + } $cnijson = [io.path]::Combine("$KubeDir", "azure-vnet.json") if ((Test-Path $cnijson)) { Remove-Item $cnijson @@ -22358,8 +22365,12 @@ Import-Module $global:HNSModule # and https://github.com/kubernetes/kubernetes/pull/78612 for <= 1.15 Get-HnsPolicyList | Remove-HnsPolicyList -.$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --hostname-override=$env:computername --kubeconfig=$KubeDir\config -`) +if (("--feature-gates=IPv6DualStack=true" | ? { $Global:ClusterConfiguration.Kubernetes.Kubelet.ConfigArgs -match $_ }) -ne $null) { + .$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --feature-gates=IPv6DualStack=true --hostname-override=$env:computername --kubeconfig=$KubeDir\config +} +else { + .$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --hostname-override=$env:computername --kubeconfig=$KubeDir\config +}`) func k8sKubeproxystartPs1Bytes() ([]byte, error) { return _k8sKubeproxystartPs1, nil @@ -23581,7 +23592,8 @@ try -KubeServiceCIDR $global:KubeServiceCIDR ` + "`" + ` -VNetCIDR $global:VNetCIDR ` + "`" + ` {{- /* Azure Stack has discrete Azure CNI config requirements */}} - -IsAzureStack {{if IsAzureStackCloud}}$true{{else}}$false{{end}} + -IsAzureStack {{if IsAzureStackCloud}}$true{{else}}$false{{end}} ` + "`" + ` + -IsDualStackEnabled {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` + "`" + ` @@ -23606,7 +23618,7 @@ try } } - New-ExternalHnsNetwork + New-ExternalHnsNetwork -IsDualStackEnabled {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} Install-KubernetesServices ` + "`" + ` -KubeDir $global:KubeDir @@ -24056,10 +24068,18 @@ Set-AzureCNIConfig [Parameter(Mandatory=$true)][string] $VNetCIDR, [Parameter(Mandatory=$true)][bool] - $IsAzureStack + $IsAzureStack, + [Parameter(Mandatory=$true)][bool] + $IsDualStackEnabled ) # Fill in DNS information for kubernetes. - $exceptionAddresses = @($KubeClusterCIDR, $MasterSubnet, $VNetCIDR) + if ($IsDualStackEnabled){ + $subnetToPass = $KubeClusterCIDR -split "," + $exceptionAddresses = @($subnetToPass[0], $MasterSubnet, $VNetCIDR) + } + else { + $exceptionAddresses = @($KubeClusterCIDR, $MasterSubnet, $VNetCIDR) + } $fileName = [Io.path]::Combine("$AzureCNIConfDir", "10-azure.conflist") $configJson = Get-Content $fileName | ConvertFrom-Json @@ -24080,7 +24100,25 @@ Set-AzureCNIConfig $configJson.plugins.AdditionalArgs[0].Value.ExceptionList = $exceptionAddresses } - $configJson.plugins.AdditionalArgs[1].Value.DestinationPrefix = $KubeServiceCIDR + if ($IsDualStackEnabled){ + $configJson.plugins[0]|Add-Member -Name "ipv6Mode" -Value "ipv6nat" -MemberType NoteProperty + $serviceCidr = $KubeServiceCIDR -split "," + $configJson.plugins[0].AdditionalArgs[1].Value.DestinationPrefix = $serviceCidr[0] + $valueObj = [PSCustomObject]@{ + Type = 'ROUTE' + DestinationPrefix = $serviceCidr[1] + NeedEncap = $True + } + + $jsonContent = [PSCustomObject]@{ + Name = 'EndpointPolicy' + Value = $valueObj + } + $configJson.plugins[0].AdditionalArgs += $jsonContent + } + else { + $configJson.plugins[0].AdditionalArgs[1].Value.DestinationPrefix = $KubeServiceCIDR + } if ($IsAzureStack) { Add-Member -InputObject $configJson.plugins[0].ipam -MemberType NoteProperty -Name "environment" -Value "mas" @@ -24242,7 +24280,13 @@ function GenerateAzureStackCNIConfig Set-ItemProperty -Path $azureCNIConfigFile -Name IsReadOnly -Value $true } -function New-ExternalHnsNetwork { +function New-ExternalHnsNetwork +{ + param ( + [Parameter(Mandatory=$true)][bool] + $IsDualStackEnabled + ) + Write-Log "Creating new HNS network ` + "`" + `"ext` + "`" + `"" $externalNetwork = "ext" $na = @(Get-NetAdapter -Physical) @@ -24259,9 +24303,14 @@ function New-ExternalHnsNetwork { $stopWatch = New-Object System.Diagnostics.Stopwatch $stopWatch.Start() - # Fixme : use a smallest range possible, that will not collide with any pod space - New-HNSNetwork -Type $global:NetworkMode -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -AdapterName $adapterName -Name $externalNetwork -Verbose + # Fixme : use a smallest range possible, that will not collide with any pod space + if ($IsDualStackEnabled) { + New-HNSNetwork -Type $global:NetworkMode -AddressPrefix @("192.168.255.0/30","192:168:255::0/127") -Gateway @("192.168.255.1","192:168:255::1") -AdapterName $adapterName -Name $externalNetwork -Verbose + } + else { + New-HNSNetwork -Type $global:NetworkMode -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -AdapterName $adapterName -Name $externalNetwork -Verbose + } # Wait for the switch to be created and the ip address to be assigned. for ($i = 0; $i -lt 60; $i++) { $mgmtIPAfterNetworkCreate = Get-NetIPAddress $managementIP -ErrorAction SilentlyContinue @@ -25401,6 +25450,8 @@ if ($hnsNetwork) { "c:\k\azure-vnet.json.lock", "c:\k\azure-vnet-ipam.json", "c:\k\azure-vnet-ipam.json.lock" + "c:\k\azure-vnet-ipamv6.json", + "c:\k\azure-vnet-ipamv6.json.lock" ) foreach ($file in $filesToRemove) { diff --git a/pkg/engine/virtualmachinescalesets.go b/pkg/engine/virtualmachinescalesets.go index cee8b15a24..65ac451815 100644 --- a/pkg/engine/virtualmachinescalesets.go +++ b/pkg/engine/virtualmachinescalesets.go @@ -489,7 +489,8 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi } var ipConfigurations []compute.VirtualMachineScaleSetIPConfiguration - + // multiple v6 configs are not supported. this flag is set after creating 1 v6 config. + var isV6ConfigCreated bool for i := 1; i <= profile.IPAddressCount; i++ { ipconfig := compute.VirtualMachineScaleSetIPConfiguration{ Name: to.StringPtr(fmt.Sprintf("ipconfig%d", i)), @@ -551,7 +552,7 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi ipconfig.VirtualMachineScaleSetIPConfigurationProperties = &ipConfigProps ipConfigurations = append(ipConfigurations, ipconfig) - if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") || cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only") { + if !isV6ConfigCreated && (cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") || cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only")) { ipconfigv6 := compute.VirtualMachineScaleSetIPConfiguration{ Name: to.StringPtr(fmt.Sprintf("ipconfig%dv6", i)), VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ @@ -563,6 +564,7 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi }, } ipConfigurations = append(ipConfigurations, ipconfigv6) + isV6ConfigCreated = true } } @@ -862,7 +864,7 @@ func associateAddonIdentitiesToVMSS(addonProfiles map[string]api.AddonProfile, v // Note: virtualMachineScaleSet.Identity is not nil and its type is None will NEVER happen in current AKS-Engine's implementation. if virtualMachineScaleSet.Identity == nil { virtualMachineScaleSet.Identity = &compute.VirtualMachineScaleSetIdentity{ - Type: compute.ResourceIdentityTypeUserAssigned, + Type: compute.ResourceIdentityTypeUserAssigned, UserAssignedIdentities: make(map[string]*compute.VirtualMachineScaleSetIdentityUserAssignedIdentitiesValue), } } else if virtualMachineScaleSet.Identity.Type == compute.ResourceIdentityTypeSystemAssigned { From eb5b8151cac9f2b306f40e40bb8ac3936e598996 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 4 Jun 2020 17:38:53 -0700 Subject: [PATCH 2/6] not lifting validation check until windows team release supported os for dualstack containers --- pkg/api/vlabs/validate.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index df1e548a76..8a2c5cf758 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -480,9 +480,9 @@ func (a *Properties) validateAgentPoolProfiles(isUpdate bool) error { } // validate os type is linux if dual stack feature is enabled - if a.FeatureFlags.IsIPv6OnlyEnabled() { + if a.FeatureFlags.IsIPv6DualStackEnabled() || a.FeatureFlags.IsIPv6OnlyEnabled() { if agentPoolProfile.OSType == Windows { - return errors.Errorf("single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfile.Name, agentPoolProfile.OSType) + return errors.Errorf("Dual stack and single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfile.Name, agentPoolProfile.OSType) } } From 0e6a581cc6d1710e3afe13837eeacd0ef9c3bc68 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 4 Jun 2020 18:38:03 -0700 Subject: [PATCH 3/6] fix UTs --- pkg/engine/virtualmachinescalesets.go | 6 ++---- pkg/engine/virtualmachinescalesets_test.go | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/engine/virtualmachinescalesets.go b/pkg/engine/virtualmachinescalesets.go index 65ac451815..48fb8fc2e7 100644 --- a/pkg/engine/virtualmachinescalesets.go +++ b/pkg/engine/virtualmachinescalesets.go @@ -489,8 +489,6 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi } var ipConfigurations []compute.VirtualMachineScaleSetIPConfiguration - // multiple v6 configs are not supported. this flag is set after creating 1 v6 config. - var isV6ConfigCreated bool for i := 1; i <= profile.IPAddressCount; i++ { ipconfig := compute.VirtualMachineScaleSetIPConfiguration{ Name: to.StringPtr(fmt.Sprintf("ipconfig%d", i)), @@ -552,7 +550,8 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi ipconfig.VirtualMachineScaleSetIPConfigurationProperties = &ipConfigProps ipConfigurations = append(ipConfigurations, ipconfig) - if !isV6ConfigCreated && (cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") || cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only")) { + // multiple v6 configs are not supported. creating 1 IPv6 config. + if i == 1 && (cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") || cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only")) { ipconfigv6 := compute.VirtualMachineScaleSetIPConfiguration{ Name: to.StringPtr(fmt.Sprintf("ipconfig%dv6", i)), VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ @@ -564,7 +563,6 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi }, } ipConfigurations = append(ipConfigurations, ipconfigv6) - isV6ConfigCreated = true } } diff --git a/pkg/engine/virtualmachinescalesets_test.go b/pkg/engine/virtualmachinescalesets_test.go index 1e70e6bf5b..4d45289f01 100644 --- a/pkg/engine/virtualmachinescalesets_test.go +++ b/pkg/engine/virtualmachinescalesets_test.go @@ -1021,7 +1021,7 @@ func getIPConfigs(lbBackendAddresPoolID *string, isStandardLB, ipv6DualStackEnab } ipConfigs = append(ipConfigs, ipconfig) - if ipv6DualStackEnabled { + if i == 1 && ipv6DualStackEnabled { ipconfigv6 := compute.VirtualMachineScaleSetIPConfiguration{ Name: to.StringPtr(fmt.Sprintf("ipconfig%dv6", i)), VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ From 83e65ecd2fae2c94d8ff888b57248d673d3509a7 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 5 Jun 2020 11:48:43 -0700 Subject: [PATCH 4/6] fixed nit space issue --- pkg/engine/virtualmachinescalesets.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/engine/virtualmachinescalesets.go b/pkg/engine/virtualmachinescalesets.go index 48fb8fc2e7..73bf498427 100644 --- a/pkg/engine/virtualmachinescalesets.go +++ b/pkg/engine/virtualmachinescalesets.go @@ -862,7 +862,7 @@ func associateAddonIdentitiesToVMSS(addonProfiles map[string]api.AddonProfile, v // Note: virtualMachineScaleSet.Identity is not nil and its type is None will NEVER happen in current AKS-Engine's implementation. if virtualMachineScaleSet.Identity == nil { virtualMachineScaleSet.Identity = &compute.VirtualMachineScaleSetIdentity{ - Type: compute.ResourceIdentityTypeUserAssigned, + Type: compute.ResourceIdentityTypeUserAssigned, UserAssignedIdentities: make(map[string]*compute.VirtualMachineScaleSetIdentityUserAssignedIdentitiesValue), } } else if virtualMachineScaleSet.Identity.Type == compute.ResourceIdentityTypeSystemAssigned { From c4080cf28b2f7e0cbfea67baab12f34b521809e8 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Mon, 8 Jun 2020 14:58:19 -0700 Subject: [PATCH 5/6] addressed review comments --- parts/k8s/kubeproxystart.ps1 | 17 ++++++++++------- parts/k8s/kuberneteswindowssetup.ps1 | 5 +++-- pkg/engine/templates_generated.go | 23 ++++++++++++++--------- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/parts/k8s/kubeproxystart.ps1 b/parts/k8s/kubeproxystart.ps1 index 6e6c4a778c..a564f78018 100644 --- a/parts/k8s/kubeproxystart.ps1 +++ b/parts/k8s/kubeproxystart.ps1 @@ -7,7 +7,8 @@ if ($Global:ClusterConfiguration.Cni.Name -eq "kubenet") { $env:KUBE_NETWORK = $KubeNetwork $global:HNSModule = "c:\k\hns.psm1" -$KubeDir = $Global:ClusterConfiguration.Install.Destination +$global:KubeDir = $Global:ClusterConfiguration.Install.Destination +$global:KubeproxyArgList = @("--v=3", "--proxy-mode=kernelspace", "--hostname-override=$env:computername", "--kubeconfig=$KubeDir\config") $hnsNetwork = Get-HnsNetwork | ? Name -EQ $KubeNetwork while (!$hnsNetwork) { @@ -16,6 +17,12 @@ while (!$hnsNetwork) { $hnsNetwork = Get-HnsNetwork | ? Name -EQ $KubeNetwork } +# add dualstack feature gate if dualstack enabled +$isDualStackEnabled = ("--feature-gates=IPv6DualStack=true" | ? { $Global:ClusterConfiguration.Kubernetes.Kubelet.ConfigArgs -match $_ }) -ne $null +if ($isDualStackEnabled) { + $global:KubeproxyArgList += @("--feature-gates=IPv6DualStack=true") +} + # # cleanup the persisted policy lists # @@ -24,9 +31,5 @@ Import-Module $global:HNSModule # and https://github.com/kubernetes/kubernetes/pull/78612 for <= 1.15 Get-HnsPolicyList | Remove-HnsPolicyList -if (("--feature-gates=IPv6DualStack=true" | ? { $Global:ClusterConfiguration.Kubernetes.Kubelet.ConfigArgs -match $_ }) -ne $null) { - .$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --feature-gates=IPv6DualStack=true --hostname-override=$env:computername --kubeconfig=$KubeDir\config -} -else { - .$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --hostname-override=$env:computername --kubeconfig=$KubeDir\config -} \ No newline at end of file +$KubeproxyCmdline = "$global:KubeDir\kube-proxy.exe "+ ($global:KubeproxyArgList -join " ") +Invoke-Expression $KubeproxyCmdline diff --git a/parts/k8s/kuberneteswindowssetup.ps1 b/parts/k8s/kuberneteswindowssetup.ps1 index 59c6f419e1..4c1b201cab 100644 --- a/parts/k8s/kuberneteswindowssetup.ps1 +++ b/parts/k8s/kuberneteswindowssetup.ps1 @@ -133,6 +133,7 @@ $global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf") # $global:NetworkPolicy = "{{WrapAsParameter "networkPolicy"}}" # BUG: unused $global:NetworkPlugin = "{{WrapAsParameter "networkPlugin"}}" $global:VNetCNIPluginsURL = "{{WrapAsParameter "vnetCniWindowsPluginsURL"}}" +$global:IsDualStackEnabled = {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} # Telemetry settings $global:EnableTelemetry = "{{WrapAsVariable "enableTelemetry" }}"; @@ -358,7 +359,7 @@ try -VNetCIDR $global:VNetCIDR ` {{- /* Azure Stack has discrete Azure CNI config requirements */}} -IsAzureStack {{if IsAzureStackCloud}}$true{{else}}$false{{end}} ` - -IsDualStackEnabled {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} + -IsDualStackEnabled $global:IsDualStackEnabled if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` @@ -383,7 +384,7 @@ try } } - New-ExternalHnsNetwork -IsDualStackEnabled {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} + New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled Install-KubernetesServices ` -KubeDir $global:KubeDir diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index 4ff8da06c6..340ca3f96a 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -22348,7 +22348,8 @@ if ($Global:ClusterConfiguration.Cni.Name -eq "kubenet") { $env:KUBE_NETWORK = $KubeNetwork $global:HNSModule = "c:\k\hns.psm1" -$KubeDir = $Global:ClusterConfiguration.Install.Destination +$global:KubeDir = $Global:ClusterConfiguration.Install.Destination +$global:KubeproxyArgList = @("--v=3", "--proxy-mode=kernelspace", "--hostname-override=$env:computername", "--kubeconfig=$KubeDir\config") $hnsNetwork = Get-HnsNetwork | ? Name -EQ $KubeNetwork while (!$hnsNetwork) { @@ -22357,6 +22358,12 @@ while (!$hnsNetwork) { $hnsNetwork = Get-HnsNetwork | ? Name -EQ $KubeNetwork } +# add dualstack feature gate if dualstack enabled +$isDualStackEnabled = ("--feature-gates=IPv6DualStack=true" | ? { $Global:ClusterConfiguration.Kubernetes.Kubelet.ConfigArgs -match $_ }) -ne $null +if ($isDualStackEnabled) { + $global:KubeproxyArgList += @("--feature-gates=IPv6DualStack=true") +} + # # cleanup the persisted policy lists # @@ -22365,12 +22372,9 @@ Import-Module $global:HNSModule # and https://github.com/kubernetes/kubernetes/pull/78612 for <= 1.15 Get-HnsPolicyList | Remove-HnsPolicyList -if (("--feature-gates=IPv6DualStack=true" | ? { $Global:ClusterConfiguration.Kubernetes.Kubelet.ConfigArgs -match $_ }) -ne $null) { - .$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --feature-gates=IPv6DualStack=true --hostname-override=$env:computername --kubeconfig=$KubeDir\config -} -else { - .$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --hostname-override=$env:computername --kubeconfig=$KubeDir\config -}`) +$KubeproxyCmdline = "$global:KubeDir\kube-proxy.exe "+ ($global:KubeproxyArgList -join " ") +Invoke-Expression $KubeproxyCmdline +`) func k8sKubeproxystartPs1Bytes() ([]byte, error) { return _k8sKubeproxystartPs1, nil @@ -23368,6 +23372,7 @@ $global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf") # $global:NetworkPolicy = "{{WrapAsParameter "networkPolicy"}}" # BUG: unused $global:NetworkPlugin = "{{WrapAsParameter "networkPlugin"}}" $global:VNetCNIPluginsURL = "{{WrapAsParameter "vnetCniWindowsPluginsURL"}}" +$global:IsDualStackEnabled = {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} # Telemetry settings $global:EnableTelemetry = "{{WrapAsVariable "enableTelemetry" }}"; @@ -23593,7 +23598,7 @@ try -VNetCIDR $global:VNetCIDR ` + "`" + ` {{- /* Azure Stack has discrete Azure CNI config requirements */}} -IsAzureStack {{if IsAzureStackCloud}}$true{{else}}$false{{end}} ` + "`" + ` - -IsDualStackEnabled {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} + -IsDualStackEnabled $global:IsDualStackEnabled if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` + "`" + ` @@ -23618,7 +23623,7 @@ try } } - New-ExternalHnsNetwork -IsDualStackEnabled {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} + New-ExternalHnsNetwork -IsDualStackEnabled $global:IsDualStackEnabled Install-KubernetesServices ` + "`" + ` -KubeDir $global:KubeDir From 610e26aa3b0061440f657c73218992c5be9b87b0 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 10 Jun 2020 19:45:03 -0700 Subject: [PATCH 6/6] addressed review comments --- parts/k8s/kubeproxystart.ps1 | 2 +- pkg/api/defaults.go | 20 ++++++++ pkg/api/defaults_test.go | 83 +++++++++++++++++++++++++++++++ pkg/engine/templates_generated.go | 2 +- 4 files changed, 105 insertions(+), 2 deletions(-) diff --git a/parts/k8s/kubeproxystart.ps1 b/parts/k8s/kubeproxystart.ps1 index a564f78018..12d1c4ccb6 100644 --- a/parts/k8s/kubeproxystart.ps1 +++ b/parts/k8s/kubeproxystart.ps1 @@ -18,7 +18,7 @@ while (!$hnsNetwork) { } # add dualstack feature gate if dualstack enabled -$isDualStackEnabled = ("--feature-gates=IPv6DualStack=true" | ? { $Global:ClusterConfiguration.Kubernetes.Kubelet.ConfigArgs -match $_ }) -ne $null +$isDualStackEnabled = ("IPv6DualStack=true" | ? { $Global:ClusterConfiguration.Kubernetes.Kubelet.ConfigArgs -match $_ }) -ne $null if ($isDualStackEnabled) { $global:KubeproxyArgList += @("--feature-gates=IPv6DualStack=true") } diff --git a/pkg/api/defaults.go b/pkg/api/defaults.go index a7ddbe1a08..e78e3f6fb0 100644 --- a/pkg/api/defaults.go +++ b/pkg/api/defaults.go @@ -280,6 +280,26 @@ func (cs *ContainerService) setOrchestratorDefaults(isUpgrade, isScale bool) { if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only") { o.KubernetesConfig.ServiceCIDR = DefaultKubernetesServiceCIDRIPv6 } + if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") { + o.KubernetesConfig.ServiceCIDR = strings.Join([]string{DefaultKubernetesServiceCIDR, DefaultKubernetesServiceCIDRIPv6}, ",") + } + } else { + if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") { + serviceAddrs := strings.Split(o.KubernetesConfig.ServiceCIDR, ",") + if len(serviceAddrs) == 1 { + ip, _, err := net.ParseCIDR(serviceAddrs[0]) + if err == nil { + if ip.To4() != nil { + // the first cidr block is ipv4, so append ipv6 + serviceAddrs = append(serviceAddrs, DefaultKubernetesServiceCIDRIPv6) + } else { + // first cidr has to be ipv4 + serviceAddrs = append([]string{DefaultKubernetesServiceCIDR}, serviceAddrs...) + } + } + o.KubernetesConfig.ServiceCIDR = strings.Join(serviceAddrs, ",") + } + } } if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.14.0") { diff --git a/pkg/api/defaults_test.go b/pkg/api/defaults_test.go index 7ea05d16e4..6654d61995 100644 --- a/pkg/api/defaults_test.go +++ b/pkg/api/defaults_test.go @@ -1831,6 +1831,89 @@ func TestMasterProfileDefaults(t *testing.T) { properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet, expectedClusterSubnet) } + // this validates service cidr default configuration for dual stack feature when both ipv4 and ipv6 subnet provided + mockCS = getMockBaseContainerService("1.16.0") + properties = mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = Kubernetes + properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "azure" + properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR = "192.168.0.0/16,ace:cab:deca::/8" + properties.FeatureFlags = &FeatureFlags{EnableIPv6DualStack: true} + _, err = mockCS.SetPropertiesDefaults(PropertiesDefaultsParams{ + IsScale: false, + IsUpgrade: false, + PkiKeySize: helpers.DefaultPkiKeySize, + }) + if err != nil { + t.Error(err) + } + expectedServiceCIDR := strings.Join([]string{"192.168.0.0/16", "ace:cab:deca::/8"}, ",") + if properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR != expectedServiceCIDR { + t.Fatalf("OrchestratorProfile.KubernetesConfig.ServiceCIDR did not have the expected configuration, got %s, expected %s", + properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR, expectedServiceCIDR) + } + + // this validates service cidr default configuration for dual stack feature when ipv4 provided + mockCS = getMockBaseContainerService("1.16.0") + properties = mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = Kubernetes + properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "azure" + properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR = "192.168.0.0/16" + properties.FeatureFlags = &FeatureFlags{EnableIPv6DualStack: true} + _, err = mockCS.SetPropertiesDefaults(PropertiesDefaultsParams{ + IsScale: false, + IsUpgrade: false, + PkiKeySize: helpers.DefaultPkiKeySize, + }) + if err != nil { + t.Error(err) + } + expectedServiceCIDR = strings.Join([]string{"192.168.0.0/16", DefaultKubernetesServiceCIDRIPv6}, ",") + if properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR != expectedServiceCIDR { + t.Fatalf("OrchestratorProfile.KubernetesConfig.ServiceCIDR did not have the expected configuration, got %s, expected %s", + properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR, expectedServiceCIDR) + } + + // this validates service cidr default configuration for dual stack feature when ipv6 provided + mockCS = getMockBaseContainerService("1.16.0") + properties = mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = Kubernetes + properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "azure" + properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR = "ace:cab:deca::/8" + properties.FeatureFlags = &FeatureFlags{EnableIPv6DualStack: true} + _, err = mockCS.SetPropertiesDefaults(PropertiesDefaultsParams{ + IsScale: false, + IsUpgrade: false, + PkiKeySize: helpers.DefaultPkiKeySize, + }) + if err != nil { + t.Error(err) + } + expectedServiceCIDR = strings.Join([]string{DefaultKubernetesServiceCIDR, "ace:cab:deca::/8"}, ",") + if properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR != expectedServiceCIDR { + t.Fatalf("OrchestratorProfile.KubernetesConfig.ServiceCIDR did not have the expected configuration, got %s, expected %s", + properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR, expectedServiceCIDR) + } + + // this validates service cidr default configuration for dual stack feature when servicecidr not provided + mockCS = getMockBaseContainerService("1.16.0") + properties = mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = Kubernetes + properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "azure" + properties.FeatureFlags = &FeatureFlags{EnableIPv6DualStack: true} + _, err = mockCS.SetPropertiesDefaults(PropertiesDefaultsParams{ + IsScale: false, + IsUpgrade: false, + PkiKeySize: helpers.DefaultPkiKeySize, + }) + if err != nil { + t.Error(err) + } + expectedServiceCIDR = strings.Join([]string{DefaultKubernetesServiceCIDR, DefaultKubernetesServiceCIDRIPv6}, ",") + if properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR != expectedServiceCIDR { + t.Fatalf("OrchestratorProfile.KubernetesConfig.ServiceCIDR did not have the expected configuration, got %s, expected %s", + properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR, expectedServiceCIDR) + } + // this validates default configurations for OutboundRuleIdleTimeoutInMinutes. mockCS = getMockBaseContainerService("1.18.2") properties = mockCS.Properties diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index 340ca3f96a..8905075ed8 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -22359,7 +22359,7 @@ while (!$hnsNetwork) { } # add dualstack feature gate if dualstack enabled -$isDualStackEnabled = ("--feature-gates=IPv6DualStack=true" | ? { $Global:ClusterConfiguration.Kubernetes.Kubelet.ConfigArgs -match $_ }) -ne $null +$isDualStackEnabled = ("IPv6DualStack=true" | ? { $Global:ClusterConfiguration.Kubernetes.Kubelet.ConfigArgs -match $_ }) -ne $null if ($isDualStackEnabled) { $global:KubeproxyArgList += @("--feature-gates=IPv6DualStack=true") }