Skip to content
This repository has been archived by the owner on Oct 24, 2023. It is now read-only.

feat: Dualstack support for Windows containers #3415

Merged
merged 6 commits into from
Jun 11, 2020
Merged

feat: Dualstack support for Windows containers #3415

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f0b65b2
dualstack changes for windows vmss
tamilmani1989 Jun 5, 2020
eb5b815
not lifting validation check until windows team release supported os …
tamilmani1989 Jun 5, 2020
0e6a581
fix UTs
tamilmani1989 Jun 5, 2020
83e65ec
fixed nit space issue
tamilmani1989 Jun 5, 2020
c4080cf
addressed review comments
tamilmani1989 Jun 8, 2020
610e26a
addressed review comments
tamilmani1989 Jun 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion parts/k8s/kubeletstart.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,14 @@ if ($global:NetworkPlugin -eq "azure") {
if ((Test-Path $cnilock)) {
Remove-Item $cnilock
}

$cnijson = [io.path]::Combine("$KubeDir", "azure-vnet-ipamv6.json")
if ((Test-Path $cnijson)) {
Remove-Item $cnijson
}
$cnilock = [io.path]::Combine("$KubeDir", "azure-vnet-ipamv6.json.lock")
if ((Test-Path $cnilock)) {
Remove-Item $cnilock
}
$cnijson = [io.path]::Combine("$KubeDir", "azure-vnet.json")
if ((Test-Path $cnijson)) {
Remove-Item $cnijson
Expand Down
7 changes: 6 additions & 1 deletion parts/k8s/kubeproxystart.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,9 @@ Import-Module $global:HNSModule
# and https://github.com/kubernetes/kubernetes/pull/78612 for <= 1.15
Get-HnsPolicyList | Remove-HnsPolicyList

.$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --hostname-override=$env:computername --kubeconfig=$KubeDir\config
if (("--feature-gates=IPv6DualStack=true" | ? { $Global:ClusterConfiguration.Kubernetes.Kubelet.ConfigArgs -match $_ }) -ne $null) {
tamilmani1989 marked this conversation as resolved.
Show resolved Hide resolved
.$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --feature-gates=IPv6DualStack=true --hostname-override=$env:computername --kubeconfig=$KubeDir\config
}
else {
.$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --hostname-override=$env:computername --kubeconfig=$KubeDir\config
}
tamilmani1989 marked this conversation as resolved.
Show resolved Hide resolved
5 changes: 3 additions & 2 deletions parts/k8s/kuberneteswindowssetup.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,8 @@ try
-KubeServiceCIDR $global:KubeServiceCIDR `
-VNetCIDR $global:VNetCIDR `
{{- /* Azure Stack has discrete Azure CNI config requirements */}}
-IsAzureStack {{if IsAzureStackCloud}}$true{{else}}$false{{end}}
-IsAzureStack {{if IsAzureStackCloud}}$true{{else}}$false{{end}} `
-IsDualStackEnabled {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}}
tamilmani1989 marked this conversation as resolved.
Show resolved Hide resolved

if ($TargetEnvironment -ieq "AzureStackCloud") {
GenerateAzureStackCNIConfig `
Expand All @@ -382,7 +383,7 @@ try
}
}

New-ExternalHnsNetwork
New-ExternalHnsNetwork -IsDualStackEnabled {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}}

Install-KubernetesServices `
-KubeDir $global:KubeDir
Expand Down
49 changes: 43 additions & 6 deletions parts/k8s/windowsazurecnifunc.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,18 @@ Set-AzureCNIConfig
[Parameter(Mandatory=$true)][string]
$VNetCIDR,
[Parameter(Mandatory=$true)][bool]
$IsAzureStack
$IsAzureStack,
[Parameter(Mandatory=$true)][bool]
$IsDualStackEnabled
)
# Fill in DNS information for kubernetes.
$exceptionAddresses = @($KubeClusterCIDR, $MasterSubnet, $VNetCIDR)
if ($IsDualStackEnabled){
$subnetToPass = $KubeClusterCIDR -split ","
$exceptionAddresses = @($subnetToPass[0], $MasterSubnet, $VNetCIDR)
}
else {
$exceptionAddresses = @($KubeClusterCIDR, $MasterSubnet, $VNetCIDR)
}

$fileName = [Io.path]::Combine("$AzureCNIConfDir", "10-azure.conflist")
$configJson = Get-Content $fileName | ConvertFrom-Json
Expand All @@ -93,7 +101,25 @@ Set-AzureCNIConfig
$configJson.plugins.AdditionalArgs[0].Value.ExceptionList = $exceptionAddresses
}

$configJson.plugins.AdditionalArgs[1].Value.DestinationPrefix = $KubeServiceCIDR
if ($IsDualStackEnabled){
$configJson.plugins[0]|Add-Member -Name "ipv6Mode" -Value "ipv6nat" -MemberType NoteProperty
$serviceCidr = $KubeServiceCIDR -split ","
$configJson.plugins[0].AdditionalArgs[1].Value.DestinationPrefix = $serviceCidr[0]
$valueObj = [PSCustomObject]@{
Type = 'ROUTE'
DestinationPrefix = $serviceCidr[1]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about the case where cluster is dual stack, but services are just single stack? The user could just have a single family CIDR for service.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so if user didn't specify ipv6 service cidr, won't aks-e assign default service v6 cidr? May be im interpreting your ask wrongly

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

aks-e default v6 CIDR only for ClusterCIDR. Having v4 and v6 CIDR for services isn't mandatory.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i believe we should add default for v6 service cidr also here if user opted for dualstack.

aks-engine/pkg/api/defaults.go

Line 278 in 24e9029

if o.KubernetesConfig.ServiceCIDR == "" {

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tamilmani1989 Having dual stack CIDRs for services is not mandatory unlike ClusterCIDRs. The user can just define single stack v4 or v6 service CIDR in dual stack cluster. Thats the reason for not appending v6 service CIDR unless the user explicitly requests it.

Copy link
Member Author

@tamilmani1989 tamilmani1989 Jun 9, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@aramase Yes you are right. Its not mandatory to create ipv6 svc but if user decided to have in future he has to recreate cluster. What's the user going to loose if we assign default svc cidr if its not explicitly specified?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@aramase what you think?

NeedEncap = $True
}

$jsonContent = [PSCustomObject]@{
Name = 'EndpointPolicy'
Value = $valueObj
}
$configJson.plugins[0].AdditionalArgs += $jsonContent
}
else {
$configJson.plugins[0].AdditionalArgs[1].Value.DestinationPrefix = $KubeServiceCIDR
}

if ($IsAzureStack) {
Add-Member -InputObject $configJson.plugins[0].ipam -MemberType NoteProperty -Name "environment" -Value "mas"
Expand Down Expand Up @@ -255,7 +281,13 @@ function GenerateAzureStackCNIConfig
Set-ItemProperty -Path $azureCNIConfigFile -Name IsReadOnly -Value $true
}

function New-ExternalHnsNetwork {
function New-ExternalHnsNetwork
{
param (
[Parameter(Mandatory=$true)][bool]
$IsDualStackEnabled
)

Write-Log "Creating new HNS network `"ext`""
$externalNetwork = "ext"
$na = @(Get-NetAdapter -Physical)
Expand All @@ -272,9 +304,14 @@ function New-ExternalHnsNetwork {

$stopWatch = New-Object System.Diagnostics.Stopwatch
$stopWatch.Start()
# Fixme : use a smallest range possible, that will not collide with any pod space
New-HNSNetwork -Type $global:NetworkMode -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -AdapterName $adapterName -Name $externalNetwork -Verbose

# Fixme : use a smallest range possible, that will not collide with any pod space
if ($IsDualStackEnabled) {
New-HNSNetwork -Type $global:NetworkMode -AddressPrefix @("192.168.255.0/30","192:168:255::0/127") -Gateway @("192.168.255.1","192:168:255::1") -AdapterName $adapterName -Name $externalNetwork -Verbose
}
else {
New-HNSNetwork -Type $global:NetworkMode -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -AdapterName $adapterName -Name $externalNetwork -Verbose
}
# Wait for the switch to be created and the ip address to be assigned.
for ($i = 0; $i -lt 60; $i++) {
$mgmtIPAfterNetworkCreate = Get-NetIPAddress $managementIP -ErrorAction SilentlyContinue
Expand Down
2 changes: 2 additions & 0 deletions parts/k8s/windowsnodereset.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ if ($hnsNetwork) {
"c:\k\azure-vnet.json.lock",
"c:\k\azure-vnet-ipam.json",
"c:\k\azure-vnet-ipam.json.lock"
"c:\k\azure-vnet-ipamv6.json",
"c:\k\azure-vnet-ipamv6.json.lock"
)

foreach ($file in $filesToRemove) {
Expand Down
73 changes: 62 additions & 11 deletions pkg/engine/templates_generated.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions pkg/engine/virtualmachinescalesets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -489,7 +489,6 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi
}

var ipConfigurations []compute.VirtualMachineScaleSetIPConfiguration

for i := 1; i <= profile.IPAddressCount; i++ {
ipconfig := compute.VirtualMachineScaleSetIPConfiguration{
Name: to.StringPtr(fmt.Sprintf("ipconfig%d", i)),
Expand Down Expand Up @@ -551,7 +550,8 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi
ipconfig.VirtualMachineScaleSetIPConfigurationProperties = &ipConfigProps
ipConfigurations = append(ipConfigurations, ipconfig)

if cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") || cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only") {
// multiple v6 configs are not supported. creating 1 IPv6 config.
if i == 1 && (cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack") || cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only")) {
ipconfigv6 := compute.VirtualMachineScaleSetIPConfiguration{
Name: to.StringPtr(fmt.Sprintf("ipconfig%dv6", i)),
VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{
Expand Down
2 changes: 1 addition & 1 deletion pkg/engine/virtualmachinescalesets_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1021,7 +1021,7 @@ func getIPConfigs(lbBackendAddresPoolID *string, isStandardLB, ipv6DualStackEnab
}
ipConfigs = append(ipConfigs, ipconfig)

if ipv6DualStackEnabled {
if i == 1 && ipv6DualStackEnabled {
ipconfigv6 := compute.VirtualMachineScaleSetIPConfiguration{
Name: to.StringPtr(fmt.Sprintf("ipconfig%dv6", i)),
VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{
Expand Down