From d75283403d6f9a8ad4a0142d835fb824a3ad5f8f Mon Sep 17 00:00:00 2001 From: Jiangtian Li Date: Tue, 21 Nov 2017 10:10:18 -0800 Subject: [PATCH 1/3] Fix custom vnet issue for Windows cluster --- parts/kuberneteswindowssetup.ps1 | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/parts/kuberneteswindowssetup.ps1 b/parts/kuberneteswindowssetup.ps1 index 4f14f28739..c642e1329a 100644 --- a/parts/kuberneteswindowssetup.ps1 +++ b/parts/kuberneteswindowssetup.ps1 @@ -54,7 +54,6 @@ $global:TenantId = "{{WrapAsVariable "tenantID"}}" $global:SubscriptionId = "{{WrapAsVariable "subscriptionId"}}" $global:ResourceGroup = "{{WrapAsVariable "resourceGroup"}}" $global:SubnetName = "{{WrapAsVariable "subnetName"}}" -$global:MasterSubnet = "{{WrapAsVariable "subnet"}}" $global:SecurityGroupName = "{{WrapAsVariable "nsgName"}}" $global:VNetName = "{{WrapAsVariable "virtualNetworkName"}}" $global:RouteTableName = "{{WrapAsVariable "routeTableName"}}" @@ -155,6 +154,14 @@ New-InfraContainer() docker build -t kubletwin/pause . } +function Get-AgentpoolSubnet() +{ + $netIPAddr = Get-NetIPAddress -InterfaceIndex (Get-NetAdapter| ? ifAlias -like "Ethernet*").ifIndex -AddressFamily IPv4 + [IPAddress]$ipAddr = $netIPAddr.IPAddress + $ipAddr.Address = $ipAddr.Address -band (([UInt32]::MaxValue) -shr (32 - $netIPAddr.PrefixLength)) + return $ipAddr.IPAddressToString + "/" + $netIPAddr.PrefixLength +} + function Write-KubernetesStartFiles($podCIDR) { @@ -180,11 +187,13 @@ c:\k\kubelet.exe --hostname-override=`$global:AzureHostname --pod-infra-containe $KubeletArgListStr = "@`($KubeletArgListStr`)" + $agentpoolSubnet = Get-AgentpoolSubnet + $kubeStartStr = @" `$global:AzureHostname = "$AzureHostname" `$global:MasterIP = "$MasterIP" `$global:KubeDnsServiceIp = "$KubeDnsServiceIp" -`$global:MasterSubnet = "$global:MasterSubnet" +`$global:AgentPoolSubnet = "$agentpoolSubnet" `$global:KubeClusterCIDR = "$global:KubeClusterCIDR" `$global:KubeServiceCIDR = "$global:KubeServiceCIDR" `$global:KubeBinariesVersion = "$global:KubeBinariesVersion" @@ -213,7 +222,7 @@ Test-PodCIDR(`$podCIDR) } function -Update-CNIConfig(`$podCIDR, `$masterSubnetGW) +Update-CNIConfig(`$podCIDR, `$agentpoolSubnetGW) { `$jsonSampleConfig = "{ @@ -245,11 +254,11 @@ Update-CNIConfig(`$podCIDR, `$masterSubnetGW) `$configJson = ConvertFrom-Json `$jsonSampleConfig `$configJson.name = `$global:NetworkMode.ToLower() `$configJson.ipam.subnet=`$podCIDR - `$configJson.ipam.routes[0].GW = `$masterSubnetGW + `$configJson.ipam.routes[0].GW = `$agentpoolSubnetGW `$configJson.dns.Nameservers[0] = `$global:KubeDnsServiceIp `$configJson.AdditionalArgs[0].Value.ExceptionList[0] = `$global:KubeClusterCIDR - `$configJson.AdditionalArgs[0].Value.ExceptionList[1] = `$global:MasterSubnet + `$configJson.AdditionalArgs[0].Value.ExceptionList[1] = `$global:AgentPoolSubnet `$configJson.AdditionalArgs[1].Value.DestinationPrefix = `$global:KubeServiceCIDR if (Test-Path `$global:CNIConfig) @@ -264,7 +273,7 @@ Update-CNIConfig(`$podCIDR, `$masterSubnetGW) try { - `$masterSubnetGW = Get-DefaultGateway `$global:MasterSubnet + `$agentpoolSubnetGW = Get-DefaultGateway `$global:AgentPoolSubnet `$podCIDR=Get-PodCIDR `$podCidrDiscovered=Test-PodCIDR(`$podCIDR) @@ -301,12 +310,12 @@ try Write-Host "No HNS network found, creating a new one..." ipmo `$global:HNSModule - `$hnsNetwork = New-HNSNetwork -Type `$global:NetworkMode -AddressPrefix `$podCIDR -Gateway `$masterSubnetGW -Name `$global:NetworkMode.ToLower() -Verbose + `$hnsNetwork = New-HNSNetwork -Type `$global:NetworkMode -AddressPrefix `$podCIDR -Gateway `$agentpoolSubnetGW -Name `$global:NetworkMode.ToLower() -Verbose } Start-Sleep 10 # Add route to all other POD networks - Update-CNIConfig `$podCIDR `$masterSubnetGW + Update-CNIConfig `$podCIDR `$agentpoolSubnetGW $KubeletCommandLine } From 5f076776600c51934392b103bff3c069f6cf17c2 Mon Sep 17 00:00:00 2001 From: Jiangtian Li Date: Tue, 21 Nov 2017 10:10:59 -0800 Subject: [PATCH 2/3] Add test case for windows custom vnet --- examples/vnet/kubernetesvnet-windows.json | 50 +++++++++++++++++++ examples/vnet/kubernetesvnet-windows.json.env | 2 + 2 files changed, 52 insertions(+) create mode 100644 examples/vnet/kubernetesvnet-windows.json create mode 100644 examples/vnet/kubernetesvnet-windows.json.env diff --git a/examples/vnet/kubernetesvnet-windows.json b/examples/vnet/kubernetesvnet-windows.json new file mode 100644 index 0000000000..3cd720e432 --- /dev/null +++ b/examples/vnet/kubernetesvnet-windows.json @@ -0,0 +1,50 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes" + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "test", + "vmSize": "Standard_D2_v2", + "vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME", + "firstConsecutiveStaticIP": "10.239.255.239" + }, + "agentPoolProfiles": [ + { + "name": "agentpri", + "count": 2, + "vmSize": "Standard_D2_v2", + "vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME", + "availabilityProfile": "AvailabilitySet" + }, + { + "name": "agentpri2", + "count": 2, + "vmSize": "Standard_D2_v2", + "vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME", + "availabilityProfile": "AvailabilitySet", + "osType": "Windows" + } + ], + "windowsProfile": { + "adminUsername": "azureuser", + "adminPassword": "replacepassword1234$" + }, + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + } + } + } \ No newline at end of file diff --git a/examples/vnet/kubernetesvnet-windows.json.env b/examples/vnet/kubernetesvnet-windows.json.env new file mode 100644 index 0000000000..1146065ee9 --- /dev/null +++ b/examples/vnet/kubernetesvnet-windows.json.env @@ -0,0 +1,2 @@ +ACSE_PREDEPLOY=examples/vnet/k8s-vnet-predeploy.sh +ACSE_POSTDEPLOY=examples/vnet/k8s-vnet-postdeploy.sh From e982e7096ffd0ef369008333dcf769160d4c372c Mon Sep 17 00:00:00 2001 From: Jiangtian Li Date: Mon, 27 Nov 2017 15:43:58 -0800 Subject: [PATCH 3/3] Add windows vnet test to regression test config --- test/acse-conf/acse-regression.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/acse-conf/acse-regression.json b/test/acse-conf/acse-regression.json index e2d8c36b4b..1249a4669d 100644 --- a/test/acse-conf/acse-regression.json +++ b/test/acse-conf/acse-regression.json @@ -93,6 +93,10 @@ "cluster_definition": "vnet/kubernetesvnet.json", "category": "network" }, + { + "cluster_definition": "vnet/kubernetesvnet-windows.json", + "category": "network" + }, { "cluster_definition": "vnet/swarmmodevnet.json", "category": "network"