diff --git a/debug b/debug new file mode 100644 index 0000000000..9c30b5cd62 Binary files /dev/null and b/debug differ diff --git a/parts/k8s/kuberneteswindowssetup.ps1 b/parts/k8s/kuberneteswindowssetup.ps1 index 3ed563839f..7b686578ae 100644 --- a/parts/k8s/kuberneteswindowssetup.ps1 +++ b/parts/k8s/kuberneteswindowssetup.ps1 @@ -68,9 +68,20 @@ $global:UseInstanceMetadata = "{{WrapAsVariable "useInstanceMetadata"}}" $global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni") $global:NetworkMode = "L2Bridge" $global:CNIConfig = [Io.path]::Combine($global:CNIPath, "config", "`$global:NetworkMode.conf") +$global:CNIConfigPath = [Io.path]::Combine("$global:CNIPath", "config") +$global:WindowsCNIKubeletOptions = " --network-plugin=cni --cni-bin-dir=$global:CNIPath --cni-conf-dir=$global:CNIConfigPath" $global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.psm1") $global:VolumePluginDir = [Io.path]::Combine("$global:KubeDir", "volumeplugins") +#azure cni +$global:NetworkPolicy = "{{WrapAsVariable "networkPolicy"}}" +$global:VNetCNIPluginsURL = "{{WrapAsVariable "vnetCniWindowsPluginsURL"}}" + +$global:AzureCNIDir = [Io.path]::Combine("$global:KubeDir", "azurecni") +$global:AzureCNIBinDir = [Io.path]::Combine("$global:AzureCNIDir", "bin") +$global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf") +$global:AzureCNIKubeletOptions = " --network-plugin=cni --cni-bin-dir=$global:AzureCNIBinDir --cni-conf-dir=$global:AzureCNIConfDir" +$global:AzureCNIEnabled = $false filter Timestamp {"$(Get-Date -Format o): $_"} @@ -168,6 +179,56 @@ New-InfraContainer() docker build -t kubletwin/pause . } +function +Set-VnetPluginMode($mode) +{ + # Sets Azure VNET CNI plugin operational mode. + $fileName = [Io.path]::Combine("$global:AzureCNIConfDir", "10-azure.conflist") + (Get-Content $fileName) | %{$_ -replace "`"mode`":.*", "`"mode`": `"$mode`","} | Out-File -encoding ASCII -filepath $fileName +} + +function +Install-VnetPlugins() +{ + # Create CNI directories. + mkdir $global:AzureCNIBinDir + mkdir $global:AzureCNIConfDir + + # Download Azure VNET CNI plugins. + # Mirror from https://github.com/Azure/azure-container-networking/releases + $zipfile = [Io.path]::Combine("$global:AzureCNIDir", "azure-vnet.zip") + Invoke-WebRequest -Uri $global:VNetCNIPluginsURL -OutFile $zipfile + Expand-Archive -path $zipfile -DestinationPath $global:AzureCNIBinDir + del $zipfile + + # Windows does not need a separate CNI loopback plugin because the Windows + # kernel automatically creates a loopback interface for each network namespace. + # Copy CNI network config file and set bridge mode. + move $global:AzureCNIBinDir/*.conflist $global:AzureCNIConfDir + + # Enable CNI in kubelet. + $global:AzureCNIEnabled = $true +} + +function +Set-AzureNetworkPolicy() +{ + # Azure VNET network policy requires tunnel (hairpin) mode because policy is enforced in the host. + Set-VnetPluginMode "tunnel" +} + +function +Set-NetworkConfig +{ + Write-Log "Configuring networking with NetworkPolicy:$global:NetworkPolicy" + + # Configure network policy. + if ($global:NetworkPolicy -eq "azure") { + Install-VnetPlugins + Set-AzureNetworkPolicy + } +} + function Write-KubernetesStartFiles($podCIDR) { @@ -184,12 +245,15 @@ c:\k\kubelet.exe --hostname-override=`$global:AzureHostname --pod-infra-containe $KubeletCommandLine += " --api-servers=https://`${global:MasterIP}:443" } - # network plugin config - $KubeletCommandLine += " --network-plugin=cni --cni-bin-dir=`$global:CNIPath --cni-conf-dir `$global:CNIPath\config" - # more time is needed to pull windows server images $KubeletCommandLine += " --image-pull-progress-deadline=20m --cgroups-per-qos=false --enforce-node-allocatable=`"`"" $KubeletCommandLine += " --volume-plugin-dir=`$global:VolumePluginDir" + # Configure kubelet to use CNI plugins if enabled. + if ($global:AzureCNIEnabled) { + $KubeletCommandLine += $global:AzureCNIKubeletOptions + } else { + $KubeletCommandLine += $global:WindowsCNIKubeletOptions + } $KubeletArgListStr = "`"" + ($KubeletArgList -join "`",`"") + "`"" @@ -208,7 +272,19 @@ c:\k\kubelet.exe --hostname-override=`$global:AzureHostname --pod-infra-containe `$global:CNIConfig = "$global:CNIConfig" `$global:HNSModule = "$global:HNSModule" `$global:VolumePluginDir = "$global:VolumePluginDir" +`$global:NetworkPolicy="$global:NetworkPolicy" +"@ + + if ($global:NetworkPolicy -eq "azure") { + $kubeStartStr += @" +Write-Host "NetworkPolicy azure, starting kubelet." +$KubeletCommandLine + +"@ + } else { + $kubeStartStr += @" + function Get-DefaultGateway(`$CIDR) { @@ -233,29 +309,29 @@ Update-CNIConfig(`$podCIDR, `$masterSubnetGW) { `$jsonSampleConfig = "{ - ""cniVersion"": ""0.2.0"", - ""name"": """", - ""type"": ""wincni.exe"", - ""master"": ""Ethernet"", - ""capabilities"": { ""portMappings"": true }, - ""ipam"": { - ""environment"": ""azure"", - ""subnet"":"""", - ""routes"": [{ + ""cniVersion"": ""0.2.0"", + ""name"": """", + ""type"": ""wincni.exe"", + ""master"": ""Ethernet"", + ""capabilities"": { ""portMappings"": true }, + ""ipam"": { + ""environment"": ""azure"", + ""subnet"":"""", + ""routes"": [{ ""GW"":"""" - }] - }, - ""dns"" : { + }] + }, + ""dns"" : { ""Nameservers"" : [ """" ] - }, - ""AdditionalArgs"" : [ + }, + ""AdditionalArgs"" : [ { - ""Name"" : ""EndpointPolicy"", ""Value"" : { ""Type"" : ""OutBoundNAT"", ""ExceptionList"": [ """", """" ] } + ""Name"" : ""EndpointPolicy"", ""Value"" : { ""Type"" : ""OutBoundNAT"", ""ExceptionList"": [ """", """" ] } }, { - ""Name"" : ""EndpointPolicy"", ""Value"" : { ""Type"" : ""ROUTE"", ""DestinationPrefix"": """", ""NeedEncap"" : true } + ""Name"" : ""EndpointPolicy"", ""Value"" : { ""Type"" : ""ROUTE"", ""DestinationPrefix"": """", ""NeedEncap"" : true } } - ] + ] }" `$configJson = ConvertFrom-Json `$jsonSampleConfig @@ -329,8 +405,11 @@ try catch { Write-Error `$_ -} +} + "@ + } + $kubeStartStr | Out-File -encoding ASCII -filepath $global:KubeletStartFile $kubeProxyStartStr = @" @@ -430,6 +509,9 @@ try Write-Log "Create the Pause Container kubletwin/pause" New-InfraContainer + Write-Log "Configure networking" + Set-NetworkConfig + Write-Log "write kubelet startfile with pod CIDR of $podCIDR" Write-KubernetesStartFiles $podCIDR @@ -450,4 +532,4 @@ try catch { Write-Error $_ -} +} \ No newline at end of file diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index 19855f8b63..b80f43ec7a 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -38,6 +38,8 @@ const ( DefaultInternalLbStaticIPOffset = 10 // NetworkPolicyNone is the string expression for no network policy NetworkPolicyNone = "none" + // NetworkPolicyAzure is the string expression for Azure CNI network policy + NetworkPolicyAzure = "azure" // NetworkPluginKubenet is the string expression for kubenet network plugin NetworkPluginKubenet = "kubenet" // DefaultNetworkPolicy defines the network policy to use by default diff --git a/pkg/acsengine/defaults-kubelet.go b/pkg/acsengine/defaults-kubelet.go index aca32aa381..9fef1dcb5d 100644 --- a/pkg/acsengine/defaults-kubelet.go +++ b/pkg/acsengine/defaults-kubelet.go @@ -30,8 +30,6 @@ func setKubeletConfig(cs *api.ContainerService) { for key, val := range staticLinuxKubeletConfig { staticWindowsKubeletConfig[key] = val } - // Windows kubelet config overrides - staticWindowsKubeletConfig["--network-plugin"] = NetworkPluginKubenet // Default Kubelet config defaultKubeletConfig := map[string]string{ diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 14865f6f99..c4fc9d72c0 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -17,7 +17,7 @@ const ( // AzureCniPluginVer specifies version of Azure CNI plugin, which has been mirrored from // https://github.com/Azure/azure-container-networking/releases/download/${AZURE_PLUGIN_VER}/azure-vnet-cni-linux-amd64-${AZURE_PLUGIN_VER}.tgz // to https://acs-mirror.azureedge.net/cni/ - AzureCniPluginVer = "v1.0.1" + AzureCniPluginVer = "v1.0.2" ) var ( @@ -313,7 +313,9 @@ func setOrchestratorDefaults(cs *api.ContainerService) { o.KubernetesConfig.EtcdVersion = DefaultEtcdVersion } if a.HasWindows() { - o.KubernetesConfig.NetworkPolicy = DefaultNetworkPolicyWindows + if o.KubernetesConfig.NetworkPolicy == "" { + o.KubernetesConfig.NetworkPolicy = DefaultNetworkPolicyWindows + } } else { if o.KubernetesConfig.NetworkPolicy == "" { o.KubernetesConfig.NetworkPolicy = DefaultNetworkPolicy diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 4c0d2263ff..7d77133d7b 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -699,7 +699,7 @@ func (a *Properties) validateNetworkPolicy() error { } // Temporary safety check, to be removed when Windows support is added. - if (networkPolicy == "calico" || networkPolicy == "azure") && a.HasWindows() { + if (networkPolicy == "calico") && a.HasWindows() { return fmt.Errorf("networkPolicy '%s' is not supporting windows agents", networkPolicy) }