Merge pull request #282 from daschott/master

updating CNI binaries+scripts for flannel host-gw

Kubernetes/flannel/l2bridge/ReadMe.md

@@ -7,7 +7,7 @@
 
 ## Instructions
 
-A more detailed version of these instructions can be found [here](https://1drv.ms/w/s!AgH65RVQdrbiglNr7P7P4VrO8Rxr).
+A more detailed version of these instructions can be found [here](https://docs.microsoft.com/en-us/virtualization/windowscontainers/kubernetes/getting-started-kubernetes-windows).
 
 #### 1. Install Docker
 ```
@@ -27,6 +27,7 @@ mkdir C:\k
   * Copy Kubeconfig file `$HOME/.kube/config` or `/etc/kubernetes/admin.conf` from Kubernetes Master and save as `config` into `C:\k`
   * Ensure the cluster CIDR (e.g. "10.244.0.0/16") is correct in:
     * [net-conf.json](./net-conf.json)
+  * Use `docker pull` to download images from `mcr.microsoft.com/windows/nanoserver` matching your host OS version.
 
 #### 4. Join the Kubernetes cluster:
 ```
@@ -42,9 +43,9 @@ Where:
 #### 5. Deploy an [example Windows service](./manifests/simpleweb.yml) (make sure container image matches host OS)
 
 ## Temp Binaries that will be removed soon
-There are several pending PRs, because of which the bins are published here
+There are several pending PRs, because of which the bins are published here. We are planning to transition to a new CNI repo separately from Microsoft/SDN.
 [host-gw: add windows support](https://github.com/coreos/flannel/pull/921)
-* flanned.exe - 
+* flanneld.exe - 
 
 [Windows CNI for overlay (vxlan) and host-gw (l2bridge) modes](https://github.com/containernetworking/plugins/pull/85)
 * cni\flannel.exe - 

Kubernetes/flannel/l2bridge/cni/config/cni.conf

@@ -3,7 +3,7 @@
     "name":  "cbr0",
     "type":  "flannel",
     "delegate":  {
-                     "type":  "l2bridge",
+                     "type":  "win-bridge",
                      "dns":  {
                                  "Nameservers":  [
                                                      "10.96.0.10"
@@ -12,34 +12,34 @@
                                                 "svc.cluster.local"
                                             ]
                              },
-                     "AdditionalArgs":  [
-                                            {
-                                                "Name":  "EndpointPolicy",
-                                                "Value":  {
-                                                              "Type":  "OutBoundNAT",
-                                                              "ExceptionList":  [
-                                                                                    "10.244.0.0/16",
-                                                                                    "10.96.0.0/12",
-                                                                                    "10.127.130.0/24"
-                                                                                ]
-                                                          }
-                                            },
-                                            {
-                                                "Name":  "EndpointPolicy",
-                                                "Value":  {
-                                                              "Type":  "ROUTE",
-                                                              "DestinationPrefix":  "10.96.0.0/12",
-                                                              "NeedEncap":  true
-                                                          }
-                                            },
-                                            {
-                                                "Name":  "EndpointPolicy",
-                                                "Value":  {
-                                                              "Type":  "ROUTE",
-                                                              "DestinationPrefix":  "10.127.130.36/32",
-                                                              "NeedEncap":  true
-                                                          }
-                                            }
-                                        ]
+                     "policies":  [
+                                      {
+                                          "Name":  "EndpointPolicy",
+                                          "Value":  {
+                                                        "Type":  "OutBoundNAT",
+                                                        "ExceptionList":  [
+                                                                              "10.244.0.0/16",
+                                                                              "10.96.0.0/12",
+                                                                              "10.127.130.0/24"
+                                                                          ]
+                                                    }
+                                      },
+                                      {
+                                          "Name":  "EndpointPolicy",
+                                          "Value":  {
+                                                        "Type":  "ROUTE",
+                                                        "DestinationPrefix":  "10.96.0.0/12",
+                                                        "NeedEncap":  true
+                                                    }
+                                      },
+                                      {
+                                          "Name":  "EndpointPolicy",
+                                          "Value":  {
+                                                        "Type":  "ROUTE",
+                                                        "DestinationPrefix":  "10.127.130.38/32",
+                                                        "NeedEncap":  true
+                                                    }
+                                      }
+                                  ]
                  }
 }

Kubernetes/flannel/l2bridge/manifests/simpleweb.yml

@@ -29,7 +29,7 @@ spec:
     spec:
       containers:
       - name: windowswebserver
-        image: microsoft/windowsservercore:1803
+        image: mcr.microsoft.com/windows/servercore:1809
         command:
         - powershell.exe
         - -command

Kubernetes/flannel/l2bridge/reset.ps1

@@ -0,0 +1,12 @@
+# Before running this script, you should unschedule Kubernetes resources from the node on which the script will be executed.
+C:\k\stop.ps1
+docker stop $(docker ps -aq)
+docker rm -f $(docker ps -aq)
+
+Get-HNSEndpoint | Remove-HNSEndpoint
+Get-HNSNetwork | ? Name -Like "cbr0" | Remove-HNSNetwork
+Remove-Item -Recurse -Force C:\var
+Remove-Item -Recurse -Force C:\usr
+Remove-Item -Recurse -Force C:\run
+Remove-Item -Recurse -Force C:\etc
+Remove-Item -Recurse -Force C:\flannel

Kubernetes/flannel/l2bridge/start-kubelet.ps1

@@ -3,6 +3,8 @@ Param(
     [parameter(Mandatory = $false)] $KubeDnsServiceIP="10.96.0.10",
     [parameter(Mandatory = $false)] $serviceCIDR="10.96.0.0/12",
     [parameter(Mandatory = $false)] $KubeDnsSuffix="svc.cluster.local",
+    [parameter(Mandatory = $false)] $InterfaceName="Ethernet",
+    [parameter(Mandatory = $false)] $LogDir = "C:\k",
     [ValidateSet("process", "hyperv")] $IsolationType="process",
     $NetworkName = "cbr0",
     [switch] $RegisterOnly
@@ -16,7 +18,7 @@ $CNIPath = [Io.path]::Combine($WorkingDir , "cni")
 $CNIConfig = [Io.path]::Combine($CNIPath, "config", "cni.conf")
 
 $endpointName = "cbr0"
-$vnicName = "vEthernet ($endpointName)"
+$vnicName = "v$InterfaceName ($endpointName)"
 
 function
 IsNodeRegistered()
@@ -99,7 +101,7 @@ ConvertTo-MaskLength
 function
 Get-MgmtSubnet
 {
-    $na = Get-NetAdapter | ? Name -Like "vEthernet (Ethernet*"
+    $na = Get-NetAdapter | ? Name -Like "v$InterfaceName (Ethernet*"
     if (!$na) {
       throw "Failed to find a suitable network adapter, check your network settings."
     }
@@ -118,12 +120,12 @@ Update-CNIConfig($podCIDR)
   "name": "<NetworkMode>",
   "type": "flannel",
   "delegate": {
-     "type": "l2bridge",
+     "type": "win-bridge",
       "dns" : {
         "Nameservers" : [ "10.96.0.10" ],
         "Search": [ "svc.cluster.local" ]
       },
-      "AdditionalArgs" : [
+      "policies" : [
         {
           "Name" : "EndpointPolicy", "Value" : { "Type" : "OutBoundNAT", "ExceptionList": [ "<ClusterCIDR>", "<ServerCIDR>", "<MgmtSubnet>" ] }
         },
@@ -140,15 +142,16 @@ Update-CNIConfig($podCIDR)
 
     $configJson =  ConvertFrom-Json $jsonSampleConfig
     $configJson.name = "cbr0"
+    $configJson.delegate.type = "win-bridge"
     $configJson.delegate.dns.Nameservers[0] = $KubeDnsServiceIP
     $configJson.delegate.dns.Search[0] = $KubeDnsSuffix
 
-    $configJson.delegate.AdditionalArgs[0].Value.ExceptionList[0] = $clusterCIDR
-    $configJson.delegate.AdditionalArgs[0].Value.ExceptionList[1] = $serviceCIDR
-    $configJson.delegate.AdditionalArgs[0].Value.ExceptionList[2] = Get-MgmtSubnet
+    $configJson.delegate.policies[0].Value.ExceptionList[0] = $clusterCIDR
+    $configJson.delegate.policies[0].Value.ExceptionList[1] = $serviceCIDR
+    $configJson.delegate.policies[0].Value.ExceptionList[2] = Get-MgmtSubnet
 
-    $configJson.delegate.AdditionalArgs[1].Value.DestinationPrefix  = $serviceCIDR
-    $configJson.delegate.AdditionalArgs[2].Value.DestinationPrefix  = "$(Get-MgmtIpAddress)/32"
+    $configJson.delegate.policies[1].Value.DestinationPrefix  = $serviceCIDR
+    $configJson.delegate.policies[2].Value.DestinationPrefix  = "$(Get-MgmtIpAddress)/32"
 
     if (Test-Path $CNIConfig) {
         Clear-Content -Path $CNIConfig
@@ -175,7 +178,7 @@ if ($IsolationType -ieq "process")
         --cluster-dns=$KubeDnsServiceIp --cluster-domain=cluster.local `
         --kubeconfig=c:\k\config --hairpin-mode=promiscuous-bridge `
         --image-pull-progress-deadline=20m --cgroups-per-qos=false `
-        --enforce-node-allocatable="" `
+        --log-dir=$LogDir --logtostderr=false --enforce-node-allocatable="" `
         --network-plugin=cni --cni-bin-dir="c:\k\cni" --cni-conf-dir "c:\k\cni\config"
 }
 elseif ($IsolationType -ieq "hyperv")
@@ -187,5 +190,6 @@ elseif ($IsolationType -ieq "hyperv")
         --kubeconfig=c:\k\config --hairpin-mode=promiscuous-bridge `
         --image-pull-progress-deadline=20m --cgroups-per-qos=false `
         --feature-gates=HyperVContainer=true --enforce-node-allocatable="" `
+        --log-dir=$LogDir --logtostderr=false `
         --network-plugin=cni --cni-bin-dir="c:\k\cni" --cni-conf-dir "c:\k\cni\config"
 }

Kubernetes/flannel/l2bridge/start-kubeproxy.ps1

@@ -1,8 +1,9 @@
 Param(
+    [parameter(Mandatory = $false)] $LogDir = "C:\k",
     $NetworkName = "cbr0"
 )
 
 $env:KUBE_NETWORK=$NetworkName.ToLower()
 ipmo c:\k\hns.psm1
 Get-HnsPolicyList | Remove-HnsPolicyList
-c:\k\kube-proxy.exe --v=4 --proxy-mode=kernelspace --hostname-override=$(hostname) --kubeconfig=c:\k\config
+c:\k\kube-proxy.exe --v=4 --proxy-mode=kernelspace --hostname-override=$(hostname) --kubeconfig=c:\k\config --log-dir=$LogDir --logtostderr=false

Kubernetes/flannel/l2bridge/start.ps1

@@ -3,27 +3,38 @@
     [parameter(Mandatory = $true)] $ManagementIP,
     [parameter(Mandatory = $true)] $KubeDnsServiceIP,
     [parameter(Mandatory = $true)] $ServiceCIDR,
+    [parameter(Mandatory = $false)] $InterfaceName="Ethernet",
+    [parameter(Mandatory = $false)] $LogDir = "C:\k",
     [ValidateSet("process", "hyperv")] $IsolationType = "process"
 )
 
-function DownloadFlannelBinaries()
+function SetupDirectories()
 {
+    md $BaseDir -ErrorAction Ignore
+    md $LogDir -ErrorAction Ignore
     md c:\flannel -ErrorAction Ignore
-    DownloadFile -Url  "https://github.com/Microsoft/SDN/raw/master/Kubernetes/flannel/l2bridge/flanneld.exe" -Destination c:\flannel\flanneld.exe
+    md $BaseDir\cni\config -ErrorAction Ignore
+    md C:\etc\kube-flannel -ErrorAction Ignore
+}
+
+function CopyFiles(){
+    cp $BaseDir\flanneld.exe c:\flannel\flanneld.exe
+    cp $BaseDir\net-conf.json C:\etc\kube-flannel\net-conf.json
+}
+
+function DownloadFlannelBinaries()
+{
+    Write-Host "Downloading Flannel binaries"
+    DownloadFile -Url  "https://github.com/Microsoft/SDN/raw/master/Kubernetes/flannel/l2bridge/flanneld.exe" -Destination $BaseDir\flanneld.exe 
 }
 
 function DownloadCniBinaries()
 {
     Write-Host "Downloading CNI binaries"
-    DownloadFlannelBinaries
-    md $BaseDir\cni\config -ErrorAction Ignore
-    md C:\etc\kube-flannel -ErrorAction Ignore
-
-    DownloadFile -Url  "https://github.com/Microsoft/SDN/raw/master/Kubernetes/flannel/l2bridge/cni/l2bridge.exe" -Destination $BaseDir\cni\l2bridge.exe
     DownloadFile -Url  "https://github.com/Microsoft/SDN/raw/master/Kubernetes/flannel/l2bridge/cni/flannel.exe" -Destination $BaseDir\cni\flannel.exe
+    DownloadFile -Url  "https://github.com/Microsoft/SDN/raw/master/Kubernetes/flannel/l2bridge/cni/win-bridge.exe" -Destination $BaseDir\cni\win-bridge.exe
     DownloadFile -Url  "https://github.com/Microsoft/SDN/raw/master/Kubernetes/flannel/l2bridge/cni/host-local.exe" -Destination $BaseDir\cni\host-local.exe
     DownloadFile -Url  "https://github.com/Microsoft/SDN/raw/master/Kubernetes/flannel/l2bridge/net-conf.json" -Destination $BaseDir\net-conf.json
-    cp $BaseDir\net-conf.json C:\etc\kube-flannel\net-conf.json
 }
 
 function DownloadWindowsKubernetesScripts()
@@ -33,49 +44,53 @@ function DownloadWindowsKubernetesScripts()
     DownloadFile -Url  https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/InstallImages.ps1 -Destination $BaseDir\InstallImages.ps1
     DownloadFile -Url  https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/Dockerfile -Destination $BaseDir\Dockerfile
     DownloadFile -Url  https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/stop.ps1 -Destination $BaseDir\stop.ps1
-    DownloadFile -Url  https://github.com/Microsoft/SDN/raw/master/Kubernetes/flannel/l2bridge/start-kubelet.ps1 -Destination $BaseDir\start-Kubelet.ps1 
-    DownloadFile -Url  https://github.com/Microsoft/SDN/raw/master/Kubernetes/flannel/l2bridge/start-kubeproxy.ps1 -Destination $BaseDir\start-Kubeproxy.ps1
 }
 
 function DownloadAllFiles()
 {
+    DownloadFlannelBinaries
     DownloadCniBinaries
-    DownloadWindowsKubernetesScripts
 }
 
+# Setup directories
 $BaseDir = "c:\k"
-md $BaseDir -ErrorAction Ignore
+SetupDirectories
+
 $helper = "c:\k\helper.psm1"
 if (!(Test-Path $helper))
 {
     Start-BitsTransfer https://raw.githubusercontent.com/Microsoft/SDN/master/Kubernetes/windows/helper.psm1 -Destination c:\k\helper.psm1
 }
 ipmo $helper
 
-# Download All the files
+DownloadWindowsKubernetesScripts
+
+# Download All the files, if needed
 DownloadAllFiles
+CopyFiles
 
 # Prepare POD infra Images
 start powershell $BaseDir\InstallImages.ps1
 
 # Prepare Network & Start Infra services
 $NetworkMode = "L2Bridge"
 $NetworkName = "cbr0"
-
 CleanupOldNetwork $NetworkName
-
 powershell $BaseDir\start-kubelet.ps1 -RegisterOnly
-
 ipmo C:\k\hns.psm1
 
-# Create a L2Bridge to trigger a vSwitch creation. Do this only once
+# Create a L2Bridge to trigger a vSwitch creation. Do this only once as it causes network blip
 if(!(Get-HnsNetwork | ? Name -EQ "External"))
 {
     New-HNSNetwork -Type $NetworkMode -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name "External" -Verbose
 }
-
+# Start Flanneld
+Start-Sleep 5
 StartFlanneld -ipaddress $ManagementIP -NetworkName $NetworkName
 
-Start powershell -ArgumentList "-File $BaseDir\start-kubelet.ps1 -clusterCIDR $ClusterCIDR -KubeDnsServiceIP $KubeDnsServiceIP -serviceCIDR $ServiceCIDR -IsolationType $IsolationType -NetworkName $NetworkName"
+# Start kubelet
+Start powershell -ArgumentList "-File $BaseDir\start-kubelet.ps1 -clusterCIDR $ClusterCIDR -KubeDnsServiceIP $KubeDnsServiceIP -serviceCIDR $ServiceCIDR -InterfaceName $InterfaceName -LogDir $LogDir -IsolationType $IsolationType -NetworkName $NetworkName"
 Start-Sleep 10
-start powershell -ArgumentList " -File $BaseDir\start-kubeproxy.ps1 -NetworkName $NetworkName"
+
+# Start kube-proxy
+start powershell -ArgumentList " -File $BaseDir\start-kubeproxy.ps1 -NetworkName $NetworkName -LogDir $LogDir"

Kubernetes/flannel/overlay/README.md

@@ -18,11 +18,10 @@ There are several pending PRs, because of which the bins are published here
 * Pod to Pod connectivity will work (Windows to Windows)
 * Outbound Internet connectivity will work
 * Node port access
-
+* Service Vip access
 
 # Pending Validation
 * Kubeproxy support for Overlay mode
-* Service Vip access
 
 # What will not work
 * Pod to Pod connectivity (Linux to Windows)

Kubernetes/windows/Dockerfile

@@ -1,3 +1,3 @@
-FROM microsoft/nanoserver
+FROM mcr.microsoft.com/windows/nanoserver
 
 CMD cmd /c ping -t localhost

Kubernetes/windows/InstallImages.ps1

@@ -2,20 +2,23 @@
 $BaseDir = "c:\k"
 
 # Prepare POD infra Images
-
-if (!(docker images microsoft/nanoserver:latest -q))
+function tagImage
 {
-    docker tag (docker images microsoft/nanoserver -q) microsoft/nanoserver
-}
+    if (!(docker images mcr.microsoft.com/windows/nanoserver:latest -q))
+    {
+        docker tag (docker images mcr.microsoft.com/windows/nanoserver -q) mcr.microsoft.com/windows/nanoserver
+    }
 
-if (!(docker images microsoft/windowsservercore:latest -q))
-{
-    docker tag (docker images microsoft/windowsservercore -q) microsoft/windowsservercore
+    if (!(docker images mcr.microsoft.com/windows/servercore:latest -q))
+    {
+        docker tag (docker images mcr.microsoft.com/windows/servercore -q) mcr.microsoft.com/windows/servercore
+    }
 }
 
 $infraPodImage=docker images kubeletwin/pause -q
 if (!$infraPodImage)
 {
+    tagImage
     pushd
     cd $BaseDir
     docker build -t kubeletwin/pause .

Kubernetes/windows/debug/collectlogs.ps1

@@ -39,6 +39,5 @@ powershell $BaseDir\dumpVfpPolicies.ps1 -switchName $Network -outfile vfpOutput.
 
 ipconfig /allcompartments /all > ip.txt
 route print > routes.txt
-netsh int ipv4 sh int > mtu.txt
 popd
 Write-Host "Logs are available at $outDir"

Kubernetes/windows/debug/startpacketcapture.cmd

@@ -1,2 +1,2 @@
-Cmd /c """netsh trace start globallevel=6 provider={0c885e0d-6eb6-476c-a048-2457eed3a5c1}  provider=Microsoft-Windows-TCPIP level=5 provider={80CE50DE-D264-4581-950D-ABADEEE0D340} provider={D0E4BC17-34C7-43fc-9A72-D89A59D6979A} provider={93f693dc-9163-4dee-af64-d855218af242} provider={564368D6-577B-4af5-AD84-1C54464848E6} scenario=Virtualization provider=Microsoft-Windows-Hyper-V-VfpExt capture=yes captureMultilayer=yes capturetype=both provider=microsoft-windows-winnat provider={AE3F6C6D-BF2A-4291-9D07-59E661274EE3} keywords=0xffffffff level=6 provider={9B322459-4AD9-4F81-8EEA-DC77CDD18CA6} keywords=0xffffffff level=6 provider={0c885e0d-6eb6-476c-a048-2457eed3a5c1} level=6 report=disabled tracefile=c:\server.etl overwrite=yes persistent=yes"""
+Cmd /c """netsh trace start globallevel=6 provider={0c885e0d-6eb6-476c-a048-2457eed3a5c1}  provider=Microsoft-Windows-TCPIP level=5 provider={80CE50DE-D264-4581-950D-ABADEEE0D340} provider={D0E4BC17-34C7-43fc-9A72-D89A59D6979A} provider={93f693dc-9163-4dee-af64-d855218af242} provider={564368D6-577B-4af5-AD84-1C54464848E6} scenario=Virtualization provider=Microsoft-Windows-Hyper-V-VfpExt capture=yes provider=microsoft-windows-winnat provider={AE3F6C6D-BF2A-4291-9D07-59E661274EE3} keywords=0xffffffff level=6 provider={9B322459-4AD9-4F81-8EEA-DC77CDD18CA6} keywords=0xffffffff level=6 provider={0c885e0d-6eb6-476c-a048-2457eed3a5c1} level=6 report=disabled tracefile=c:\server.etl overwrite=yes persistent=yes"""