kubernetes-sigs · k8s-ci-robot · Nov 10, 2020 · Oct 26, 2020
diff --git a/kubeadm/flannel/flannel-host-gw.yml b/kubeadm/flannel/flannel-host-gw.yml
@@ -17,15 +17,34 @@ data:
     mkdir -force /host/k/flannel
     mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount
 
-    $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json
+    $containerRuntime = "docker"
+    if (Test-Path /host/etc/cni/net.d/0-containerd-nat.json) {
+      $containerRuntime = "containerd"
+    }
+
+    Write-Host "Configuring CNI for $containerRuntime"
+
     $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet
     $podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet
     $networkJson = wins cli net get | convertfrom-json
 
-    $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet, $networkJson.SubnetCIDR
-    $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet
-    $cniJson.delegate.policies[2].Value.DestinationPrefix = $networkJson.AddressCIDR
-    Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
+    if ($containerRuntime -eq "docker") {
+      $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json
+
+      $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet, $networkJson.SubnetCIDR
+      $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet
+      $cniJson.delegate.policies[2].Value.DestinationPrefix = $networkJson.AddressCIDR
+
+      Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
+    } elseif ($containerRuntime -eq "containerd") {
+      $cniJson = get-content /etc/kube-flannel-windows/cni-conf-containerd.json | ConvertFrom-Json
+
+      $cniJson.delegate.AdditionalArgs[0].Value.Settings.Exceptions = $serviceSubnet, $podSubnet, $networkJson.SubnetCIDR
+      $cniJson.delegate.AdditionalArgs[1].Value.Settings.DestinationPrefix = $serviceSubnet
+      $cniJson.delegate.AdditionalArgs[2].Value.Settings.DestinationPrefix = $networkJson.AddressCIDR
+
+      Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
+    }
 
     cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel
     cp -force -recurse /cni/* /host/opt/cni/bin
@@ -74,6 +93,53 @@ data:
         ]
       }
     }
+  cni-conf-containerd.json: |
+    {
+      "cniVersion": "0.2.0",
+      "name": "cbr0",
+      "type": "flannel",
+      "capabilities": {
+        "portMappings": true,
+        "dns": true
+      },
+      "delegate": {
+        "type": "sdnbridge",
+        "optionalFlags" : {
+          "forceBridgeGateway" : true
+        },
+        "AdditionalArgs": [
+          {
+            "Name": "EndpointPolicy",
+            "Value": {
+              "Type": "OutBoundNAT",
+              "Settings": {
+                "Exceptions": []
+              }
+            }
+          },
+          {
+            "Name": "EndpointPolicy",
+            "Value": {
+              "Type": "SDNROUTE",
+              "Settings": {
+                "DestinationPrefix": "",
+                "NeedEncap": true
+              }
+            }
+          },
+          {
+            "Name": "EndpointPolicy",
+            "Value": {
+              "Type": "SDNROUTE",
+              "Settings": {
+                "DestinationPrefix": "",
+                "NeedEncap": true
+              }
+            }
+          }
+        ]
+      }
+    }
 ---
 apiVersion: apps/v1
 kind: DaemonSet

diff --git a/kubeadm/flannel/flannel-overlay.yml b/kubeadm/flannel/flannel-overlay.yml
@@ -17,20 +17,40 @@ data:
     mkdir -force /host/k/flannel
     mkdir -force /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount
 
-    $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json
+    $containerRuntime = "docker"
+    if (Test-Path /host/etc/cni/net.d/0-containerd-nat.json) {
+      $containerRuntime = "containerd"
+    }
+
+    Write-Host "Configuring CNI for $containerRuntime"
+
     $serviceSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.serviceSubnet
     $podSubnet = yq r /etc/kubeadm-config/ClusterConfiguration networking.podSubnet
     $networkJson = wins cli net get | convertfrom-json
 
-    $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet
-    $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet
-    Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
+    if ($containerRuntime -eq "docker") {
+      $cniJson = get-content /etc/kube-flannel-windows/cni-conf.json | ConvertFrom-Json
+
+      $cniJson.delegate.policies[0].Value.ExceptionList = $serviceSubnet, $podSubnet
+      $cniJson.delegate.policies[1].Value.DestinationPrefix = $serviceSubnet
+
+      Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
+    } elseif ($containerRuntime -eq "containerd") {
+      $cniJson = get-content /etc/kube-flannel-windows/cni-conf-containerd.json | ConvertFrom-Json
+
+      $cniJson.delegate.AdditionalArgs[0].Value.Settings.Exceptions = $serviceSubnet, $podSubnet
+      $cniJson.delegate.AdditionalArgs[1].Value.Settings.DestinationPrefix = $serviceSubnet
+      $cniJson.delegate.AdditionalArgs[2].Value.Settings.ProviderAddress = $networkJson.AddressCIDR.Split('/')[0]
+
+      Set-Content -Path /host/etc/cni/net.d/10-flannel.conf ($cniJson | ConvertTo-Json -depth 100)
+    }
 
     cp -force /etc/kube-flannel/net-conf.json /host/etc/kube-flannel
     cp -force -recurse /cni/* /host/opt/cni/bin
     cp -force /k/flannel/* /host/k/flannel/
     cp -force /kube-proxy/kubeconfig.conf /host/k/flannel/kubeconfig.yml
     cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/k/flannel/var/run/secrets/kubernetes.io/serviceaccount/
+
     wins cli process run --path /k/flannel/setup.exe --args "--mode=overlay --interface=Ethernet"
     wins cli route add --addresses 169.254.169.254
     wins cli process run --path /k/flannel/flanneld.exe --args "--kube-subnet-mgr --kubeconfig-file /k/flannel/kubeconfig.yml" --envs "POD_NAME=$env:POD_NAME POD_NAMESPACE=$env:POD_NAMESPACE"
@@ -63,6 +83,49 @@ data:
         ]
       }
     }
+  cni-conf-containerd.json: |
+    {
+      "name": "flannel.4096",
+      "cniVersion": "0.2.0",
+      "type": "flannel",
+      "capabilities": {
+        "portMappings": true,
+        "dns": true
+      },
+      "delegate": {
+        "type": "sdnoverlay",
+        "AdditionalArgs": [
+          {
+            "Name": "EndpointPolicy",
+            "Value": {
+              "Type": "OutBoundNAT",
+              "Settings" : {
+                "Exceptions": []
+              }
+            }
+          },
+          {
+            "Name": "EndpointPolicy",
+            "Value": {
+              "Type": "SDNROUTE",
+              "Settings": {
+                "DestinationPrefix": "",
+                "NeedEncap": true
+              }
+            }
+          },
+          {
+            "Name":"EndpointPolicy",
+            "Value":{
+              "Type":"ProviderAddress",
+                "Settings":{
+                    "ProviderAddress":""
+              }
+            }
+          }
+        ]
+      }
+    }
 ---
 apiVersion: apps/v1
 kind: DaemonSet

diff --git a/kubeadm/kube-proxy/kube-proxy.yml b/kubeadm/kube-proxy/kube-proxy.yml
@@ -2,14 +2,38 @@ apiVersion: v1
 data:
   run-script.ps1: |-
     $ErrorActionPreference = "Stop";
+
+    # Get newest cni conf file that is not 0-containerd-nat.json or spin until one shows up.
+    # With Docker the kube-proxy pod should not be scheduled to Windows nodes until host networking is configured.
+    # With contianerD host networking is required to schedule any pod including the CNI pods so a basic nat network is
+    #  configured. This network should not be used by kube-proxy.
+    function Get-NetConfFile {
+      while ($true) {
+        if (Test-Path /host/etc/cni/net.d/) {
+          $files = @()
+          $files += Get-ChildItem -Path /host/etc/cni/net.d/ -Exclude "0-containerd-nat.json"
+
+          if ($files.Length -gt 0) {
+            $file = (($files | Sort-Object LastWriteTime | Select-Object -Last 1).Name)
+            Write-Host "Using CNI conf file: $file"
+            return $file
+          }
+        }
+
+        Write-Host "Waiting for CNI file..."
+        Start-Sleep 10
+      }
+    }
+
     mkdir -force /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount
     mkdir -force /host/k/kube-proxy
 
     cp -force /k/kube-proxy/* /host/k/kube-proxy
     cp -force /var/lib/kube-proxy/* /host/var/lib/kube-proxy
     cp -force /var/run/secrets/kubernetes.io/serviceaccount/* /host/var/lib/kube-proxy/var/run/secrets/kubernetes.io/serviceaccount #FIXME?
 
-    $networkName = (Get-Content /host/etc/cni/net.d/* | ConvertFrom-Json).name
+    $cniConfFile = Get-NetConfFile
+    $networkName = (Get-Content "/host/etc/cni/net.d/$cniConfFile" | ConvertFrom-Json).name
     $sourceVip = ($env:POD_IP -split "\.")[0..2] + 0 -join "."
     yq w -i /host/var/lib/kube-proxy/config.conf winkernel.sourceVip $sourceVip
     yq w -i /host/var/lib/kube-proxy/config.conf winkernel.networkName $networkName

diff --git a/kubeadm/scripts/Install-Containerd.ps1 b/kubeadm/scripts/Install-Containerd.ps1
@@ -0,0 +1,167 @@
+<#
+.SYNOPSIS
+Installs ContainerD on a Windows machines in preperation for joining the node to a Kubernetes cluster.
+
+.DESCRIPTION
+This script
+- Verifies that Windows Features requried for running contianers are enabled (and enables then if they are not)
+- Downloads ContainerD binaries from from at the version specified.
+- Downloads Windows SND CNI plugins.
+- Sets up a basic nat networking config for ContainerD to use until another CNI is configured
+- Registers ContainerD as a windows service.
+
+.PARAMETER ContainerDVersion
+ContainerD version to download and use.
+
+.PARAMETER netAdapterName
+Name of network adapter to use when configuring basic nat network.
+
+.EXAMPLE
+PS> .\Install-Conatinerd.ps1
+
+#>
+
+Param(
+    [parameter(HelpMessage = "ContainerD version to use")]
+    [string] $ContainerDVersion = "1.4.1",
+    [parameter(HelpMessage = "Name of network adapter to use when configuring basic nat network")]
+    [string] $netAdapterName = "Ethernet"
+)
+
+$ErrorActionPreference = 'Stop'
+
+function DownloadFile($destination, $source) {
+    Write-Host("Downloading $source to $destination")
+    curl.exe --silent --fail -Lo $destination $source
+
+    if (!$?) {
+        Write-Error "Download $source failed"
+        exit 1
+    }
+}
+
+<#
+.DESCRIPTION
+Computes a subnet for a gateway from the IPv4 IPAddress and PrefixLength properties
+for a given network adapter. This value is used for IPAM in a nat CNI config required for
+containerd.
+
+.NOTES 
+This logic is adapted from 
+https://github.com/containerd/containerd/blob/4a6b47d470d9f2dfc3d49f2819b968861dfa123e/script/setup/install-cni-windows
+
+.EXAMPLE
+PS> CalculateSubNet -gateway 172.16.5.8 -prefixLength 24
+172.16.5.0/8
+#>
+function CalculateSubNet {
+    param (
+        [string]$gateway,
+        [int]$prefixLength
+    )
+    $len = $prefixLength
+    $parts = $gateway.Split('.')
+    $result = @()
+    for ($i = 0; $i -le 3; $i++) {
+        if ($len -ge 8) {
+            $mask = 255
+
+        }
+        elseif ($len -gt 0) {
+            $mask = ((256 - 2 * (8 - $len)))
+        }
+        else {
+            $mask = 0
+        }
+        $len -= 8
+        $result += ([int]$parts[$i] -band $mask)
+    }
+
+    $subnetIp = [string]::Join('.', $result)
+    $cidr = 32 - $prefixLength
+    return "${subnetIp}/$cidr"
+}
+
+$requiredWindowsFeatures = @(
+    "Containers",
+    "Hyper-V",
+    "Hyper-V-PowerShell")
+
+function ValidateWindowsFeatures {
+    $allFeaturesInstalled = $true
+    foreach ($feature in $requiredWindowsFeatures) {
+        $f = Get-WindowsFeature -Name $feature
+        if (-not $f.Installed) {
+            Write-Warning "Windows feature: '$feature' is not installed."
+            $allFeaturesInstalled = $false
+        }
+    }
+    return $allFeaturesInstalled
+}
+
+if (-not (ValidateWindowsFeatures)) {
+    Write-Output "Installing required windows features..."
+
+    foreach ($feature in $requiredWindowsFeatures) {
+        Install-WindowsFeature -Name $feature
+    }
+
+    Write-Output "Please reboot and re-run this script."
+    exit 0
+}
+
+Write-Output "Getting ContainerD binaries"
+$global:ConainterDPath = "$env:ProgramFiles\containerd"
+mkdir -Force $global:ConainterDPath | Out-Null
+DownloadFile "$global:ConainterDPath\containerd.tar.gz" https://github.com/containerd/containerd/releases/download/v${ContainerDVersion}/containerd-${ContainerDVersion}-windows-amd64.tar.gz
+tar.exe -xvf "$global:ConainterDPath\containerd.tar.gz" --strip=1 -C $global:ConainterDPath
+$env:Path += ";$global:ConainterDPath"
+[Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
+containerd.exe config default | Out-File "$global:ConainterDPath\config.toml" -Encoding ascii
+#config file fixups
+$config = Get-Content "$global:ConainterDPath\config.toml"
+$config = $config -replace "bin_dir = (.)*$", "bin_dir = `"c:/opt/cni/bin`""
+$config = $config -replace "conf_dir = (.)*$", "conf_dir = `"c:/etc/cni/net.d`""
+$config | Set-Content "$global:ConainterDPath\config.toml" -Force 
+
+mkdir -Force c:\opt\cni\bin | Out-Null
+mkdir -Force c:\etc\cni\net.d | Out-Null
+
+Write-Output "Getting SDN CNI binaries"
+DownloadFile "c:\opt\cni\cni-plugins.zip" https://github.com/microsoft/windows-container-networking/releases/download/v0.2.0/windows-container-networking-cni-amd64-v0.2.0.zip
+Expand-Archive -Path "c:\opt\cni\cni-plugins.zip" -DestinationPath "c:\opt\cni\bin" -Force
+
+Write-Output "Creating network config for nat network"
+$gateway = (Get-NetIPAddress -InterfaceAlias $netAdapterName -AddressFamily IPv4).IPAddress
+$prefixLength = (Get-NetIPAddress -InterfaceAlias $netAdapterName -AddressFamily IPv4).PrefixLength
+
+$subnet = CalculateSubNet -gateway $gateway -prefixLength $prefixLength
+
+@"
+{
+    "cniVersion": "0.2.0",
+    "name": "nat",
+    "type": "nat",
+    "master": "Ethernet",
+    "ipam": {
+        "subnet": "$subnet",
+        "routes": [
+            {
+                "GW": "$gateway"
+            }
+        ]
+    },
+    "capabilities": {
+        "portMappings": true,
+        "dns": true
+    }
+}
+"@ | Set-Content "c:\etc\cni\net.d\0-containerd-nat.json" -Force
+
+Write-Output "Registering ContainerD as a service"
+containerd.exe --register-service
+
+Write-Output "Starting ContainerD service"
+Start-Service containerd
+
+Write-Output "Done - please remember to add '--cri-socket `"npipe:////./pipe/containerd-containerd`"' to your kubeadm join command"