[release-1.24] Backports for 2022-10

docs/adrs/servicelb-ccm.md

@@ -0,0 +1,36 @@
+# Move the ServiceLB load-balancer controller into the K3s cloud provider
+
+Date: 2022-09-29
+
+## Status
+
+Accepted
+
+## Context
+
+K3s includes a stub cloud-provider the implements just enough node lifecycle functionality (the
+`cloudprovider.Instances` interface) to get node addresses set properly, and clear the Uninitialized taint
+that is added to nodes when they first join the cluster. The cloud-provider interface also has extension
+points for load-balancer controllers, but we did not implement these, in favor of running a standalone
+ServiceLB controller that is directly hooked into the core Wrangler controllers.
+
+Because it doesn't make use of the existing load-balancer interface, the ServiceLB controller must implement
+all the logic to watch Services, ensure that it's only handling services of the correct type and state, manage
+finalizers, and so on.  This would all be handled by core Kubernetes code if we implemented the
+`cloudprovider.LoadBalancer` interface.
+
+## Decision
+
+We will move the ServiceLB code into the cloud-controller, as a backend for the LoadBalancer interface
+implementation. Existing behavior for disabling node lifecycle functionality will be retained, such that users
+can still use ServiceLB alongside other cloud-controller-managers that handle node lifecycle. Support for
+customizing ServiceLB behavior via node labels will be retained.
+
+## Consequences
+
+* K3s uses less resources when ServiceLB is disabled, as several core controllers are no longer started
+unconditionally.
+* The `--disable-cloud-controller` flag now disables the CCM's `cloud-node` and `cloud-node-lifecycle`
+controllers that were historically the only supported controllers.
+* The `--disable=servicelb` flag now disables the CCM's `service` controller.
+* If the cloud-controller and servicelb are both disabled, the cloud-controller-manager is not run at all.

manifests/ccm.yaml

@@ -24,21 +24,22 @@ rules:
   resources:
   - nodes
   verbs:
-  - '*'
+  - "*"
 - apiGroups:
   - ""
   resources:
   - nodes/status
+  - services/status
   verbs:
   - patch
 - apiGroups:
   - ""
   resources:
   - services
+  - pods
   verbs:
+  - get
   - list
-  - patch
-  - update
   - watch
 - apiGroups:
   - ""
@@ -49,22 +50,16 @@ rules:
 - apiGroups:
   - ""
   resources:
-  - persistentvolumes
+  - namespaces
   verbs:
+  - create
   - get
-  - list
-  - update
-  - watch
 - apiGroups:
-  - ""
+  - apps
   resources:
-  - endpoints
+  - daemonsets
   verbs:
-  - create
-  - get
-  - list
-  - watch
-  - update
+  - "*"
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

manifests/metrics-server/metrics-server-deployment.yaml

@@ -39,10 +39,10 @@ spec:
         emptyDir: {}
       containers:
       - name: metrics-server
-        image: %{SYSTEM_DEFAULT_REGISTRY}%rancher/mirrored-metrics-server:v0.5.2
+        image: %{SYSTEM_DEFAULT_REGISTRY}%rancher/mirrored-metrics-server:v0.6.1
         args:
         - --cert-dir=/tmp
-        - --secure-port=4443
+        - --secure-port=10250
         - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
         - --kubelet-use-node-status-port
         - --metric-resolution=15s
@@ -52,7 +52,7 @@ spec:
             memory: 70Mi
         ports:
         - name: https
-          containerPort: 4443
+          containerPort: 10250
           protocol: TCP
         readinessProbe:
           httpGet:

manifests/metrics-server/resource-reader.yaml

@@ -4,13 +4,17 @@ kind: ClusterRole
 metadata:
   name: system:metrics-server
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes/metrics
+  verbs:
+  - get
 - apiGroups:
   - ""
   resources:
   - pods
   - nodes
-  - nodes/stats
-  - namespaces
   verbs:
   - get
   - list

manifests/traefik.yaml

@@ -5,15 +5,15 @@ metadata:
   name: traefik-crd
   namespace: kube-system
 spec:
-  chart: https://%{KUBERNETES_API}%/static/charts/traefik-crd-10.19.300.tgz
+  chart: https://%{KUBERNETES_API}%/static/charts/traefik-crd-12.0.000.tgz
 ---
 apiVersion: helm.cattle.io/v1
 kind: HelmChart
 metadata:
   name: traefik
   namespace: kube-system
 spec:
-  chart: https://%{KUBERNETES_API}%/static/charts/traefik-10.19.300.tgz
+  chart: https://%{KUBERNETES_API}%/static/charts/traefik-12.0.000.tgz
   set:
     global.systemDefaultRegistry: "%{SYSTEM_DEFAULT_REGISTRY_RAW}%"
   valuesContent: |-
@@ -33,7 +33,7 @@ spec:
     priorityClassName: "system-cluster-critical"
     image:
       name: "rancher/mirrored-library-traefik"
-      tag: "2.6.2"
+      tag: "2.9.1"
     tolerations:
     - key: "CriticalAddonsOnly"
       operator: "Exists"

pkg/agent/tunnel/tunnel.go

@@ -7,7 +7,9 @@ import (
 	"net"
 	"os"
 	"reflect"
+	"strconv"
 	"sync"
+	"time"
 
 	"github.com/gorilla/websocket"
 	agentconfig "github.com/k3s-io/k3s/pkg/agent/config"
@@ -22,6 +24,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -31,10 +34,11 @@ import (
 )
 
 type agentTunnel struct {
-	client kubernetes.Interface
-	cidrs  cidranger.Ranger
-	ports  map[string]bool
-	mode   string
+	client      kubernetes.Interface
+	cidrs       cidranger.Ranger
+	ports       map[string]bool
+	mode        string
+	kubeletPort string
 }
 
 // explicit interface check
@@ -85,6 +89,9 @@ func Setup(ctx context.Context, config *daemonconfig.Node, proxy proxy.Proxy) er
 		close(apiServerReady)
 	}()
 
+	// Allow the kubelet port, as published via our node object
+	go tunnel.setKubeletPort(ctx, apiServerReady)
+
 	switch tunnel.mode {
 	case daemonconfig.EgressSelectorModeCluster:
 		// In Cluster mode, we allow the cluster CIDRs, and any connections to the node's IPs for pods using host network.
@@ -135,6 +142,23 @@ func Setup(ctx context.Context, config *daemonconfig.Node, proxy proxy.Proxy) er
 	return nil
 }
 
+// setKubeletPort retrieves the configured kubelet port from our node object
+func (a *agentTunnel) setKubeletPort(ctx context.Context, apiServerReady <-chan struct{}) {
+	<-apiServerReady
+
+	wait.PollImmediateWithContext(ctx, time.Second, util.DefaultAPIServerReadyTimeout, func(ctx context.Context) (bool, error) {
+		nodeName := os.Getenv("NODE_NAME")
+		node, err := a.client.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+		if err != nil {
+			logrus.Debugf("Tunnel authorizer failed to get Kubelet Port: %v", err)
+			return false, nil
+		}
+		a.kubeletPort = strconv.FormatInt(int64(node.Status.DaemonEndpoints.KubeletEndpoint.Port), 10)
+		logrus.Infof("Tunnel authorizer set Kubelet Port %s", a.kubeletPort)
+		return true, nil
+	})
+}
+
 func (a *agentTunnel) clusterAuth(config *daemonconfig.Node) {
 	// In Cluster mode, we add static entries for the Node IPs and Cluster CIDRs
 	for _, ip := range config.AgentConfig.NodeIPs {
@@ -304,7 +328,7 @@ func (a *agentTunnel) authorized(ctx context.Context, proto, address string) boo
 	logrus.Debugf("Tunnel authorizer checking dial request for %s", address)
 	host, port, err := net.SplitHostPort(address)
 	if err == nil {
-		if proto == "tcp" && daemonconfig.KubeletReservedPorts[port] && (host == "127.0.0.1" || host == "::1") {
+		if a.isKubeletPort(proto, host, port) {
 			return true
 		}
 		if ip := net.ParseIP(host); ip != nil {
@@ -359,3 +383,8 @@ func (a *agentTunnel) connect(rootCtx context.Context, waitGroup *sync.WaitGroup
 
 	return cancel
 }
+
+// isKubeletPort returns true if the connection is to a reserved TCP port on a loopback address.
+func (a *agentTunnel) isKubeletPort(proto, host, port string) bool {
+	return proto == "tcp" && (host == "127.0.0.1" || host == "::1") && (port == a.kubeletPort || port == daemonconfig.StreamServerPort)
+}

pkg/cli/server/server.go

@@ -115,8 +115,8 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	serverConfig.ControlConfig.DataDir = cfg.DataDir
 	serverConfig.ControlConfig.KubeConfigOutput = cfg.KubeConfigOutput
 	serverConfig.ControlConfig.KubeConfigMode = cfg.KubeConfigMode
-	serverConfig.Rootless = cfg.Rootless
-	serverConfig.ServiceLBNamespace = cfg.ServiceLBNamespace
+	serverConfig.ControlConfig.Rootless = cfg.Rootless
+	serverConfig.ControlConfig.ServiceLBNamespace = cfg.ServiceLBNamespace
 	serverConfig.ControlConfig.SANs = cfg.TLSSan
 	serverConfig.ControlConfig.BindAddress = cfg.BindAddress
 	serverConfig.ControlConfig.SupervisorPort = cfg.SupervisorPort
@@ -364,10 +364,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		}
 	}
 	if serverConfig.ControlConfig.Skips["servicelb"] {
-		serverConfig.DisableServiceLB = true
+		serverConfig.ControlConfig.DisableServiceLB = true
 	}
 
-	if serverConfig.ControlConfig.DisableCCM {
+	if serverConfig.ControlConfig.DisableCCM && serverConfig.ControlConfig.DisableServiceLB {
 		serverConfig.ControlConfig.Skips["ccm"] = true
 		serverConfig.ControlConfig.Disables["ccm"] = true
 	}
@@ -484,7 +484,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	agentConfig.ServerURL = url
 	agentConfig.Token = token
 	agentConfig.DisableLoadBalancer = !serverConfig.ControlConfig.DisableAPIServer
-	agentConfig.DisableServiceLB = serverConfig.DisableServiceLB
+	agentConfig.DisableServiceLB = serverConfig.ControlConfig.DisableServiceLB
 	agentConfig.ETCDAgent = serverConfig.ControlConfig.DisableAPIServer
 	agentConfig.ClusterReset = serverConfig.ControlConfig.ClusterReset
 	agentConfig.Rootless = cfg.Rootless