[release-1.24] Backports for 2023-08 release

pkg/agent/templates/templates_linux.go

@@ -131,6 +131,7 @@ enable_keychain = true
 func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
 	out := new(bytes.Buffer)
 	t := template.Must(template.New("compiled_template").Parse(templateBuffer))
+	template.Must(t.New("base").Parse(ContainerdConfigTemplate))
 	if err := t.Execute(out, config); err != nil {
 		return "", err
 	}

pkg/agent/templates/templates_windows.go

@@ -179,6 +179,7 @@ func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string,
 		},
 	}
 	t := template.Must(template.New("compiled_template").Funcs(funcs).Parse(templateBuffer))
+	template.Must(t.New("base").Parse(ContainerdConfigTemplate))
 	if err := t.Execute(out, config); err != nil {
 		return "", err
 	}

pkg/cloudprovider/servicelb.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/rancher/wrangler/pkg/condition"
 	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
@@ -484,12 +485,12 @@ func (k *k3s) newDaemonSet(svc *core.Service) (*apps.DaemonSet, error) {
 					},
 					Tolerations: []core.Toleration{
 						{
-							Key:      "node-role.kubernetes.io/master",
+							Key:      util.MasterRoleLabelKey,
 							Operator: "Exists",
 							Effect:   "NoSchedule",
 						},
 						{
-							Key:      "node-role.kubernetes.io/control-plane",
+							Key:      util.ControlPlaneRoleLabelKey,
 							Operator: "Exists",
 							Effect:   "NoSchedule",
 						},

pkg/cluster/address_controller.go

@@ -0,0 +1,61 @@
+package cluster
+
+import (
+	"context"
+
+	"github.com/k3s-io/k3s/pkg/util"
+	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	"github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+)
+
+func registerAddressHandlers(ctx context.Context, c *Cluster) {
+	nodes := c.config.Runtime.Core.Core().V1().Node()
+	a := &addressesHandler{
+		nodeController: nodes,
+		allowed:        map[string]bool{},
+	}
+
+	for _, cn := range c.config.SANs {
+		a.allowed[cn] = true
+	}
+
+	logrus.Infof("Starting dynamiclistener CN filter node controller")
+	nodes.OnChange(ctx, "server-cn-filter", a.sync)
+	c.cnFilterFunc = a.filterCN
+}
+
+type addressesHandler struct {
+	nodeController controllerv1.NodeController
+	allowed        map[string]bool
+}
+
+// filterCN filters a list of potential server CNs (hostnames or IPs), removing any which do not correspond to
+// valid cluster servers (control-plane or etcd), or an address explicitly added via the tls-san option.
+func (a *addressesHandler) filterCN(cns ...string) []string {
+	if !a.nodeController.Informer().HasSynced() {
+		return cns
+	}
+
+	filteredCNs := make([]string, 0, len(cns))
+	for _, cn := range cns {
+		if a.allowed[cn] {
+			filteredCNs = append(filteredCNs, cn)
+		} else {
+			logrus.Debugf("CN filter controller rejecting certificate CN: %s", cn)
+		}
+	}
+	return filteredCNs
+}
+
+// sync updates the allowed address list to include addresses for the node
+func (a *addressesHandler) sync(key string, node *v1.Node) (*v1.Node, error) {
+	if node != nil {
+		if node.Labels[util.ControlPlaneRoleLabelKey] != "" || node.Labels[util.ETCDRoleLabelKey] != "" {
+			for _, address := range node.Status.Addresses {
+				a.allowed[address.String()] = true
+			}
+		}
+	}
+	return node, nil
+}

pkg/cluster/cluster.go

@@ -24,6 +24,7 @@ type Cluster struct {
 	storageStarted   bool
 	saveBootstrap    bool
 	shouldBootstrap  bool
+	cnFilterFunc     func(...string) []string
 }
 
 // Start creates the dynamic tls listener, http request handler,

pkg/cluster/https.go

@@ -51,18 +51,23 @@ func (c *Cluster) newListener(ctx context.Context) (net.Listener, http.Handler,
 	if err != nil {
 		return nil, nil, err
 	}
+	c.config.SANs = append(c.config.SANs, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc."+c.config.ClusterDomain)
+	c.config.Runtime.ClusterControllerStarts["server-cn-filter"] = func(ctx context.Context) {
+		registerAddressHandlers(ctx, c)
+	}
 	storage := tlsStorage(ctx, c.config.DataDir, c.config.Runtime)
 	return wrapHandler(dynamiclistener.NewListener(tcp, storage, cert, key, dynamiclistener.Config{
 		ExpirationDaysCheck: config.CertificateRenewDays,
 		Organization:        []string{version.Program},
-		SANs:                append(c.config.SANs, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc."+c.config.ClusterDomain),
+		SANs:                c.config.SANs,
 		CN:                  version.Program,
 		TLSConfig: &tls.Config{
 			ClientAuth:   tls.RequestClientCert,
 			MinVersion:   c.config.TLSMinVersion,
 			CipherSuites: c.config.TLSCipherSuites,
 			NextProtos:   []string{"h2", "http/1.1"},
 		},
+		FilterCN: c.filterCN,
 		RegenerateCerts: func() bool {
 			const regenerateDynamicListenerFile = "dynamic-cert-regenerate"
 			dynamicListenerRegenFilePath := filepath.Join(c.config.DataDir, "tls", regenerateDynamicListenerFile)
@@ -75,6 +80,13 @@ func (c *Cluster) newListener(ctx context.Context) (net.Listener, http.Handler,
 	}))
 }
 
+func (c *Cluster) filterCN(cn ...string) []string {
+	if c.cnFilterFunc != nil {
+		return c.cnFilterFunc(cn...)
+	}
+	return cn
+}
+
 // initClusterAndHTTPS sets up the dynamic tls listener, request router,
 // and cluster database. Once the database is up, it starts the supervisor http server.
 func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error {

pkg/daemons/control/server.go

@@ -174,8 +174,10 @@ func apiServer(ctx context.Context, cfg *config.Control) error {
 	} else {
 		argsMap["bind-address"] = cfg.APIServerBindAddress
 	}
-	argsMap["enable-aggregator-routing"] = "true"
-	argsMap["egress-selector-config-file"] = runtime.EgressSelectorConfig
+	if cfg.EgressSelectorMode != config.EgressSelectorModeDisabled {
+		argsMap["enable-aggregator-routing"] = "true"
+		argsMap["egress-selector-config-file"] = runtime.EgressSelectorConfig
+	}
 	argsMap["tls-cert-file"] = runtime.ServingKubeAPICert
 	argsMap["tls-private-key-file"] = runtime.ServingKubeAPIKey
 	argsMap["service-account-key-file"] = runtime.ServiceKey

pkg/etcd/etcd.go

@@ -73,10 +73,6 @@ const (
 	maxBackupRetention     = 5
 	maxConcurrentSnapshots = 1
 	compressedExtension    = ".zip"
-
-	MasterLabel       = "node-role.kubernetes.io/master"
-	ControlPlaneLabel = "node-role.kubernetes.io/control-plane"
-	EtcdRoleLabel     = "node-role.kubernetes.io/etcd"
 )
 
 var (

pkg/etcd/member_controller.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
@@ -40,7 +41,7 @@ func (e *etcdMemberHandler) sync(key string, node *v1.Node) (*v1.Node, error) {
 		return nil, nil
 	}
 
-	if _, ok := node.Labels[EtcdRoleLabel]; !ok {
+	if _, ok := node.Labels[util.ETCDRoleLabelKey]; !ok {
 		logrus.Debugf("Node %s was not labeled etcd node, skipping sync", key)
 		return node, nil
 	}
@@ -98,7 +99,7 @@ func (e *etcdMemberHandler) sync(key string, node *v1.Node) (*v1.Node, error) {
 }
 
 func (e *etcdMemberHandler) onRemove(key string, node *v1.Node) (*v1.Node, error) {
-	if _, ok := node.Labels[EtcdRoleLabel]; !ok {
+	if _, ok := node.Labels[util.ETCDRoleLabelKey]; !ok {
 		logrus.Debugf("Node %s was not labeled etcd node, skipping etcd member removal", key)
 		return node, nil
 	}

pkg/etcd/metadata_controller.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/k3s-io/k3s/pkg/util"
 	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
@@ -51,7 +52,7 @@ func (m *metadataHandler) handleSelf(node *v1.Node) (*v1.Node, error) {
 	if m.etcd.config.DisableETCD {
 		if node.Annotations[NodeNameAnnotation] == "" &&
 			node.Annotations[NodeAddressAnnotation] == "" &&
-			node.Labels[EtcdRoleLabel] == "" {
+			node.Labels[util.ETCDRoleLabelKey] == "" {
 			return node, nil
 		}
 
@@ -65,14 +66,14 @@ func (m *metadataHandler) handleSelf(node *v1.Node) (*v1.Node, error) {
 
 		delete(node.Annotations, NodeNameAnnotation)
 		delete(node.Annotations, NodeAddressAnnotation)
-		delete(node.Labels, EtcdRoleLabel)
+		delete(node.Labels, util.ETCDRoleLabelKey)
 
 		return m.nodeController.Update(node)
 	}
 
 	if node.Annotations[NodeNameAnnotation] == m.etcd.name &&
 		node.Annotations[NodeAddressAnnotation] == m.etcd.address &&
-		node.Labels[EtcdRoleLabel] == "true" {
+		node.Labels[util.ETCDRoleLabelKey] == "true" {
 		return node, nil
 	}
 
@@ -86,7 +87,7 @@ func (m *metadataHandler) handleSelf(node *v1.Node) (*v1.Node, error) {
 
 	node.Annotations[NodeNameAnnotation] = m.etcd.name
 	node.Annotations[NodeAddressAnnotation] = m.etcd.address
-	node.Labels[EtcdRoleLabel] = "true"
+	node.Labels[util.ETCDRoleLabelKey] = "true"
 
 	return m.nodeController.Update(node)
 }

pkg/secretsencrypt/controller.go

@@ -29,7 +29,6 @@ const (
 	secretsProgressEvent       string = "SecretsProgress"
 	secretsUpdateCompleteEvent string = "SecretsUpdateComplete"
 	secretsUpdateErrorEvent    string = "SecretsUpdateError"
-	controlPlaneRoleLabelKey   string = "node-role.kubernetes.io/control-plane"
 )
 
 type handler struct {
@@ -186,7 +185,7 @@ func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) (
 	if err != nil {
 		return false, err
 	}
-	labelSelector := labels.Set{controlPlaneRoleLabelKey: "true"}.String()
+	labelSelector := labels.Set{util.ControlPlaneRoleLabelKey: "true"}.String()
 	nodes, err := h.nodes.List(metav1.ListOptions{LabelSelector: labelSelector})
 	if err != nil {
 		return false, err

pkg/server/secrets-encrypt.go

@@ -16,6 +16,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/cluster"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/secretsencrypt"
+	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/rancher/wrangler/pkg/generated/controllers/core"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -304,7 +305,7 @@ func getEncryptionHashAnnotation(core core.Interface) (string, string, error) {
 	if err != nil {
 		return "", "", err
 	}
-	if _, ok := node.Labels[ControlPlaneRoleLabelKey]; !ok {
+	if _, ok := node.Labels[util.ControlPlaneRoleLabelKey]; !ok {
 		return "", "", fmt.Errorf("cannot manage secrets encryption on non control-plane node %s", nodeName)
 	}
 	if ann, ok := node.Annotations[secretsencrypt.EncryptionHashAnnotation]; ok {
@@ -323,7 +324,7 @@ func verifyEncryptionHashAnnotation(runtime *config.ControlRuntime, core core.In
 	var firstHash string
 	var firstNodeName string
 	first := true
-	labelSelector := labels.Set{ControlPlaneRoleLabelKey: "true"}.String()
+	labelSelector := labels.Set{util.ControlPlaneRoleLabelKey: "true"}.String()
 	nodes, err := core.V1().Node().List(metav1.ListOptions{LabelSelector: labelSelector})
 	if err != nil {
 		return err

pkg/server/server.go

@@ -39,12 +39,6 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 )
 
-const (
-	MasterRoleLabelKey       = "node-role.kubernetes.io/master"
-	ControlPlaneRoleLabelKey = "node-role.kubernetes.io/control-plane"
-	ETCDRoleLabelKey         = "node-role.kubernetes.io/etcd"
-)
-
 func ResolveDataDir(dataDir string) (string, error) {
 	dataDir, err := datadir.Resolve(dataDir)
 	return filepath.Join(dataDir, "server"), err
@@ -581,10 +575,10 @@ func setNodeLabelsAndAnnotations(ctx context.Context, nodes v1.NodeClient, confi
 		if node.Labels == nil {
 			node.Labels = make(map[string]string)
 		}
-		v, ok := node.Labels[ControlPlaneRoleLabelKey]
+		v, ok := node.Labels[util.ControlPlaneRoleLabelKey]
 		if !ok || v != "true" {
-			node.Labels[ControlPlaneRoleLabelKey] = "true"
-			node.Labels[MasterRoleLabelKey] = "true"
+			node.Labels[util.ControlPlaneRoleLabelKey] = "true"
+			node.Labels[util.MasterRoleLabelKey] = "true"
 		}
 
 		if config.ControlConfig.EncryptSecrets {

pkg/util/labels.go

@@ -0,0 +1,7 @@
+package util
+
+const (
+	MasterRoleLabelKey       = "node-role.kubernetes.io/master"
+	ControlPlaneRoleLabelKey = "node-role.kubernetes.io/control-plane"
+	ETCDRoleLabelKey         = "node-role.kubernetes.io/etcd"
+)

scripts/test-run-compat

@@ -29,7 +29,7 @@ export -f test-post-hook
 
 REPO=${REPO:-rancher}
 IMAGE_NAME=${IMAGE_NAME:-k3s}
-PREVIOUS_CHANNEL=$(grep 'k8s.io/kubernetes v' go.mod | head -n1 | awk '{print $2}' | awk -F. '{print "v1." ($2 - 1)}')
+PREVIOUS_CHANNEL=$(echo ${VERSION_K8S} | awk -F. '{print "v1." ($2 - 1)}')
 PREVIOUS_VERSION=$(curl -s https://update.k3s.io/v1-release/channels/${PREVIOUS_CHANNEL} -o /dev/null -w '%{redirect_url}' | awk -F/ '{print gensub(/\+/, "-", "g", $NF)}')
 STABLE_VERSION=$(curl -s https://update.k3s.io/v1-release/channels/stable -o /dev/null -w '%{redirect_url}' | awk -F/ '{print gensub(/\+/, "-", "g", $NF)}')
 LATEST_VERSION=$(curl -s https://update.k3s.io/v1-release/channels/latest -o /dev/null -w '%{redirect_url}' | awk -F/ '{print gensub(/\+/, "-", "g", $NF)}')

scripts/test-run-etcd

@@ -14,7 +14,7 @@ export SERVER_1_ARGS="--cluster-init"
 
 REPO=${REPO:-rancher}
 IMAGE_NAME=${IMAGE_NAME:-k3s}
-PREVIOUS_CHANNEL=$(grep 'k8s.io/kubernetes v' go.mod | head -n1 | awk '{print $2}' | awk -F. '{print "v1." ($2 - 1)}')
+PREVIOUS_CHANNEL=$(echo ${VERSION_K8S} | awk -F. '{print "v1." ($2 - 1)}')
 PREVIOUS_VERSION=$(curl -s https://update.k3s.io/v1-release/channels/${PREVIOUS_CHANNEL} -o /dev/null -w '%{redirect_url}' | awk -F/ '{print gensub(/\+/, "-", "g", $NF)}')
 STABLE_VERSION=$(curl -s https://update.k3s.io/v1-release/channels/stable -o /dev/null -w '%{redirect_url}' | awk -F/ '{print gensub(/\+/, "-", "g", $NF)}')
 LATEST_VERSION=$(curl -s https://update.k3s.io/v1-release/channels/latest -o /dev/null -w '%{redirect_url}' | awk -F/ '{print gensub(/\+/, "-", "g", $NF)}')

scripts/test-run-upgrade

@@ -13,7 +13,7 @@ export WAIT_SERVICES="${all_services[@]}"
 
 REPO=${REPO:-rancher}
 IMAGE_NAME=${IMAGE_NAME:-k3s}
-CURRENT_CHANNEL=$(grep 'k8s.io/kubernetes v' go.mod | head -n1 | awk '{print $2}' | awk -F. '{print "v1." $2}')
+CURRENT_CHANNEL=$(echo ${VERSION_K8S} | awk -F. '{print "v1." $2}')
 CURRENT_VERSION=$(curl -s https://update.k3s.io/v1-release/channels/${CURRENT_CHANNEL} -o /dev/null -w '%{redirect_url}' | awk -F/ '{print gensub(/\+/, "-", "g", $NF)}')
 export K3S_IMAGE_SERVER=${REPO}/${IMAGE_NAME}:${CURRENT_VERSION}
 export K3S_IMAGE_AGENT=${REPO}/${IMAGE_NAME}:${CURRENT_VERSION}

scripts/version.sh

@@ -22,38 +22,43 @@ if [ -d .git ]; then
     fi
 fi
 
+get-module-version(){
+  go list -m -f '{{if .Replace}}{{.Replace.Version}}{{else}}{{.Version}}{{end}}' $1
+}
+
 # We're building k3s against containerd 1.5 in go.mod because 1.6 has dependency
 # conflicts with Kubernetes, but we still need to bundle containerd 1.6.
-VERSION_CONTAINERD="v1.7.1-k3s1"
+VERSION_CONTAINERD="v1.7.3-k3s1"
 
-VERSION_CRICTL=$(grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}')
+VERSION_CRICTL=$(get-module-version github.com/kubernetes-sigs/cri-tools)
 if [ -z "$VERSION_CRICTL" ]; then
     VERSION_CRICTL="v0.0.0"
 fi
 
-VERSION_K8S=$(grep 'k8s.io/kubernetes v' go.mod | head -n1 | awk '{print $2}')
+VERSION_K8S_K3S=$(get-module-version k8s.io/kubernetes)
+VERSION_K8S=${VERSION_K8S_K3S%"-k3s1"}
 if [ -z "$VERSION_K8S" ]; then
     VERSION_K8S="v0.0.0"
 fi
 
-VERSION_RUNC=$(grep github.com/opencontainers/runc go.mod | head -n1 | awk '{print $4}')
+VERSION_RUNC=$(get-module-version github.com/opencontainers/runc)
 if [ -z "$VERSION_RUNC" ]; then
     VERSION_RUNC="v0.0.0"
 fi
 
-VERSION_FLANNEL=$(grep github.com/flannel-io/flannel go.mod | head -n1 | awk '{print $2}')
+VERSION_FLANNEL=$(get-module-version github.com/flannel-io/flannel)
 if [ -z "$VERSION_FLANNEL" ]; then
   VERSION_FLANNEL="v0.0.0"
 fi
 
-VERSION_CRI_DOCKERD=$(grep github.com/Mirantis/cri-dockerd go.mod | head -n1 | awk '{print $4}')
+VERSION_CRI_DOCKERD=$(get-module-version github.com/Mirantis/cri-dockerd)
 if [ -z "$VERSION_CRI_DOCKERD" ]; then
   VERSION_CRI_DOCKERD="v0.0.0"
 fi
 
 VERSION_CNIPLUGINS="v1.3.0-k3s1"
 
-VERSION_KUBE_ROUTER=$(grep github.com/k3s-io/kube-router go.mod | head -n1 | awk '{print $4}')
+VERSION_KUBE_ROUTER=$(get-module-version github.com/cloudnativelabs/kube-router/v2)
 if [ -z "$VERSION_KUBE_ROUTER" ]; then
     VERSION_KUBE_ROUTER="v0.0.0"
 fi