From 22ac54c6716364cef629076c315a1c91a6d57317 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Fri, 7 Jul 2023 22:49:07 -0700 Subject: [PATCH] Fix Karpenter failure to start on IPv6 clusters --- ...s_s3_object_minimal.example.com-addons-bootstrap_content | 4 ++-- ...xample.com-addons-coredns.addons.k8s.io-k8s-1.12_content | 4 ++++ ...minimal.example.com-addons-karpenter.sh-k8s-1.19_content | 2 +- .../addons/coredns.addons.k8s.io/k8s-1.12.yaml.template | 6 ++++++ .../resources/addons/karpenter.sh/k8s-1.19.yaml.template | 3 ++- .../bootstrapchannelbuilder/bootstrapchannelbuilder.go | 2 +- upup/pkg/fi/cloudup/template_functions.go | 3 +++ 7 files changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index aece22aae7924..af8b62f0c96b3 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -14,7 +14,7 @@ spec: version: 9.99.0 - id: k8s-1.12 manifest: coredns.addons.k8s.io/k8s-1.12.yaml - manifestHash: d2bbb7cbee5835c3891fe80fbacf8963508359ef9159f8480325ce9a7174f14a + manifestHash: 8834e41010ae2fb8a533107c4a32cf068ac161359956e7a52921b2a07ad8ebf5 name: coredns.addons.k8s.io selector: k8s-addon: coredns.addons.k8s.io @@ -69,7 +69,7 @@ spec: version: 9.99.0 - id: k8s-1.19 manifest: karpenter.sh/k8s-1.19.yaml - manifestHash: f59d4c21751b3fc33c84e664fb41199b8efb58cc5976ade6e937abc109cb612b + manifestHash: aab89cad4f4a52b8620f581548694a6fc096bdbd1a297310beda01b57d3550ae name: karpenter.sh prune: kinds: diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content index 477b37ebbee8e..a4c4b015e1ec6 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content @@ -188,6 +188,10 @@ spec: tolerations: - key: CriticalAddonsOnly operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists topologySpreadConstraints: - labelSelector: matchLabels: diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content index bb1f2aedd746e..a1d5a31764ab5 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content @@ -1752,7 +1752,7 @@ spec: - mountPath: /var/run/secrets/amazonaws.com/ name: token-amazonaws-com readOnly: true - dnsPolicy: Default + dnsPolicy: ClusterFirst priorityClassName: system-cluster-critical securityContext: fsGroup: 1000 diff --git a/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/k8s-1.12.yaml.template index e4693d5d2a852..06ea0e0366f54 100644 --- a/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/k8s-1.12.yaml.template +++ b/upup/models/cloudup/resources/addons/coredns.addons.k8s.io/k8s-1.12.yaml.template @@ -132,6 +132,12 @@ spec: - key: "CriticalAddonsOnly" operator: "Exists" {{- end }} + {{- if KarpenterEnabled }} + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + {{- end }} nodeSelector: kubernetes.io/os: linux {{- if .KubeDNS.Affinity }} diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template index 02bc4f321bae3..66085694d1c12 100644 --- a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template +++ b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template @@ -1478,7 +1478,8 @@ spec: securityContext: fsGroup: 1000 priorityClassName: "system-cluster-critical" - dnsPolicy: Default + # Must use ClusterFirst on IPv6 clusters in order to get DNS64 + dnsPolicy: ClusterFirst containers: - name: controller image: public.ecr.aws/karpenter/controller:v0.28.1 diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index 9b934f24985de..627bf4da9b763 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -1242,7 +1242,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) }) } } - if b.Cluster.Spec.Karpenter != nil && fi.ValueOf(&b.Cluster.Spec.Karpenter.Enabled) { + if b.Cluster.Spec.Karpenter != nil && b.Cluster.Spec.Karpenter.Enabled { key := "karpenter.sh" { diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index e7d7fc4b8e302..ef1fb39732822 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -395,6 +395,9 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS return nodeup.UsesInstanceIDForNodeName(tf.Cluster) } + dest["KarpenterEnabled"] = func() bool { + return cluster.Spec.Karpenter != nil && cluster.Spec.Karpenter.Enabled + } dest["KarpenterInstanceTypes"] = func(ig kops.InstanceGroupSpec) ([]string, error) { return karpenterInstanceTypes(tf.cloud.(awsup.AWSCloud), ig) }