Added annotations (kubernetes-sigs#383)

* Added annotations

* added changes to changelog

* Added coredns pdb

* fixed coredns pdb

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.17.0-0.3.5 (2023-12-1)
+
+*[Core] Added cluster-autoscaler annotation to evict local volumes
+
 ## 0.17.0-0.3.4 (2023-11-17)
 
 * [Core] Conditionally increase replicas for capi controllers

pkg/cluster/internal/create/actions/createworker/aws.go

@@ -142,7 +142,8 @@ func (b *AWSBuilder) installCloudProvider(n nodes.Node, k string, keosCluster co
 func (b *AWSBuilder) installCSI(n nodes.Node, k string) error {
 	c := "helm install aws-ebs-csi-driver /stratio/helm/aws-ebs-csi-driver" +
 		" --kubeconfig " + k +
-		" --namespace " + b.csiNamespace
+		" --namespace " + b.csiNamespace +
+		" --set controller.podAnnotations.\"cluster-autoscaler\\.kubernetes\\.io/safe-to-evict-local-volumes=socket-dir\""
 	_, err := commons.ExecuteCommand(n, c)
 	if err != nil {
 		return errors.Wrap(err, "failed to deploy AWS EBS CSI driver Helm Chart")
@@ -327,3 +328,19 @@ func (b *AWSBuilder) getOverrideVars(p ProviderParams, networks commons.Networks
 	}
 	return overrideVars, nil
 }
+
+func (b *AWSBuilder) postInstallPhase(n nodes.Node, k string) error {
+	if b.capxManaged {
+		err := patchDeploy(n, k, "kube-system", "coredns", "{\"spec\": {\"template\": {\"metadata\": {\"annotations\": {\""+postInstallAnnotation+"\": \"tmp\"}}}}}")
+		if err != nil {
+			return errors.Wrap(err, "failed to add podAnnotation to coredns")
+		}
+
+		err = patchDeploy(n, k, "kube-system", "ebs-csi-controller", "{\"spec\": {\"template\": {\"metadata\": {\"annotations\": {\""+postInstallAnnotation+"\": \"socket-dir\"}}}}}")
+		if err != nil {
+			return errors.Wrap(err, "failed to add podAnnotation to ebs-csi-controller")
+		}
+	}
+
+	return nil
+}

pkg/cluster/internal/create/actions/createworker/azure.go

@@ -154,7 +154,8 @@ func (b *AzureBuilder) installCSI(n nodes.Node, k string) error {
 	// Deploy disk CSI driver
 	c = "helm install azuredisk-csi-driver /stratio/helm/azuredisk-csi-driver " +
 		" --kubeconfig " + k +
-		" --namespace " + b.csiNamespace
+		" --namespace " + b.csiNamespace +
+		" --set controller.podAnnotations.\"cluster-autoscaler\\.kubernetes\\.io/safe-to-evict-local-volumes=socket-dir\\,azure-cred\""
 	_, err = commons.ExecuteCommand(n, c)
 	if err != nil {
 		return errors.Wrap(err, "failed to deploy Azure Disk CSI driver Helm Chart")
@@ -163,7 +164,8 @@ func (b *AzureBuilder) installCSI(n nodes.Node, k string) error {
 	// Deploy file CSI driver
 	c = "helm install azurefile-csi-driver /stratio/helm/azurefile-csi-driver " +
 		" --kubeconfig " + k +
-		" --namespace " + b.csiNamespace
+		" --namespace " + b.csiNamespace +
+		" --set controller.podAnnotations.\"cluster-autoscaler\\.kubernetes\\.io/safe-to-evict-local-volumes=socket-dir\\,azure-cred\""
 	_, err = commons.ExecuteCommand(n, c)
 	if err != nil {
 		return errors.Wrap(err, "failed to deploy Azure File CSI driver Helm Chart")
@@ -314,3 +316,27 @@ func (b *AzureBuilder) getOverrideVars(p ProviderParams, networks commons.Networ
 	}
 	return overrideVars, nil
 }
+
+func (b *AzureBuilder) postInstallPhase(n nodes.Node, k string) error {
+	if b.capxManaged {
+		err := patchDeploy(n, k, "kube-system", "coredns", "{\"spec\": {\"template\": {\"metadata\": {\"annotations\": {\""+postInstallAnnotation+"\": \"tmp\"}}}}}")
+		if err != nil {
+			return errors.Wrap(err, "failed to add podAnnotation to coredns")
+		}
+		err = patchDeploy(n, k, "tigera-operator", "tigera-operator", "{\"spec\": {\"template\": {\"metadata\": {\"annotations\": {\""+postInstallAnnotation+"\": \"var-lib-calico\"}}}}}")
+		if err != nil {
+			return errors.Wrap(err, "failed to add podAnnotation to tigera-operator")
+		}
+		err = patchDeploy(n, k, "kube-system", "metrics-server", "{\"spec\": {\"template\": {\"metadata\": {\"annotations\": {\""+postInstallAnnotation+"\": \"tmp-dir\"}}}}}")
+		if err != nil {
+			return errors.Wrap(err, "failed to add podAnnotation to metrics-server")
+		}
+
+	} else {
+		err := patchDeploy(n, k, "kube-system", "cloud-controller-manager", "{\"spec\": {\"template\": {\"metadata\": {\"annotations\": {\""+postInstallAnnotation+"\": \"etc-kubernetes,ssl-mount,msi\"}}}}}")
+		if err != nil {
+			return errors.Wrap(err, "failed to add podAnnotation to cloud-controller-manager")
+		}
+	}
+	return nil
+}

pkg/cluster/internal/create/actions/createworker/createworker.go

@@ -656,6 +656,16 @@ func (a *action) Execute(ctx *actions.ActionContext) error {
 		}
 	}
 
+	ctx.Status.Start("Executing post-install steps 🎖️")
+	defer ctx.Status.End(false)
+
+	err = infra.postInstallPhase(n, kubeconfigPath)
+	if err != nil {
+		return err
+	}
+
+	ctx.Status.End(true)
+
 	ctx.Status.Start("Generating the KEOS descriptor 📝")
 	defer ctx.Status.End(false)
 

pkg/cluster/internal/create/actions/createworker/files/common/coredns_pdb.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: coredns
+  labels:
+    k8s-app: kube-dns
+  namespace: kube-system
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      k8s-app: kube-dns

pkg/cluster/internal/create/actions/createworker/files/gcp/gcp-compute-persistent-disk-csi-driver.yaml

@@ -416,6 +416,8 @@ spec:
       app: gcp-compute-persistent-disk-csi-driver
   template:
     metadata:
+      annotations:
+        cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: socket-dir
       labels:
         app: gcp-compute-persistent-disk-csi-driver
     spec:

pkg/cluster/internal/create/actions/createworker/gcp.go

@@ -262,3 +262,8 @@ func (b *GCPBuilder) getOverrideVars(p ProviderParams, networks commons.Networks
 	}
 	return overrideVars, nil
 }
+
+func (b *GCPBuilder) postInstallPhase(n nodes.Node, k string) error {
+
+	return nil
+}

pkg/cluster/internal/create/actions/createworker/provider.go

@@ -48,7 +48,7 @@ var denyAllEgressIMDSgnpFiles embed.FS
 var allowEgressIMDSgnpFiles embed.FS
 
 //go:embed files/*/*_pdb.yaml
-var capxPDBFile embed.FS
+var commonsPDBFile embed.FS
 
 const (
 	CAPICoreProvider         = "cluster-api:v1.5.1"
@@ -59,6 +59,9 @@ const (
 
 	keosClusterChart = "0.1.4"
 	keosClusterImage = "0.1.4"
+
+	postInstallAnnotation = "cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes"
+	corednsPdbPath        = "/kind/coredns-pdb.yaml"
 )
 
 const machineHealthCheckWorkerNodePath = "/kind/manifests/machinehealthcheckworkernode.yaml"
@@ -79,6 +82,7 @@ type PBuilder interface {
 	getAzs(p ProviderParams, networks commons.Networks) ([]string, error)
 	internalNginx(p ProviderParams, networks commons.Networks) (bool, error)
 	getOverrideVars(p ProviderParams, networks commons.Networks) (map[string][]byte, error)
+	postInstallPhase(n nodes.Node, k string) error
 }
 
 type Provider struct {
@@ -133,6 +137,11 @@ type helmRepository struct {
 	pass string
 }
 
+type calicoHelmParams struct {
+	Spec        commons.Spec
+	Annotations map[string]string
+}
+
 var scTemplate = DefaultStorageClass{
 	APIVersion: "storage.k8s.io/v1",
 	Kind:       "StorageClass",
@@ -201,6 +210,14 @@ func (i *Infra) getAzs(p ProviderParams, networks commons.Networks) ([]string, e
 	return i.builder.getAzs(p, networks)
 }
 
+func (i *Infra) postInstallPhase(n nodes.Node, k string) error {
+	err := installCorednsPdb(n, k)
+	if err != nil {
+		return err
+	}
+	return i.builder.postInstallPhase(n, k)
+}
+
 func (p *Provider) getDenyAllEgressIMDSGNetPol() (string, error) {
 	denyAllEgressIMDSGNetPolLocalPath := "files/" + p.capxProvider + "/deny-all-egress-imds_gnetpol.yaml"
 	denyAllEgressIMDSgnpFile, err := denyAllEgressIMDSgnpFiles.Open(denyAllEgressIMDSGNetPolLocalPath)
@@ -231,14 +248,13 @@ func (p *Provider) getAllowCAPXEgressIMDSGNetPol() (string, error) {
 	return string(allowEgressIMDSgnpContent), nil
 }
 
-func (p *Provider) getcapxPDB() (string, error) {
-	capxPDBLocalPath := "files/" + p.capxProvider + "/" + p.capxName + "_pdb.yaml"
-	capxPDBFile, err := capxPDBFile.Open(capxPDBLocalPath)
+func getcapxPDB(commonsPDBLocalPath string) (string, error) {
+	commonsPDBFile, err := commonsPDBFile.Open(commonsPDBLocalPath)
 	if err != nil {
 		return "", errors.Wrap(err, "error opening the PodDisruptionBudget file")
 	}
-	defer capxPDBFile.Close()
-	capaPDBContent, err := ioutil.ReadAll(capxPDBFile)
+	defer commonsPDBFile.Close()
+	capaPDBContent, err := ioutil.ReadAll(commonsPDBFile)
 	if err != nil {
 		return "", err
 	}
@@ -363,8 +379,15 @@ func installCalico(n nodes.Node, k string, keosCluster commons.KeosCluster, allo
 
 	calicoTemplate := "/kind/calico-helm-values.yaml"
 
+	calicoHelmParams := calicoHelmParams{
+		Spec: keosCluster.Spec,
+		Annotations: map[string]string{
+			postInstallAnnotation: "var-lib-calico",
+		},
+	}
+
 	// Generate the calico helm values
-	calicoHelmValues, err := getManifest("common", "calico-helm-values.tmpl", keosCluster.Spec)
+	calicoHelmValues, err := getManifest("common", "calico-helm-values.tmpl", calicoHelmParams)
 	if err != nil {
 		return errors.Wrap(err, "failed to generate calico helm values")
 	}
@@ -521,8 +544,8 @@ func (p *Provider) installCAPXWorker(n nodes.Node, kubeconfigPath string, allowA
 	}
 
 	// Define PodDisruptionBudget for capa service
-
-	capxPDB, err := p.getcapxPDB()
+	capxPDBLocalPath := "files/" + p.capxProvider + "/" + p.capxName + "_pdb.yaml"
+	capxPDB, err := getcapxPDB(capxPDBLocalPath)
 	if err != nil {
 		return err
 	}
@@ -880,3 +903,41 @@ func getManifest(parentPath string, name string, params interface{}) (string, er
 	}
 	return tpl.String(), nil
 }
+
+func patchDeploy(n nodes.Node, k string, ns string, deployName string, patch string) error {
+	c := "kubectl --kubeconfig " + k + " patch deploy -n " + ns + " " + deployName + " -p '" + patch + "'"
+	_, err := commons.ExecuteCommand(n, c)
+	if err != nil {
+		return err
+	}
+	return rolloutStatus(n, k, ns, deployName)
+}
+
+func rolloutStatus(n nodes.Node, k string, ns string, deployName string) error {
+	c := "kubectl --kubeconfig " + k + " rollout status deploy -n " + ns + " " + deployName + " --timeout=5m"
+	_, err := commons.ExecuteCommand(n, c)
+	return err
+}
+
+func installCorednsPdb(n nodes.Node, k string) error {
+
+	// Define PodDisruptionBudget for coredns service
+	corednsPDBLocalPath := "files/common/coredns_pdb.yaml"
+	corednsPDB, err := getcapxPDB(corednsPDBLocalPath)
+	if err != nil {
+		return err
+	}
+
+	c := "echo \"" + corednsPDB + "\" > " + corednsPdbPath
+	_, err = commons.ExecuteCommand(n, c)
+	if err != nil {
+		return errors.Wrap(err, "failed to create coredns PodDisruptionBudget file")
+	}
+
+	c = "kubectl --kubeconfig " + kubeconfigPath + " apply -f " + corednsPdbPath
+	_, err = commons.ExecuteCommand(n, c)
+	if err != nil {
+		return errors.Wrap(err, "failed to apply coredns PodDisruptionBudget")
+	}
+	return nil
+}

pkg/cluster/internal/create/actions/createworker/templates/common/calico-helm-values.tmpl

@@ -17,17 +17,17 @@ certs:
 imagePullSecrets: {}
 installation:
   calicoNetwork:
-    bgp: {{- if or ($.ControlPlane.Managed) (eq $.InfraProvider "azure") }} Disabled {{- else }} Enabled {{- end }}
-  {{- if not $.ControlPlane.Managed }}
-    {{- if eq $.InfraProvider "azure" }}
+    bgp: {{- if or ($.Spec.ControlPlane.Managed) (eq $.Spec.InfraProvider "azure") }} Disabled {{- else }} Enabled {{- end }}
+  {{- if not $.Spec.ControlPlane.Managed }}
+    {{- if eq $.Spec.InfraProvider "azure" }}
     mtu: 1350
     {{- end }}
     ipPools:
-      - cidr: {{- if $.Networks.PodsCidrBlock }} {{ $.Networks.PodsCidrBlock }} {{- else }} 192.168.0.0/16 {{- end }}
-        encapsulation: {{- if eq $.InfraProvider "azure" }} VXLAN {{- else }} IPIP {{- end }}
+      - cidr: {{- if $.Spec.Networks.PodsCidrBlock }} {{ $.Spec.Networks.PodsCidrBlock }} {{- else }} 192.168.0.0/16 {{- end }}
+        encapsulation: {{- if eq $.Spec.InfraProvider "azure" }} VXLAN {{- else }} IPIP {{- end }}
   {{- end }}
   cni:
-  {{- if and ($.ControlPlane.Managed) (eq $.InfraProvider "aws") }}
+  {{- if and ($.Spec.ControlPlane.Managed) (eq $.Spec.InfraProvider "aws") }}
     ipam:
       type: AmazonVPC
     type: AmazonVPC
@@ -37,15 +37,18 @@ installation:
     type: Calico
   {{- end }}
   enabled: true
-  kubernetesProvider: {{- if and ($.ControlPlane.Managed) (eq $.InfraProvider "aws") }} EKS {{- else }} "" {{- end }}
+  kubernetesProvider: {{- if and ($.Spec.ControlPlane.Managed) (eq $.Spec.InfraProvider "aws") }} EKS {{- else }} "" {{- end }}
   nodeMetricsPort: 9191
   registry: docker.io
   typhaMetricsPort: 9093
 # NodeSelector for the tigera/operator pod.
 nodeSelector:
   kubernetes.io/os: linux
 # Custom annotations for the tigera/operator pod.
-podAnnotations: {}
+podAnnotations:
+{{- range $key, $value := $.Annotations }}
+  {{ $key }}: {{ $value }}
+{{- end }}
 # Custom labels for the tigera/operator pod.
 podLabels: {}
 # Resource requests and limits for the tigera/operator pod.