fix: preserve coredns config during cluster restart using the coredns-custom configmap (!)

pkg/client/cluster.go

@@ -31,6 +31,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/docker/go-connections/nat"
@@ -54,6 +55,10 @@ import (
 	goyaml "gopkg.in/yaml.v2"
 )
 
+//go:embed templates/coredns-custom.yaml.tmpl
+var customDNSTemplateStr string
+var customDNSTemplate = template.Must(template.New("customDNS").Parse(customDNSTemplateStr))
+
 // ClusterRun orchestrates the steps of cluster creation, configuration and starting
 func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *config.ClusterConfig) error {
 	/*
@@ -1058,11 +1063,18 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 			// -> inject hostAliases and network members into CoreDNS configmap
 			if len(servers) > 0 {
 				postStartErrgrp.Go(func() error {
-					hosts := ""
+					type record struct {
+						IP       string
+						Hostname string
+					}
+
+					records := make([]record, 0)
 
 					// hosts: hostAliases (including host.k3d.internal)
 					for _, hostAlias := range clusterStartOpts.HostAliases {
-						hosts += fmt.Sprintf("%s %s\n", hostAlias.IP, strings.Join(hostAlias.Hostnames, " "))
+						for _, hostname := range hostAlias.Hostnames {
+							records = append(records, record{IP: hostAlias.IP, Hostname: hostname})
+						}
 					}
 
 					// more hosts: network members ("neighbor" containers)
@@ -1071,45 +1083,21 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 						return fmt.Errorf("failed to get cluster network %s to inject host records into CoreDNS: %w", cluster.Network.Name, err)
 					}
 					for _, member := range net.Members {
-						hosts += fmt.Sprintf("%s %s\n", member.IP.String(), member.Name)
+						records = append(records, record{IP: member.IP.String(), Hostname: member.Name})
 					}
 
 					// inject CoreDNS configmap
 					l.Log().Infof("Injecting records for hostAliases (incl. host.k3d.internal) and for %d network members into CoreDNS configmap...", len(net.Members))
-					act := actions.RewriteFileAction{
+					var custom_dns bytes.Buffer
+					err = customDNSTemplate.Execute(&custom_dns, records)
+					if err != nil {
+						return fmt.Errorf("failed to render template: %w", err)
+					}
+					act := actions.WriteFileAction{
 						Runtime: runtime,
-						Path:    "/var/lib/rancher/k3s/server/manifests/coredns.yaml",
+						Content: []byte(custom_dns.Bytes()),
+						Dest:    "/var/lib/rancher/k3s/server/manifests/coredns-custom.yaml",
 						Mode:    0744,
-						RewriteFunc: func(input []byte) ([]byte, error) {
-							split, err := util.SplitYAML(input)
-							if err != nil {
-								return nil, fmt.Errorf("error splitting yaml: %w", err)
-							}
-
-							var outputBuf bytes.Buffer
-							outputEncoder := util.NewYAMLEncoder(&outputBuf)
-
-							for _, d := range split {
-								var doc map[string]interface{}
-								if err := yaml.Unmarshal(d, &doc); err != nil {
-									return nil, err
-								}
-								if kind, ok := doc["kind"]; ok {
-									if strings.ToLower(kind.(string)) == "configmap" {
-										configmapData, ok := doc["data"].(map[string]interface{})
-										if !ok {
-											return nil, fmt.Errorf("invalid ConfigMap data type: %T", doc["data"])
-										}
-										configmapData["NodeHosts"] = hosts
-									}
-								}
-								if err := outputEncoder.Encode(doc); err != nil {
-									return nil, err
-								}
-							}
-							_ = outputEncoder.Close()
-							return outputBuf.Bytes(), nil
-						},
 					}
 
 					// get the first server in the list and run action on it once it's ready for it

pkg/client/templates/coredns-custom.yaml.tmpl

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns-custom
+  namespace: kube-system
+data:
+  hosts.override: |
+    file /etc/coredns/custom/additional-dns.db
+
+  # a SOA record is required
+  additional-dns.db: |
+    @ 3600 IN SOA a.root-servers.net. nstld.verisign-grs.com. 2024061200 1800 900 604800 86400
+    {{- range . }}
+    {{ .Hostname }} IN A {{ .IP }}
+    {{- end }}