CoreDNS requires restart after scaling up nodes to be able to resolve new hostnames

CHANGELOG-1.1.md

@@ -38,6 +38,7 @@
 - [#2332](https://github.com/epiphany-platform/epiphany/issues/2332) - [Elasticsearch] Error when having multiple VMs and non-clustered mode
 - [#1576](https://github.com/epiphany-platform/epiphany/issues/1576) - [Kafka] Incorrect number of brokers/queues available after scaling up/down
 - [#2381](https://github.com/epiphany-platform/epiphany/issues/2381) - Kibana fails to upgrade from epicli v1.0 to v1.1 (version comparison issue)
+- [#2345](https://github.com/epiphany-platform/epiphany/issues/2345) - CoreDNS requires restart after scaling up nodes to be able to resolve new hostnames
 
 ### Updated
 

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/coredns-configmap.yml

@@ -1,6 +1,9 @@
 # Based on https://github.com/kubernetes/kubernetes/blob/v1.18.6/cluster/addons/dns/coredns/coredns.yaml.in
 # Hosts plugin added
 
+# Note: /etc/hosts in pods is managed by kubelet thus we mount it under different path,
+# see https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/#why-does-kubelet-manage-the-hosts-file
+
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -16,7 +19,7 @@ data:
             lameduck 5s
         }
         ready
-        hosts {
+        hosts /etc/hosts-mounted {
             fallthrough
         }
         kubernetes cluster.local in-addr.arpa ip6.arpa {

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/files/coredns-deployment-patch.yml

@@ -8,7 +8,7 @@ spec:
             ephemeral-storage: 2Mi
         volumeMounts:
         - name: hosts-volume
-          mountPath: /etc/hosts
+          mountPath: /etc/hosts-mounted
           readOnly: true
       volumes:
       - name: hosts-volume

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_master/tasks/master-init.yml

@@ -70,5 +70,9 @@
 - name: Patch CoreDNS
   include_tasks: patch-coredns.yml
 
+# TODO: Restart conditionally (only when /etc/hosts was updated) or check whether newer CoreDNS would solve issue #2345
+- name: Restart CoreDNS pods
+  command: kubectl rollout restart deployment coredns --namespace kube-system
+
 - name: Apply Kubernetes Dashboard
   include_tasks: apply-dashboard.yml