From 4128489ddaa08c3e5cadf4947c5668fe2c38bd42 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Thu, 28 Mar 2024 09:43:52 -0600 Subject: [PATCH 1/5] Update DHCP broadcast interface handling: Use 127.1.1.1/32 for the DHCP broadcast interface instead of the load balancer IP. Using the load balancer IP can cause instability with that address and routing to Kubernetes services. Make the DHCP broadcast interface name static. The dynamic number added to the name Helm to restarts on every Helm deploy. Add ipvlan support for the DHCP broadcast interface. This allows deployment where creating and broadcasting a new mac address is prohibited. Vmware for example. Signed-off-by: Jacob Weinstock --- tinkerbell/stack/templates/nginx.yaml | 20 ++++++++++++++------ tinkerbell/stack/values.yaml | 5 ++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/tinkerbell/stack/templates/nginx.yaml b/tinkerbell/stack/templates/nginx.yaml index 96008bcb..626f8624 100644 --- a/tinkerbell/stack/templates/nginx.yaml +++ b/tinkerbell/stack/templates/nginx.yaml @@ -1,6 +1,10 @@ {{- if .Values.stack.enabled }} {{- $sourceInterface := .Values.stack.relay.sourceInterface -}} -{{- $macvlanInterfaceName := printf "%s%s" "macvlan" (randNumeric 2) -}} +{{- $dhcpInterfaceType := "macvlan" -}} +{{- if eq .Values.stack.relay.interfaceMode "ipvlan" -}} +{{- $dhcpInterfaceType = "ipvlan" -}} +{{- end -}} +{{- $dhcpInterfaceName := printf "%s0" $dhcpInterfaceType -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -74,7 +78,7 @@ spec: {{- end }} - name: {{ .Values.stack.relay.name }} image: {{ .Values.stack.relay.image }} - args: ["-m", "{{ .Values.stack.relay.presentGiaddrAction }}", "-c", "{{ .Values.stack.relay.maxHopCount }}", "-id", "{{ $macvlanInterfaceName }}", "-iu", "eth0", "-U", "eth0", "smee.{{ .Release.Namespace }}.svc.{{ .Values.stack.clusterDomain }}."] + args: ["-m", "{{ .Values.stack.relay.presentGiaddrAction }}", "-c", "{{ .Values.stack.relay.maxHopCount }}", "-id", "{{ $dhcpInterfaceName }}", "-iu", "eth0", "-U", "eth0", "smee.{{ .Release.Namespace }}.svc.{{ .Values.stack.clusterDomain }}."] ports: - containerPort: 67 protocol: UDP @@ -119,14 +123,18 @@ spec: srcInterface=$(nsenter -t1 -n ip route | awk '/default/ {print $5}' | head -n1) fi # Create a macvlan interface. TODO: If this fails, try again with a different name? - nsenter -t1 -n ip link add {{ $macvlanInterfaceName }} link ${srcInterface} type macvlan mode bridge + {{- if eq $dhcpInterfaceType "ipvlan" }} + nsenter -t1 -n ip link add {{ $dhcpInterfaceName }} link ${srcInterface} type ipvlan mode l2 + {{- else }} + nsenter -t1 -n ip link add {{ $dhcpInterfaceName }} link ${srcInterface} type macvlan mode bridge + {{- end }} # Move the interface into the POD. pid=$(echo $$) - nsenter -t1 -n ip link set {{ $macvlanInterfaceName }} netns ${pid} || nsenter -t1 -n ip link delete {{ $macvlanInterfaceName }} + nsenter -t1 -n ip link set {{ $dhcpInterfaceName }} netns ${pid} || nsenter -t1 -n ip link delete {{ $dhcpInterfaceName }} # Set the macvlan interface up - ip link set {{ $macvlanInterfaceName }} up + ip link set {{ $dhcpInterfaceName }} up # Set the IP address - ip addr add {{ .Values.stack.loadBalancerIP }}/32 dev {{ $macvlanInterfaceName }} noprefixroute + ip addr add 127.1.1.1/32 dev {{ $dhcpInterfaceName }} noprefixroute image: alpine securityContext: privileged: true diff --git a/tinkerbell/stack/values.yaml b/tinkerbell/stack/values.yaml index d2e36e0e..fbba74ad 100644 --- a/tinkerbell/stack/values.yaml +++ b/tinkerbell/stack/values.yaml @@ -26,7 +26,7 @@ stack: kubevip: enabled: true name: kube-vip - image: ghcr.io/kube-vip/kube-vip:v0.6.3 + image: ghcr.io/kube-vip/kube-vip:v0.7.2 imagePullPolicy: IfNotPresent roleName: kube-vip-role roleBindingName: kube-vip-rolebinding @@ -50,6 +50,9 @@ stack: # When unset, the interface from the default route will be used. # sourceInterface: eno1 # TODO(jacobweinstock): add feature to be able to disable listening for broadcast traffic. + # interfaceMode determines how we create the interface needed to listen for DHCP broadcast traffic. + # by default macvlan is used. ipvlan is the only other option. + # interfaceMode: ipvlan # -- Overrides # The values defined here override those in the individual charts. Some of them require tweaking From 99ba9959841d47bb1b4540d736bf0f35f29e4d45 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Thu, 28 Mar 2024 15:50:36 -0600 Subject: [PATCH 2/5] Update tinkerbell/stack/templates/nginx.yaml fail Helm if an invalid mode is specified. Co-authored-by: Chris Doherty --- tinkerbell/stack/templates/nginx.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tinkerbell/stack/templates/nginx.yaml b/tinkerbell/stack/templates/nginx.yaml index 626f8624..43b75e1c 100644 --- a/tinkerbell/stack/templates/nginx.yaml +++ b/tinkerbell/stack/templates/nginx.yaml @@ -1,8 +1,8 @@ {{- if .Values.stack.enabled }} {{- $sourceInterface := .Values.stack.relay.sourceInterface -}} -{{- $dhcpInterfaceType := "macvlan" -}} -{{- if eq .Values.stack.relay.interfaceMode "ipvlan" -}} -{{- $dhcpInterfaceType = "ipvlan" -}} +{{- $ifaceModes := dict "ipvlan" "" "macvlan" "" -}} +{{- if not hasKey $ifaceModes .Values.stack.relay.interfaceMode -}} + {{- fail "invalid value in .stack.relay.interfaceMode: valid modes include ipvlan and macvlan" -}} {{- end -}} {{- $dhcpInterfaceName := printf "%s0" $dhcpInterfaceType -}} apiVersion: apps/v1 From 0af9007e136c5aa1ddfeda31f29e92c05a80c856 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Thu, 28 Mar 2024 16:12:20 -0600 Subject: [PATCH 3/5] Add back dhcpInterfaceType setting Signed-off-by: Jacob Weinstock --- tinkerbell/stack/templates/nginx.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tinkerbell/stack/templates/nginx.yaml b/tinkerbell/stack/templates/nginx.yaml index 43b75e1c..0bd1ecbf 100644 --- a/tinkerbell/stack/templates/nginx.yaml +++ b/tinkerbell/stack/templates/nginx.yaml @@ -1,8 +1,9 @@ {{- if .Values.stack.enabled }} {{- $sourceInterface := .Values.stack.relay.sourceInterface -}} {{- $ifaceModes := dict "ipvlan" "" "macvlan" "" -}} -{{- if not hasKey $ifaceModes .Values.stack.relay.interfaceMode -}} - {{- fail "invalid value in .stack.relay.interfaceMode: valid modes include ipvlan and macvlan" -}} +{{- $dhcpInterfaceType := default "macvlan" .Values.stack.relay.interfaceMode -}} +{{- if not (hasKey $ifaceModes $dhcpInterfaceType) -}} + {{- fail "invalid value at .stack.relay.interfaceMode: valid modes include ipvlan and macvlan" -}} {{- end -}} {{- $dhcpInterfaceName := printf "%s0" $dhcpInterfaceType -}} apiVersion: apps/v1 From c089a97266dff6e6ad21b7b82bd8b439edbbff6c Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Fri, 29 Mar 2024 20:36:50 -0600 Subject: [PATCH 4/5] Allow enable/disable of listening for broadcast traffic: In environments where a DHCP relay agent is employed it can be useful to not listen for broadcast traffic at all. Signed-off-by: Jacob Weinstock --- tinkerbell/stack/templates/nginx.yaml | 9 +++++++++ tinkerbell/stack/values.yaml | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tinkerbell/stack/templates/nginx.yaml b/tinkerbell/stack/templates/nginx.yaml index 0bd1ecbf..8399401b 100644 --- a/tinkerbell/stack/templates/nginx.yaml +++ b/tinkerbell/stack/templates/nginx.yaml @@ -6,6 +6,13 @@ {{- fail "invalid value at .stack.relay.interfaceMode: valid modes include ipvlan and macvlan" -}} {{- end -}} {{- $dhcpInterfaceName := printf "%s0" $dhcpInterfaceType -}} +{{- $listenBroadcast := true -}} +{{- if not (quote .Values.stack.relay.listenBroadcastTraffic | empty) -}} + {{- $listenBroadcast = .Values.stack.relay.listenBroadcastTraffic -}} +{{- end -}} +{{- if not $listenBroadcast -}} + {{- $dhcpInterfaceName = "eth0" -}} +{{- end -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -108,6 +115,7 @@ spec: path: {{ .Values.stack.hook.downloadsDest }} type: DirectoryOrCreate {{- end }} + {{- if $listenBroadcast }} initContainers: - name: relay-macvlan-interface command: @@ -139,6 +147,7 @@ spec: image: alpine securityContext: privileged: true + {{- end }} {{- if .Values.stack.service.enabled }} --- apiVersion: v1 diff --git a/tinkerbell/stack/values.yaml b/tinkerbell/stack/values.yaml index fbba74ad..aa33b547 100644 --- a/tinkerbell/stack/values.yaml +++ b/tinkerbell/stack/values.yaml @@ -49,7 +49,9 @@ stack: # sourceInterface is the Host/Node interface to use for listening for DHCP broadcast packets. # When unset, the interface from the default route will be used. # sourceInterface: eno1 - # TODO(jacobweinstock): add feature to be able to disable listening for broadcast traffic. + # enable/disable listening for broadcast traffic. Useful when the environment employs DHCP relay agent(s). + # default is true. + listenBroadcastTraffic: true # interfaceMode determines how we create the interface needed to listen for DHCP broadcast traffic. # by default macvlan is used. ipvlan is the only other option. # interfaceMode: ipvlan From aa9ae1c22417c87a032bb49c265fa6d86762677d Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Sat, 30 Mar 2024 15:46:40 -0600 Subject: [PATCH 5/5] Don't use default: Helm recommends setting the default value in the values.yaml. Signed-off-by: Jacob Weinstock --- tinkerbell/stack/templates/nginx.yaml | 8 ++++---- tinkerbell/stack/values.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tinkerbell/stack/templates/nginx.yaml b/tinkerbell/stack/templates/nginx.yaml index 8399401b..f1c2c535 100644 --- a/tinkerbell/stack/templates/nginx.yaml +++ b/tinkerbell/stack/templates/nginx.yaml @@ -1,7 +1,7 @@ {{- if .Values.stack.enabled }} {{- $sourceInterface := .Values.stack.relay.sourceInterface -}} {{- $ifaceModes := dict "ipvlan" "" "macvlan" "" -}} -{{- $dhcpInterfaceType := default "macvlan" .Values.stack.relay.interfaceMode -}} +{{- $dhcpInterfaceType := .Values.stack.relay.interfaceMode -}} {{- if not (hasKey $ifaceModes $dhcpInterfaceType) -}} {{- fail "invalid value at .stack.relay.interfaceMode: valid modes include ipvlan and macvlan" -}} {{- end -}} @@ -117,7 +117,7 @@ spec: {{- end }} {{- if $listenBroadcast }} initContainers: - - name: relay-macvlan-interface + - name: relay-broadcast-interface command: - /bin/sh - -c @@ -131,7 +131,7 @@ spec: if [ -z "$srcInterface" ]; then srcInterface=$(nsenter -t1 -n ip route | awk '/default/ {print $5}' | head -n1) fi - # Create a macvlan interface. TODO: If this fails, try again with a different name? + # Create the interface. TODO: If this fails, try again with a different name? {{- if eq $dhcpInterfaceType "ipvlan" }} nsenter -t1 -n ip link add {{ $dhcpInterfaceName }} link ${srcInterface} type ipvlan mode l2 {{- else }} @@ -140,7 +140,7 @@ spec: # Move the interface into the POD. pid=$(echo $$) nsenter -t1 -n ip link set {{ $dhcpInterfaceName }} netns ${pid} || nsenter -t1 -n ip link delete {{ $dhcpInterfaceName }} - # Set the macvlan interface up + # Set the interface up ip link set {{ $dhcpInterfaceName }} up # Set the IP address ip addr add 127.1.1.1/32 dev {{ $dhcpInterfaceName }} noprefixroute diff --git a/tinkerbell/stack/values.yaml b/tinkerbell/stack/values.yaml index aa33b547..e76d2573 100644 --- a/tinkerbell/stack/values.yaml +++ b/tinkerbell/stack/values.yaml @@ -54,7 +54,7 @@ stack: listenBroadcastTraffic: true # interfaceMode determines how we create the interface needed to listen for DHCP broadcast traffic. # by default macvlan is used. ipvlan is the only other option. - # interfaceMode: ipvlan + interfaceMode: macvlan # -- Overrides # The values defined here override those in the individual charts. Some of them require tweaking