prod: fix NIC device names via udev rules

Previously we had homogeneous worker node hardware, however, with the addition of 2x new NVIDIA V100 worker hosts this is no longer the case. These new hosts have different NIC devices than the rest of the worker nodes. This adds the following new machineconfigs in order to map all of the NICs in the cluster to nic1 and nic2 devices: - disable-net-ifnames - custom-udev-rules The configure-bond0 machineconfig has been updated to use nic1 and nic2 vs the old eno1 and eno2 device names. Previously, this machineconfig was only being applied on the worker nodes. This patch also extends that configuration to the controllers in order to setup an LACP bond on those hosts.
OCP-on-NERC · Sep 21, 2023 · 9007d68 · 9007d68
1 parent 33de1ee
commit 9007d68
Show file tree

Hide file tree

Showing 23 changed files with 243 additions and 15 deletions.
diff --git a/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml
@@ -25,6 +25,8 @@ resources:
 - apiserver/cluster.yaml
 - secretstores
 - issuers
+- machineconfigs/disable-net-ifnames
+- machineconfigs/udev-rules
 - machineconfigs/configure-bond0
 - nodenetworkconfigurationpolicies
 - clusterversion.yaml

diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/Makefile b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/Makefile
@@ -1,5 +1,6 @@
 MACHINECONFIGS = \
-	machineconfig.yaml
+	bond0-controller.yaml \
+	bond0-worker.yaml
 
 %.yaml: src/%.bu $(wildcard src/*.nmconnection)
 	butane -o $@ -d src $<

diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/README.md b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/README.md
@@ -0,0 +1,7 @@
+Including file content in ignition configs is a pain, because it has to be base64 encoded. The `Makefile` in this directory uses [Butane][] to transpile `MachineConfig` resources, automatically including and encoding content from files.
+
+Read "[Creating machine configs with Butane][]" in the [OpenShift documentation][] for more information.
+
+[butane]: https://coreos.github.io/butane/
+[openshift documentation]: https://docs.openshift.com/container-platform/4.10/installing/install_config/installing-customizing.html
+[creating machine configs with butane]: https://docs.openshift.com/container-platform/4.10/installing/install_config/installing-customizing.html#installation-special-config-butane_installing-customizing
diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/bond0-controller.yaml b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/bond0-controller.yaml
@@ -0,0 +1,28 @@
+# Generated by Butane; do not edit
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: master
+  name: configure-bond0-controller
+spec:
+  config:
+    ignition:
+      version: 3.2.0
+    storage:
+      files:
+        - contents:
+            compression: gzip
+            source: data:;base64,H4sIAAAAAAAC/0zMwaqDMBCF4f15FyXeKxfvYp5EskiTKQY6M6IToW9fLEK7O/BzvjmbKmevphG10M20BPhz5fdEVeftnjJ3moSvnJrb9SPfGuOD9F+t2x/p4J0GYGZfeFP2CPFG/yEEYD61CLHCNIWf/jcVSK1iSsN49roeY4SwL1bohFGWvHZeha05TX9jCHgFAAD///WlE5PCAAAA
+          mode: 384
+          path: /etc/NetworkManager/system-connections/bond0.nmconnection
+        - contents:
+            compression: ""
+            source: data:,%5Bconnection%5D%0Aid%3Dnic1%0Atype%3Dethernet%0Ainterface-name%3Dnic1%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A
+          mode: 384
+          path: /etc/NetworkManager/system-connections/nic1.nmconnection
+        - contents:
+            compression: ""
+            source: data:,%5Bconnection%5D%0Aid%3Dnic2%0Atype%3Dethernet%0Ainterface-name%3Dnic2%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A
+          mode: 384
+          path: /etc/NetworkManager/system-connections/nic2.nmconnection
diff --git a/...onfigs/configure-bond0/machineconfig.yaml → ...configs/configure-bond0/bond0-worker.yaml b/...onfigs/configure-bond0/machineconfig.yaml → ...configs/configure-bond0/bond0-worker.yaml
@@ -4,7 +4,7 @@ kind: MachineConfig
 metadata:
   labels:
     machineconfiguration.openshift.io/role: worker
-  name: configure-bond0
+  name: configure-bond0-worker
 spec:
   config:
     ignition:
@@ -18,11 +18,11 @@ spec:
           path: /etc/NetworkManager/system-connections/bond0.nmconnection
         - contents:
             compression: ""
-            source: data:,%5Bconnection%5D%0Aid%3Deno1%0Atype%3Dethernet%0Ainterface-name%3Deno1%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A
+            source: data:,%5Bconnection%5D%0Aid%3Dnic1%0Atype%3Dethernet%0Ainterface-name%3Dnic1%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A
           mode: 384
-          path: /etc/NetworkManager/system-connections/eno1.nmconnection
+          path: /etc/NetworkManager/system-connections/nic1.nmconnection
         - contents:
             compression: ""
-            source: data:,%5Bconnection%5D%0Aid%3Deno2%0Atype%3Dethernet%0Ainterface-name%3Deno2%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A
+            source: data:,%5Bconnection%5D%0Aid%3Dnic2%0Atype%3Dethernet%0Ainterface-name%3Dnic2%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A
           mode: 384
-          path: /etc/NetworkManager/system-connections/eno2.nmconnection
+          path: /etc/NetworkManager/system-connections/nic2.nmconnection
diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/kustomization.yaml
@@ -1,4 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- machineconfig.yaml
+  - bond0-controller.yaml
+  - bond0-worker.yaml
diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/src/bond0-controller.bu b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/src/bond0-controller.bu
@@ -0,0 +1,20 @@
+variant: openshift
+version: 4.10.0
+metadata:
+  name: configure-bond0-controller
+  labels:
+    machineconfiguration.openshift.io/role: master
+storage:
+  files:
+    - path: /etc/NetworkManager/system-connections/bond0.nmconnection
+      mode: 0600
+      contents:
+        local: bond0.nmconnection
+    - path: /etc/NetworkManager/system-connections/nic1.nmconnection
+      mode: 0600
+      contents:
+        local: nic1.nmconnection
+    - path: /etc/NetworkManager/system-connections/nic2.nmconnection
+      mode: 0600
+      contents:
+        local: nic2.nmconnection
diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/src/bond0-worker.bu b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/src/bond0-worker.bu
@@ -0,0 +1,20 @@
+variant: openshift
+version: 4.10.0
+metadata:
+  name: configure-bond0-worker
+  labels:
+    machineconfiguration.openshift.io/role: worker
+storage:
+  files:
+    - path: /etc/NetworkManager/system-connections/bond0.nmconnection
+      mode: 0600
+      contents:
+        local: bond0.nmconnection
+    - path: /etc/NetworkManager/system-connections/nic1.nmconnection
+      mode: 0600
+      contents:
+        local: nic1.nmconnection
+    - path: /etc/NetworkManager/system-connections/nic2.nmconnection
+      mode: 0600
+      contents:
+        local: nic2.nmconnection
diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/src/machineconfig.bu b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/configure-bond0/src/machineconfig.bu
@@ -10,11 +10,11 @@ storage:
       mode: 0600
       contents:
         local: bond0.nmconnection
-    - path: /etc/NetworkManager/system-connections/eno1.nmconnection
+    - path: /etc/NetworkManager/system-connections/nic1.nmconnection
       mode: 0600
       contents:
-        local: eno1.nmconnection
-    - path: /etc/NetworkManager/system-connections/eno2.nmconnection
+        local: nic1.nmconnection
+    - path: /etc/NetworkManager/system-connections/nic2.nmconnection
       mode: 0600
       contents:
-        local: eno2.nmconnection
+        local: nic2.nmconnection
diff --git a/...igs/configure-bond0/src/eno1.nmconnection → ...igs/configure-bond0/src/nic1.nmconnection b/...igs/configure-bond0/src/eno1.nmconnection → ...igs/configure-bond0/src/nic1.nmconnection
@@ -1,7 +1,7 @@
 [connection]
-id=eno1
+id=nic1
 type=ethernet
-interface-name=eno1
+interface-name=nic1
 master=bond0
 slave-type=bond
 autoconnect=true

diff --git a/...igs/configure-bond0/src/eno2.nmconnection → ...igs/configure-bond0/src/nic2.nmconnection b/...igs/configure-bond0/src/eno2.nmconnection → ...igs/configure-bond0/src/nic2.nmconnection
@@ -1,7 +1,7 @@
 [connection]
-id=eno2
+id=nic2
 type=ethernet
-interface-name=eno2
+interface-name=nic2
 master=bond0
 slave-type=bond
 autoconnect=true

diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/disable-net-ifnames/controller.yaml b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/disable-net-ifnames/controller.yaml
@@ -0,0 +1,13 @@
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  name: disable-net-ifnames-controller
+  labels:
+    machineconfiguration.openshift.io/role: master
+spec:
+  config:
+    ignition:
+      version: 3.2.0
+  kernelArguments:
+    - net.ifnames=0
+    - biosdevname=0
diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/disable-net-ifnames/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/disable-net-ifnames/kustomization.yaml
@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - controller.yaml
+  - worker.yaml
diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/disable-net-ifnames/worker.yaml b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/disable-net-ifnames/worker.yaml
@@ -0,0 +1,13 @@
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  name: disable-net-ifnames-worker
+  labels:
+    machineconfiguration.openshift.io/role: worker
+spec:
+  config:
+    ignition:
+      version: 3.2.0
+  kernelArguments:
+    - net.ifnames=0
+    - biosdevname=0
diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/Makefile b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/Makefile
@@ -0,0 +1,11 @@
+MACHINECONFIGS = \
+	custom-udev-rules-controller.yaml \
+	custom-udev-rules-worker.yaml
+
+%.yaml: src/%.bu
+	butane -o $@ -d src $<
+
+all: $(MACHINECONFIGS)
+
+clean:
+	rm -f $(MACHINECONFIGS)
diff --git a/...-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/custom-udev-rules-controller.yaml b/...-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/custom-udev-rules-controller.yaml
@@ -0,0 +1,28 @@
+# Generated by Butane; do not edit
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: master
+  name: custom-udev-rules-controller
+spec:
+  config:
+    ignition:
+      version: 3.2.0
+    storage:
+      files:
+        - contents:
+            compression: gzip
+            source: data:;base64,H4sIAAAAAAAC/woOdQqODA5x9bW1VcpLLVHScQnyDHMNCra1VcrNqTCNT84vSlXS8XYN8nP1AQkaGBgYWBmYWRkY6Bko6Tg6h3j6+9naKiWmpCjp+Dn6utoq5WUmGypxUWCuIU5zjZS4AAEAAP//AAlzT7AAAAA=
+          mode: 420
+          path: /etc/udev/rules.d/90-mlx5-core.rules
+        - contents:
+            compression: gzip
+            source: data:;base64,H4sIAAAAAAAC/woOdQqODA5x9bW1VcpLLVHScQnyDHMNCra1VUrKqzCqUNLxdg3yc/UBCRgYGBhYGRhaGRjoGSjpODqHePr72doqJaakKOn4Ofq62irlZSYbKnGRaaYhTjONlLgAAQAA//9tGntmqAAAAA==
+          mode: 420
+          path: /etc/udev/rules.d/90-bnx2x.rules
+        - contents:
+            compression: gzip
+            source: data:;base64,H4sIAAAAAAAC/woOdQqODA5x9bW1VcpLLVHScQnyDHMNCra1VUrKqyiJT81T0vF2DfJz9QEJGRgYGFgZWlgZGOgZKOk4Ood4+vvZ2iolpqQo6fg5+rraKuVlJhsqcZFtqiFOU42UuAABAAD//64yhGWsAAAA
+          mode: 420
+          path: /etc/udev/rules.d/90-bnxt_en.rules
diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/custom-udev-rules-worker.yaml b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/custom-udev-rules-worker.yaml
@@ -0,0 +1,28 @@
+# Generated by Butane; do not edit
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: worker
+  name: custom-udev-rules-worker
+spec:
+  config:
+    ignition:
+      version: 3.2.0
+    storage:
+      files:
+        - contents:
+            compression: gzip
+            source: data:;base64,H4sIAAAAAAAC/woOdQqODA5x9bW1VcpLLVHScQnyDHMNCra1VcrNqTCNT84vSlXS8XYN8nP1AQkaGBgYWBmYWRkY6Bko6Tg6h3j6+9naKiWmpCjp+Dn6utoq5WUmGypxUWCuIU5zjZS4AAEAAP//AAlzT7AAAAA=
+          mode: 420
+          path: /etc/udev/rules.d/90-mlx5-core.rules
+        - contents:
+            compression: gzip
+            source: data:;base64,H4sIAAAAAAAC/woOdQqODA5x9bW1VcpLLVHScQnyDHMNCra1VUrKqzCqUNLxdg3yc/UBCRgYGBhYGRhaGRjoGSjpODqHePr72doqJaakKOn4Ofq62irlZSYbKnGRaaYhTjONlLgAAQAA//9tGntmqAAAAA==
+          mode: 420
+          path: /etc/udev/rules.d/90-bnx2x.rules
+        - contents:
+            compression: gzip
+            source: data:;base64,H4sIAAAAAAAC/woOdQqODA5x9bW1VcpLLVHScQnyDHMNCra1VUrKqyiJT81T0vF2DfJz9QEJGRgYGFgZWlgZGOgZKOk4Ood4+vvZ2iolpqQo6fg5+rraKuVlJhsqcZFtqiFOU42UuAABAAD//64yhGWsAAAA
+          mode: 420
+          path: /etc/udev/rules.d/90-bnxt_en.rules
diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/kustomization.yaml
@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - custom-udev-rules-controller.yaml
+  - custom-udev-rules-worker.yaml
diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/src/bnx2x.rules b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/src/bnx2x.rules
@@ -0,0 +1,2 @@
+SUBSYSTEM=="net",DRIVERS=="bnx2x",KERNELS=="0000:01:00.0",ACTION=="add",NAME="nic1"
+SUBSYSTEM=="net",DRIVERS=="bnx2x",KERNELS=="0000:01:00.1",ACTION=="add",NAME="nic2"
diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/src/bnxt_en.rules b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/src/bnxt_en.rules
@@ -0,0 +1,2 @@
+SUBSYSTEM=="net",DRIVERS=="bnxt_en",KERNELS=="0000:18:00.0",ACTION=="add",NAME="nic1"
+SUBSYSTEM=="net",DRIVERS=="bnxt_en",KERNELS=="0000:18:00.1",ACTION=="add",NAME="nic2"
diff --git a/...cope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/src/custom-udev-rules-controller.bu b/...cope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/src/custom-udev-rules-controller.bu
@@ -0,0 +1,20 @@
+variant: openshift
+version: 4.10.0
+metadata:
+  name: custom-udev-rules-controller
+  labels:
+    machineconfiguration.openshift.io/role: master
+storage:
+  files:
+    - path: /etc/udev/rules.d/90-mlx5-core.rules
+      contents:
+        local: mlx5_core.rules
+      mode: 0644
+    - path: /etc/udev/rules.d/90-bnx2x.rules
+      contents:
+        local: bnx2x.rules
+      mode: 0644
+    - path: /etc/udev/rules.d/90-bnxt_en.rules
+      contents:
+        local: bnxt_en.rules
+      mode: 0644
diff --git a/...er-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/src/custom-udev-rules-worker.bu b/...er-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/src/custom-udev-rules-worker.bu
@@ -0,0 +1,20 @@
+variant: openshift
+version: 4.10.0
+metadata:
+  name: custom-udev-rules-worker
+  labels:
+    machineconfiguration.openshift.io/role: worker
+storage:
+  files:
+    - path: /etc/udev/rules.d/90-mlx5-core.rules
+      contents:
+        local: mlx5_core.rules
+      mode: 0644
+    - path: /etc/udev/rules.d/90-bnx2x.rules
+      contents:
+        local: bnx2x.rules
+      mode: 0644
+    - path: /etc/udev/rules.d/90-bnxt_en.rules
+      contents:
+        local: bnxt_en.rules
+      mode: 0644
diff --git a/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/src/mlx5_core.rules b/cluster-scope/overlays/nerc-ocp-prod/machineconfigs/udev-rules/src/mlx5_core.rules
@@ -0,0 +1,2 @@
+SUBSYSTEM=="net",DRIVERS=="mlx5_core",KERNELS=="0000:06:00.0",ACTION=="add",NAME="nic1"
+SUBSYSTEM=="net",DRIVERS=="mlx5_core",KERNELS=="0000:06:00.1",ACTION=="add",NAME="nic2"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		SUBSYSTEM=="net",DRIVERS=="bnx2x",KERNELS=="0000:01:00.0",ACTION=="add",NAME="nic1"
		SUBSYSTEM=="net",DRIVERS=="bnx2x",KERNELS=="0000:01:00.1",ACTION=="add",NAME="nic2"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		SUBSYSTEM=="net",DRIVERS=="bnxt_en",KERNELS=="0000:18:00.0",ACTION=="add",NAME="nic1"
		SUBSYSTEM=="net",DRIVERS=="bnxt_en",KERNELS=="0000:18:00.1",ACTION=="add",NAME="nic2"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		SUBSYSTEM=="net",DRIVERS=="mlx5_core",KERNELS=="0000:06:00.0",ACTION=="add",NAME="nic1"
		SUBSYSTEM=="net",DRIVERS=="mlx5_core",KERNELS=="0000:06:00.1",ACTION=="add",NAME="nic2"