playbooks: Add a playbook to configure NICs IRQs affinity

This playbook is used to configure the NICs IRQs affinity to the CPUs.
It create a systemd service that will run a python script to set the
IRQs affinity of the NICs to the CPUs.

To use it add the Ansible variable nics_affinity to the inventory file.
For example:
nics_affinity:
  - eth0: 0-3,4-7
  - eth1: 8-11,12-15

Signed-off-by: Mathieu Dupré <[email protected]>

examples/inventories/advanced_inventory_cluster_example.yaml

@@ -77,6 +77,12 @@ all:
               # /boot/syslinux.cfg on BIOS machine. Default is
               # /boot/EFI/BOOT/grub.cfg used by UEFI machines.
               bootloader_config_file: /boot/syslinux.cfg
+              # Configure Host NICs IRQs affinity. Useful when you use the
+              # macvlan driver the VMs or containers.
+              nics_affinity:
+              - eth0: 0-3,4-7
+              - eth1: 8-11,12-15
+
 
 
           # hypervisors groupe variables

examples/inventories/advanced_inventory_standalone_example.yaml

@@ -38,6 +38,11 @@ all:
               # bridge, and it also allows hypervisors and VMs to be configured
               # via a single interface.
               create_br0_bridge: true
+              # Configure Host NICs IRQs affinity. Useful when you use the
+              # macvlan driver the VMs or containers.
+              nics_affinity:
+              - eth0: 0-3,4-7
+              - eth1: 8-11,12-15
               # OVS bridges and ports description.
               # See OVS_configuration.adoc for more details.
               ovs_bridges:

playbooks/cluster_setup_configure_nic_irq_affinity.yaml

@@ -0,0 +1,15 @@
+# Copyright (C) 2024 Savoir-faire Linux, Inc
+# SPDX-License-Identifier: Apache-2.0
+
+# Configure the hosts NICs IRQs affinity
+# This is useful is you use macvlan driver for your containers or VMs
+
+---
+- name: Configure NICs IRQs affinity
+  hosts: hypervisors
+  gather_facts: false
+  become: true
+  tasks:
+    - name: Run the NICs IRQs affinity configuration tasks
+      include_tasks: tasks/setup_nic_irq_affinity.yaml
+      when: nics_affinity is defined

playbooks/tasks/setup_nic_irq_affinity.yaml

@@ -0,0 +1,28 @@
+# Copyright (C) 2024 Savoir-faire Linux, Inc
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Copy NIC irq affinity script
+  copy:
+    src: ../src/setup_nic_irq_affinity.py
+    dest: /etc/systemd/system/setup_nic_irq_affinity.py
+    mode: 0755
+
+- name: Copy NIC irq affinity service
+  copy:
+    src: ../src/setup_nic_irq_affinity.service
+    dest: /etc/systemd/system/setup_nic_irq_affinity.service
+    mode: 0644
+
+- name: Copy NIC irq affinity environment file
+  template:
+    src: ../templates/setup_nic_irq_affinity.j2
+    dest: /etc/default/setup_nic_irq_affinity
+    mode: 0644
+
+- name: Enable and start NIC irq affinity service
+  systemd:
+    name: setup_nic_irq_affinity.service
+    enabled: yes
+    state: started
+    daemon_reload: yes

src/setup_nic_irq_affinity.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# Copyright (C) 2024 Savoir-faire Linux, Inc
+# SPDX-License-Identifier: Apache-2.0
+
+# This script is used to set the IRQs affinity of the NICs to the CPUs
+# in order to optimize the performance of the network interface.
+# This takes takes a list of NICs and a list of CPUs and sets the IRQs
+
+import argparse
+import os
+import sys
+
+
+def args_parser():
+    parser = argparse.ArgumentParser(
+        description="Set the IRQs affinity of the NICs to the CPUs"
+    )
+    parser.add_argument(
+        "--nic",
+        action="append",
+        help="NICs to set the IRQs affinity",
+        required=True,
+        type=str,
+    )
+    parser.add_argument(
+        "--cpu",
+        help="CPUs to set the IRQs affinity (in the format 0-3,4-7,8-11,12-15)",
+        action="append",
+        type=str,
+        required=True,
+    )
+    return parser.parse_args()
+
+
+def get_irqs(nic):
+    irqs = []
+    # Iterate over the files in /proc/irq
+    for irq in os.listdir("/proc/irq"):
+        # Check if the file is a directory
+        if os.path.isdir("/proc/irq/" + irq):
+            # Check if there is a directory which begins with the NIC name
+            # Iterate over the files in /proc/irq/irq
+            for irq_file in os.listdir("/proc/irq/" + irq):
+                # Check if the file is a directory
+                if os.path.isdir("/proc/irq/" + irq + "/" + irq_file):
+                    # Check if there is a directory which begins with the NIC name
+                    if irq_file.startswith(nic):
+                        # Append the IRQ number to the list
+                        irqs.append(irq)
+    return irqs
+
+
+def set_irqs_affinity(irqs, cpus):
+    for irq in irqs:
+        # Set the affinity of the IRQ to the CPUs
+        with open("/proc/irq/" + irq + "/smp_affinity_list", "w") as f:
+            f.write(cpus)
+
+
+def main():
+    args = args_parser()
+    i = 0
+    if len(args.nic) != len(args.cpu):
+        print("The number of NICs and CPUs must be the same", file=sys.stderr)
+        sys.exit(1)
+    for nic in args.nic:
+        irqs = get_irqs(nic)
+        set_irqs_affinity(irqs, args.cpu[i])
+        i += 1
+
+
+if __name__ == "__main__":
+    main()

src/setup_nic_irq_affinity.service

@@ -0,0 +1,13 @@
+# Copyright (C) 2024 Savoir-faire Linux, Inc
+# SPDX-License-Identifier: Apache-2.0
+[Unit]
+Description=Configure NIC IRQs affinity
+After=irqbalance.service
+
+[Service]
+Type=oneshot
+EnvironmentFile=/etc/default/setup_nic_irq_affinity
+ExecStart=/etc/systemd/system/setup_nic_irq_affinity.py $SETUP_IRQ_AFFINITY_NICS $SETUP_IRQ_AFFINITY_CPUS
+
+[Install]
+WantedBy=irqbalance.service

templates/setup_nic_irq_affinity.j2

@@ -0,0 +1,14 @@
+# Copyright (C) 2024 Savoir-faire Linux, Inc
+# SPDX-License-Identifier: Apache-2.0
+
+{% set nics = [] %}
+{% set cpus = [] %}
+{% for item in nics_affinity %}
+  {% for nic, cpu in item.items() %}
+    {{ nics.append(nic) }}
+    {{ cpus.append(cpu) }}
+  {% endfor %}
+{% endfor %}
+
+SETUP_IRQ_AFFINITY_NICS=--nic {{ nics | join(' --nic ') }}
+SETUP_IRQ_AFFINITY_CPUS=--cpu {{ cpus | join(' --cpu ')}}