simplify monitor address setting

this drops the following parameters: - monitor_address_block - monitor_interface - monitor_address The monitor address will be automatically set from `public_network` parameter. Signed-off-by: Guillaume Abrioux <[email protected]>
ceph · Mar 8, 2024 · 256ee06 · 256ee06
1 parent 62e48fc
commit 256ee06
Show file tree

Hide file tree

Showing 73 changed files with 61 additions and 230 deletions.
diff --git a/Vagrantfile b/Vagrantfile
@@ -88,7 +88,6 @@ ansible_provision = proc do |ansible|
   if DOCKER then
     ansible.extra_vars = ansible.extra_vars.merge({
       containerized_deployment: 'true',
-      monitor_interface: ETH,
       ceph_mon_docker_subnet: ansible.extra_vars[:public_network],
       devices: settings['disks'],
       radosgw_interface: ETH,
@@ -97,16 +96,13 @@ ansible_provision = proc do |ansible|
   else
     ansible.extra_vars = ansible.extra_vars.merge({
       devices: settings['disks'],
-      monitor_interface: ETH,
       radosgw_interface: ETH,
       os_tuning_params: settings['os_tuning_params'],
     })
   end
 
   if BOX == 'linode' then
     ansible.sudo = true
-    # Use monitor_address_block instead of monitor_interface:
-    ansible.extra_vars.delete(:monitor_interface)
     # Use radosgw_address_block instead of radosgw_interface:
     ansible.extra_vars.delete(:radosgw_interface)
     ansible.extra_vars = ansible.extra_vars.merge({

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -228,7 +228,6 @@ An example configuration that deploys the upstream ``octopus`` version of Ceph w
    ceph_repository: community
    public_network: "192.168.3.0/24"
    cluster_network: "192.168.4.0/24"
-   monitor_interface: eth1
    devices:
      - '/dev/sda'
      - '/dev/sdb'
@@ -238,7 +237,6 @@ selection or other aspects of your cluster.
 
 - ``ceph_origin``
 - ``public_network``
-- ``monitor_interface`` or ``monitor_address``
 
 
 When deploying RGW instance(s) you are required to set the ``radosgw_interface`` or ``radosgw_address`` config option.

diff --git a/docs/source/testing/scenarios.rst b/docs/source/testing/scenarios.rst
@@ -130,15 +130,15 @@ way so that a vagrant environment can be isolated to the given scenario.
 The ``hosts`` file should contain the hosts needed for the scenario. This might
 seem a bit repetitive since machines are already defined in
 :ref:`vagrant_variables` but it allows granular changes to hosts (for example
-defining an interface vs. an IP on a monitor) which can help catch issues in
+defining different public_network values between monitors) which can help catch issues in
 ``ceph-ansible`` configuration. For example:
 
 .. code-block:: ini
 
    [mons]
-   mon0 monitor_address=192.168.5.10
-   mon1 monitor_address=192.168.5.11
-   mon2 monitor_interface=eth1
+   mon0 public_network=192.168.1.0/24
+   mon1 public_network=192.168.2.0/24
+   mon2 public_network=192.168.3.0/24
 
 .. _group_vars:
 

diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample
@@ -311,14 +311,6 @@ dummy:
 #rbd_client_admin_socket_path: /var/run/ceph # must be writable by QEMU and allowed by SELinux or AppArmor
 
 ## Monitor options
-#
-# You must define either monitor_interface, monitor_address or monitor_address_block.
-# These variables must be defined at least in all.yml and overrided if needed (inventory host file or group_vars/*.yml).
-# Eg. If you want to specify for each monitor which address the monitor will bind to you can set it in your **inventory host file** by using 'monitor_address' variable.
-# Preference will go to monitor_address if both monitor_address and monitor_interface are defined.
-#monitor_interface: interface
-#monitor_address: x.x.x.x
-#monitor_address_block: subnet
 # set to either ipv4 or ipv6, whichever your network is using
 #ip_version: ipv4
 

diff --git a/group_vars/mons.yml.sample b/group_vars/mons.yml.sample
@@ -62,7 +62,7 @@ dummy:
 #    - --default-mon-cluster-log-to-stderr=true
 #    - -i={{ monitor_name }}
 #    - --mon-data=/var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }}
-#    - --public-addr={{ _current_monitor_address }}
+#    - --public-addr={{ _monitor_addresses[inventory_hostname] }}
 #    - --mon-initial-members={{ groups[mon_group_name] | join(',') }}
 
 

diff --git a/infrastructure-playbooks/cephadm.yml b/infrastructure-playbooks/cephadm.yml
@@ -30,15 +30,6 @@
       run_once: true
       when: groups[mgr_group_name] is undefined or groups[mgr_group_name] | length == 0
 
-    - name: Validate monitor network configuration
-      ansible.builtin.fail:
-        msg: "Either monitor_address, monitor_address_block or monitor_interface must be provided"
-      when:
-        - mon_group_name in group_names
-        - monitor_address == 'x.x.x.x'
-        - monitor_address_block == 'subnet'
-        - monitor_interface == 'interface'
-
     - name: Validate dashboard configuration
       when: dashboard_enabled | bool
       run_once: true
@@ -158,7 +149,7 @@
 
     - name: Bootstrap the new cluster
       cephadm_bootstrap:
-        mon_ip: "{{ _current_monitor_address }}"
+        mon_ip: "{{ _monitor_addresses[inventory_hostname] }}"
         image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
         docker: "{{ true if container_binary == 'docker' else false }}"
         pull: false

diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml
@@ -304,7 +304,7 @@
           delegate_facts: true
 
         - name: Non container | waiting for the monitor to join the quorum...
-          ansible.builtin.command: ceph --cluster "{{ cluster }}" -m "{{ hostvars[groups[mon_group_name][0]]['_current_monitor_address'] }}" quorum_status --format json
+          ansible.builtin.command: ceph --cluster "{{ cluster }}" -m "{{ _monitor_addresses[groups['mons'][0]] }}" quorum_status --format json
           register: ceph_health_raw
           until:
             - ceph_health_raw.rc == 0
@@ -317,7 +317,7 @@
 
         - name: Container | waiting for the containerized monitor to join the quorum...
           ansible.builtin.command: >
-            {{ container_binary }} exec ceph-mon-{{ ansible_facts['hostname'] }} ceph --cluster "{{ cluster }}" -m "{{ hostvars[groups[mon_group_name][0]]['_current_monitor_address'] }}" quorum_status --format json
+            {{ container_binary }} exec ceph-mon-{{ ansible_facts['hostname'] }} ceph --cluster "{{ cluster }}" -m "{{ _monitor_addresses[groups['mons'][0]] }}" quorum_status --format json
           register: ceph_health_raw
           until:
             - ceph_health_raw.rc == 0

diff --git a/roles/ceph-config/templates/ceph.conf.j2 b/roles/ceph-config/templates/ceph.conf.j2
@@ -13,11 +13,11 @@ auth client required = none
 
 fsid = {{ fsid }}
 mon host = {% if nb_mon > 0 %}
-{% for host in _monitor_addresses -%}
+{% for name, addr in _monitor_addresses.items() -%}
 {% if mon_host_v1.enabled | bool %}
-{% set _v1 = ',v1:' + host.addr + mon_host_v1.suffix %}
+{% set _v1 = ',v1:' + addr + mon_host_v1.suffix %}
 {% endif %}
-[{{ "v2:" + host.addr + mon_host_v2.suffix }}{{ _v1 | default('') }}]
+[{{ "v2:" + addr + mon_host_v2.suffix }}{{ _v1 | default('') }}]
 {%- if not loop.last -%},{%- endif %}
 {%- endfor %}
 {% elif nb_mon == 0 %}

diff --git a/roles/ceph-defaults/defaults/main.yml b/roles/ceph-defaults/defaults/main.yml
@@ -303,14 +303,6 @@ rbd_client_log_file: "{{ rbd_client_log_path }}/qemu-guest-$pid.log" # must be w
 rbd_client_admin_socket_path: /var/run/ceph # must be writable by QEMU and allowed by SELinux or AppArmor
 
 ## Monitor options
-#
-# You must define either monitor_interface, monitor_address or monitor_address_block.
-# These variables must be defined at least in all.yml and overrided if needed (inventory host file or group_vars/*.yml).
-# Eg. If you want to specify for each monitor which address the monitor will bind to you can set it in your **inventory host file** by using 'monitor_address' variable.
-# Preference will go to monitor_address if both monitor_address and monitor_interface are defined.
-monitor_interface: interface
-monitor_address: x.x.x.x
-monitor_address_block: subnet
 # set to either ipv4 or ipv6, whichever your network is using
 ip_version: ipv4
 

diff --git a/roles/ceph-facts/tasks/set_monitor_address.yml b/roles/ceph-facts/tasks/set_monitor_address.yml
@@ -1,59 +1,14 @@
 ---
-- name: Set_fact _monitor_addresses to monitor_address_block ipv4
+- name: Set_fact _monitor_addresses - ipv4
   ansible.builtin.set_fact:
-    _monitor_addresses: "{{ _monitor_addresses | default([]) + [{'name': item, 'addr': hostvars[item]['ansible_facts']['all_ipv4_addresses'] | ips_in_ranges(hostvars[item]['monitor_address_block'].split(',')) | first}] }}"
+    _monitor_addresses: "{{ _monitor_addresses | default({}) | combine({item: hostvars[item]['ansible_facts']['all_ipv4_addresses'] | ips_in_ranges(hostvars[item]['public_network'].split(',')) | first}) }}"
   with_items: "{{ groups.get(mon_group_name, []) }}"
   when:
-    - "item not in _monitor_addresses | default([]) | selectattr('name', 'defined') |  map(attribute='name') | list"
-    - hostvars[item]['monitor_address_block'] is defined
-    - hostvars[item]['monitor_address_block'] != 'subnet'
     - ip_version == 'ipv4'
 
-- name: Set_fact _monitor_addresses to monitor_address_block ipv6
+- name: Set_fact _monitor_addresses - ipv6
   ansible.builtin.set_fact:
-    _monitor_addresses: "{{ _monitor_addresses | default([]) + [{'name': item, 'addr': hostvars[item]['ansible_facts']['all_ipv6_addresses'] | ips_in_ranges(hostvars[item]['monitor_address_block'].split(',')) | last | ansible.utils.ipwrap}] }}"
+    _monitor_addresses: "{{ _monitor_addresses | default({}) | combine({item: hostvars[item]['ansible_facts']['all_ipv6_addresses'] | ips_in_ranges(hostvars[item]['public_network'].split(',')) | last | ansible.utils.ipwrap}) }}"
   with_items: "{{ groups.get(mon_group_name, []) }}"
   when:
-    - "item not in _monitor_addresses | default([]) | selectattr('name', 'defined') |  map(attribute='name') | list"
-    - hostvars[item]['monitor_address_block'] is defined
-    - hostvars[item]['monitor_address_block'] != 'subnet'
     - ip_version == 'ipv6'
-
-- name: Set_fact _monitor_addresses to monitor_address
-  ansible.builtin.set_fact:
-    _monitor_addresses: "{{ _monitor_addresses | default([]) + [{'name': item, 'addr': hostvars[item]['monitor_address'] | ansible.utils.ipwrap}] }}"
-  with_items: "{{ groups.get(mon_group_name, []) }}"
-  when:
-    - "item not in _monitor_addresses | default([]) | selectattr('name', 'defined') | map(attribute='name') | list"
-    - hostvars[item]['monitor_address'] is defined
-    - hostvars[item]['monitor_address'] != 'x.x.x.x'
-
-- name: Set_fact _monitor_addresses to monitor_interface - ipv4
-  ansible.builtin.set_fact:
-    _monitor_addresses: "{{ _monitor_addresses | default([]) + [{'name': item, 'addr': hostvars[item]['ansible_facts'][(hostvars[item]['monitor_interface'] | replace('-', '_'))][ip_version]['address'] | ansible.utils.ipwrap}] }}"
-  with_items: "{{ groups.get(mon_group_name, []) }}"
-  when:
-    - "item not in _monitor_addresses | default([]) | selectattr('name', 'defined') | map(attribute='name') | list"
-    - ip_version == 'ipv4'
-    - hostvars[item]['monitor_address_block'] | default('subnet')  == 'subnet'
-    - hostvars[item]['monitor_address'] | default('x.x.x.x') == 'x.x.x.x'
-    - hostvars[item]['monitor_interface'] | default('interface') != 'interface'
-
-- name: Set_fact _monitor_addresses to monitor_interface - ipv6
-  ansible.builtin.set_fact:
-    _monitor_addresses: "{{ _monitor_addresses | default([]) + [{'name': item, 'addr': hostvars[item]['ansible_facts'][(hostvars[item]['monitor_interface'] | replace('-', '_'))][ip_version][0]['address'] | ansible.utils.ipwrap}] }}"
-  with_items: "{{ groups.get(mon_group_name, []) }}"
-  when:
-    - "item not in _monitor_addresses | default([]) | selectattr('name', 'defined') | map(attribute='name') | list"
-    - ip_version == 'ipv6'
-    - hostvars[item]['monitor_address_block'] | default('subnet')  == 'subnet'
-    - hostvars[item]['monitor_address'] | default('x.x.x.x') == 'x.x.x.x'
-    - hostvars[item]['monitor_interface'] | default('interface') != 'interface'
-
-- name: Set_fact _current_monitor_address
-  ansible.builtin.set_fact:
-    _current_monitor_address: "{{ item.addr }}"
-  with_items: "{{ _monitor_addresses }}"
-  when:
-    - (inventory_hostname == item.name and not rolling_update | default(False) | bool)
-      or (rolling_update | default(False) | bool and item.name == groups.get(mon_group_name, [])[0])
diff --git a/roles/ceph-mon/defaults/main.yml b/roles/ceph-mon/defaults/main.yml
@@ -54,7 +54,7 @@ ceph_mon_container_params:
     - --default-mon-cluster-log-to-stderr=true
     - -i={{ monitor_name }}
     - --mon-data=/var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }}
-    - --public-addr={{ _current_monitor_address }}
+    - --public-addr={{ _monitor_addresses[inventory_hostname] }}
     - --mon-initial-members={{ groups[mon_group_name] | map('extract', hostvars, 'ansible_facts') | map(attribute='hostname') | join(',') }}
 
 

diff --git a/roles/ceph-mon/tasks/deploy_monitors.yml b/roles/ceph-mon/tasks/deploy_monitors.yml
@@ -158,13 +158,13 @@
   ansible.builtin.command: >
     {{ ceph_monmaptool_cmd }}
     --create
-    {% for host in _monitor_addresses -%}
+    {% for name, addr in _monitor_addresses.items() -%}
     --addv
-    {{ host.name }}
+    {{ hostvars[name]['ansible_facts']['hostname'] }}
     {% if mon_host_v1.enabled | bool %}
-    {% set _v1 = ',v1:' + host.addr + mon_host_v1.suffix %}
+    {% set _v1 = ',v1:' + addr + mon_host_v1.suffix %}
     {% endif %}
-    [{{ "v2:" + host.addr + mon_host_v2.suffix }}{{ _v1 | default('') }}]
+    [{{ "v2:" + addr + mon_host_v2.suffix }}{{ _v1 | default('') }}]
     {# {%- if not loop.last -%},{%- endif %} #}
     {%- endfor %}
     --enable-all-features

diff --git a/roles/ceph-validate/tasks/check_eth_mon.yml b/roles/ceph-validate/tasks/check_eth_mon.yml
diff --git a/roles/ceph-validate/tasks/check_ipaddr_mon.yml b/roles/ceph-validate/tasks/check_ipaddr_mon.yml
@@ -1,5 +1,5 @@
 ---
-- name: Check if network interface has an IP address in `monitor_address_block`
+- name: Check if network interface has an IP address in public_network
   ansible.builtin.fail:
-    msg: "{{ inventory_hostname }} does not have any {{ ip_version }} address on {{ monitor_address_block }}"
-  when: hostvars[inventory_hostname]['ansible_facts']['all_' + ip_version + '_addresses'] | ips_in_ranges(hostvars[inventory_hostname]['monitor_address_block'].split(',')) | length == 0
+    msg: "{{ inventory_hostname }} does not have any {{ ip_version }} address on {{ public_network }}"
+  when: hostvars[inventory_hostname]['ansible_facts']['all_' + ip_version + '_addresses'] | ips_in_ranges(hostvars[inventory_hostname]['public_network'].split(',')) | length == 0
diff --git a/roles/ceph-validate/tasks/main.yml b/roles/ceph-validate/tasks/main.yml
@@ -11,15 +11,6 @@
     msg: "osd_objectstore must be 'bluestore''"
   when: osd_objectstore not in ['bluestore']
 
-- name: Validate monitor network configuration
-  ansible.builtin.fail:
-    msg: "Either monitor_address, monitor_address_block or monitor_interface must be provided"
-  when:
-    - mon_group_name in group_names
-    - monitor_address == 'x.x.x.x'
-    - monitor_address_block == 'subnet'
-    - monitor_interface == 'interface'
-
 - name: Validate radosgw network configuration
   ansible.builtin.fail:
     msg: "Either radosgw_address, radosgw_address_block or radosgw_interface must be provided"
@@ -107,22 +98,6 @@
     - osd_group_name in group_names
     - not osd_auto_discovery | default(False) | bool
 
-- name: Include check_eth_mon.yml
-  ansible.builtin.include_tasks: check_eth_mon.yml
-  when:
-    - mon_group_name in group_names
-    - monitor_interface != "dummy"
-    - monitor_address == "x.x.x.x"
-    - monitor_address_block == "subnet"
-
-- name: Include check_ipaddr_mon.yml
-  ansible.builtin.include_tasks: check_ipaddr_mon.yml
-  when:
-    - mon_group_name in group_names
-    - monitor_interface == "interface"
-    - monitor_address == "x.x.x.x"
-    - monitor_address_block != "subnet"
-
 - name: Include check_eth_rgw.yml
   ansible.builtin.include_tasks: check_eth_rgw.yml
   when:

diff --git a/tests/functional/add-mdss/container/group_vars/all b/tests/functional/add-mdss/container/group_vars/all
@@ -6,7 +6,6 @@ containerized_deployment: true
 cluster: ceph
 public_network: "192.168.63.0/24"
 cluster_network: "192.168.64.0/24"
-monitor_interface: eth1
 radosgw_interface: eth1
 journal_size: 100
 osd_objectstore: "bluestore"

diff --git a/tests/functional/add-mdss/group_vars/all b/tests/functional/add-mdss/group_vars/all
@@ -4,7 +4,6 @@ ceph_repository: community
 cluster: ceph
 public_network: "192.168.61.0/24"
 cluster_network: "192.168.62.0/24"
-monitor_interface: eth1
 radosgw_interface: eth1
 journal_size: 100
 osd_objectstore: "bluestore"

diff --git a/tests/functional/add-mgrs/container/group_vars/all b/tests/functional/add-mgrs/container/group_vars/all
@@ -6,7 +6,6 @@ containerized_deployment: true
 cluster: ceph
 public_network: "192.168.75.0/24"
 cluster_network: "192.168.76.0/24"
-monitor_interface: eth1
 radosgw_interface: eth1
 journal_size: 100
 osd_objectstore: "bluestore"

diff --git a/tests/functional/add-mgrs/group_vars/all b/tests/functional/add-mgrs/group_vars/all
@@ -4,7 +4,6 @@ ceph_repository: dev
 cluster: ceph
 public_network: "192.168.73.0/24"
 cluster_network: "192.168.74.0/24"
-monitor_interface: eth1
 radosgw_interface: eth1
 journal_size: 100
 osd_objectstore: "bluestore"

diff --git a/tests/functional/add-mons/container/group_vars/all b/tests/functional/add-mons/container/group_vars/all
@@ -6,7 +6,6 @@ containerized_deployment: true
 cluster: ceph
 public_network: "192.168.55.0/24"
 cluster_network: "192.168.56.0/24"
-monitor_interface: eth1
 radosgw_interface: eth1
 journal_size: 100
 osd_objectstore: "bluestore"

diff --git a/tests/functional/add-mons/group_vars/all b/tests/functional/add-mons/group_vars/all
@@ -4,7 +4,6 @@ ceph_repository: dev
 cluster: ceph
 public_network: "192.168.53.0/24"
 cluster_network: "192.168.54.0/24"
-monitor_interface: eth1
 radosgw_interface: eth1
 journal_size: 100
 osd_objectstore: "bluestore"

diff --git a/tests/functional/add-osds/container/group_vars/all b/tests/functional/add-osds/container/group_vars/all
@@ -6,7 +6,6 @@ containerized_deployment: true
 cluster: ceph
 public_network: "192.168.55.0/24"
 cluster_network: "192.168.56.0/24"
-monitor_interface: "{{ 'eth1' if ansible_facts['distribution'] == 'CentOS' else 'ens6' }}"
 radosgw_interface: "{{ 'eth1' if ansible_facts['distribution'] == 'CentOS' else 'ens6' }}"
 journal_size: 100
 osd_objectstore: "bluestore"

diff --git a/tests/functional/add-osds/group_vars/all b/tests/functional/add-osds/group_vars/all
@@ -4,7 +4,6 @@ ceph_repository: community
 cluster: ceph
 public_network: "192.168.53.0/24"
 cluster_network: "192.168.54.0/24"
-monitor_interface: "{{ 'eth1' if ansible_facts['distribution'] == 'CentOS' else 'ens6' }}"
 radosgw_interface: "{{ 'eth1' if ansible_facts['distribution'] == 'CentOS' else 'ens6' }}"
 journal_size: 100
 osd_objectstore: "bluestore"

diff --git a/tests/functional/add-rbdmirrors/container/group_vars/all b/tests/functional/add-rbdmirrors/container/group_vars/all
@@ -6,7 +6,6 @@ containerized_deployment: true
 cluster: ceph
 public_network: "192.168.67.0/24"
 cluster_network: "192.168.68.0/24"
-monitor_interface: "{{ 'eth1' if ansible_facts['distribution'] == 'CentOS' else 'ens6' }}"
 radosgw_interface: "{{ 'eth1' if ansible_facts['distribution'] == 'CentOS' else 'ens6' }}"
 journal_size: 100
 osd_objectstore: "bluestore"