systemd: export params as a varaible

This can help to have extra params or modify the existing ones via group vars.

Signed-off-by: Seena Fallah <[email protected]>

group_vars/all.yml.sample

@@ -541,8 +541,6 @@ dummy:
 #    CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
 #    TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}"
 #  args:
-#    - --setuser=ceph
-#    - --setgroup=ceph
 #    - --default-log-to-file=false
 #    - --default-log-to-stderr=true
 #    - --default-log-stderr-prefix="debug "

group_vars/mdss.yml.sample

@@ -30,11 +30,15 @@ dummy:
 #ceph_mds_docker_memory_limit: "{{ ansible_facts['memtotal_mb'] }}m"
 #ceph_mds_docker_cpu_limit: 4
 
-# we currently for MDS_NAME to hostname because of a bug in ceph-docker
-# fix here: https://github.com/ceph/ceph-docker/pull/770
-# this will go away soon.
-#ceph_mds_docker_extra_env: -e MDS_NAME={{ ansible_facts['hostname'] }}
 #ceph_config_keys: [] # DON'T TOUCH ME
+# If you want to add parameters, you should retain the existing ones and include the new ones.
+#ceph_mds_container_params:
+#  volumes:
+#    - /var/lib/ceph/bootstrap-mds:/var/lib/ceph/bootstrap-mds:z
+#    - /var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:/var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:z
+#  args:
+#    - -f
+#    - -i={{ ansible_facts['hostname'] }}
 
 
 ###########

group_vars/mgrs.yml.sample

@@ -40,12 +40,18 @@ dummy:
 # Resource limitation
 # For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
 # Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
-# These options can be passed using the 'ceph_mgr_docker_extra_env' variable.
 #ceph_mgr_docker_memory_limit: "{{ ansible_facts['memtotal_mb'] }}m"
 #ceph_mgr_docker_cpu_limit: 1
 
-#ceph_mgr_docker_extra_env:
 #ceph_config_keys: [] # DON'T TOUCH ME
+# If you want to add parameters, you should retain the existing ones and include the new ones.
+#ceph_mgr_container_params:
+#  volumes:
+#    - /var/lib/ceph/mgr:/var/lib/ceph/mgr:z,rshared
+#    - /var/lib/ceph/bootstrap-mgr:/var/lib/ceph/bootstrap-mgr:z
+#  args:
+#    - -f
+#    - -i={{ ansible_facts['hostname'] }}
 
 
 ###########

group_vars/mons.yml.sample

@@ -44,18 +44,26 @@ dummy:
 # Resource limitation
 # For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
 # Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
-# These options can be passed using the 'ceph_mon_docker_extra_env' variable.
 #ceph_mon_docker_memory_limit: "{{ ansible_facts['memtotal_mb'] }}m"
 #ceph_mon_docker_cpu_limit: 1
 #ceph_mon_container_listen_port: 3300
 
-# Use this variable to add extra env configuration to run your mon container.
-# If you want to set a custom admin keyring you can set this variable like following:
-# ceph_mon_docker_extra_env: -e ADMIN_SECRET={{ admin_secret }}
-#ceph_mon_docker_extra_env:
+# Use this variable to modify the configuration to run your mon container.
 #mon_docker_privileged: false
 #mon_docker_net_host: true
 #ceph_config_keys: [] # DON'T TOUCH ME
+# If you want to add parameters, you should retain the existing ones and include the new ones.
+#ceph_mon_container_params:
+#  volumes:
+#    - /var/lib/ceph/mon:/var/lib/ceph/mon:z,rshared
+#  args:
+#    - -f
+#    - --default-mon-cluster-log-to-file=false
+#    - --default-mon-cluster-log-to-stderr=true
+#    - -i={{ monitor_name }}
+#    - --mon-data=/var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }}
+#    - --public-addr={{ _current_monitor_address }}
+#    - --mon-initial-members={{ groups[mon_group_name] | join(',') }}
 
 
 ###########

group_vars/osds.yml.sample

@@ -168,7 +168,6 @@ dummy:
 # Resource limitation
 # For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
 # Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
-# These options can be passed using the 'ceph_osd_docker_extra_env' variable.
 #ceph_osd_docker_memory_limit: "{{ ansible_facts['memtotal_mb'] }}m"
 #ceph_osd_docker_cpu_limit: 4
 
@@ -189,9 +188,22 @@ dummy:
 
 # ACTIVATE DEVICE
 #
-#ceph_osd_docker_extra_env:
 #ceph_osd_numactl_opts: ""
 
+# If you want to add parameters, you should retain the existing ones and include the new ones.
+#ceph_osd_container_params:
+#  volumes:
+#    - /dev:/dev
+#    - /var/lib/ceph/bootstrap-osd/ceph.keyring:/var/lib/ceph/bootstrap-osd/ceph.keyring:z
+#    - /var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":/var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":z
+#    - /var/run/udev/:/var/run/udev/
+#    - /run/lvm/:/run/lvm/
+#  envs:
+#    OSD_ID: ${OSD_ID}
+#  args:
+#    - -f
+#    - -i=${OSD_ID}
+
 ###########
 # SYSTEMD #
 ###########

group_vars/rgws.yml.sample

@@ -78,15 +78,21 @@ dummy:
 # Resource limitation
 # For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
 # Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
-# These options can be passed using the 'ceph_rgw_docker_extra_env' variable.
 #ceph_rgw_docker_memory_limit: "4096m"
 #ceph_rgw_docker_cpu_limit: 8
 # ceph_rgw_docker_cpuset_cpus: "0,2,4,6,8,10,12,14,16"
 # ceph_rgw_docker_cpuset_mems: "0"
 
-#ceph_rgw_docker_extra_env:
 #ceph_config_keys: [] # DON'T TOUCH ME
 #rgw_config_keys: "/" # DON'T TOUCH ME
+# If you want to add parameters, you should retain the existing ones and include the new ones.
+#ceph_rgw_container_params:
+#  volumes:
+#    - /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:z
+#  args:
+#    - -f
+#    - -n=client.rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}
+#    - -k=/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}/keyring
 
 ###########
 # SYSTEMD #

roles/ceph-mds/defaults/main.yml

@@ -22,11 +22,15 @@ copy_admin_key: false
 ceph_mds_docker_memory_limit: "{{ ansible_facts['memtotal_mb'] }}m"
 ceph_mds_docker_cpu_limit: 4
 
-# we currently for MDS_NAME to hostname because of a bug in ceph-docker
-# fix here: https://github.com/ceph/ceph-docker/pull/770
-# this will go away soon.
-ceph_mds_docker_extra_env: -e MDS_NAME={{ ansible_facts['hostname'] }}
 ceph_config_keys: [] # DON'T TOUCH ME
+# If you want to add parameters, you should retain the existing ones and include the new ones.
+ceph_mds_container_params:
+  volumes:
+    - /var/lib/ceph/bootstrap-mds:/var/lib/ceph/bootstrap-mds:z
+    - /var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:/var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:z
+  args:
+    - -f
+    - -i={{ ansible_facts['hostname'] }}
 
 
 ###########

roles/ceph-mds/templates/ceph-mds.service.j2

@@ -28,22 +28,16 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   --security-opt label=disable \
   --memory={{ ceph_mds_docker_memory_limit }} \
   --cpus={{ cpu_limit }} \
-  -v /var/lib/ceph/bootstrap-mds:/var/lib/ceph/bootstrap-mds:z \
-  -v /var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:/var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:z \
-{% for v in ceph_common_container_params['volumes'] %}
+{% for v in ceph_common_container_params['volumes'] + ceph_mds_container_params['volumes'] | default([]) %}
   -v {{ v }} \
 {% endfor %}
-{% for k, v in ceph_common_container_params['envs'].items() %}
+{% for k, v in (ceph_common_container_params['envs'] | combine(ceph_mds_container_params['envs'] | default({}))).items() %}
   -e {{ k }}={{ v }} \
 {% endfor %}
-  {{ ceph_mds_docker_extra_env }} \
   --name=ceph-mds-{{ ansible_facts['hostname'] }} \
   --entrypoint=/usr/bin/ceph-mds \
   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-{% for arg in ceph_common_container_params['args'] %}
-  {{ arg }} \
-{% endfor %}
-  -f -i {{ ansible_facts['hostname'] }}
+  {{ (ceph_common_container_params['args'] + ceph_mds_container_params['args'] | default([])) | join(' ') }}
 {% if container_binary == 'podman' %}
 ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"
 {% else %}

roles/ceph-mgr/defaults/main.yml

@@ -32,12 +32,18 @@ ceph_mgr_packages:
 # Resource limitation
 # For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
 # Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
-# These options can be passed using the 'ceph_mgr_docker_extra_env' variable.
 ceph_mgr_docker_memory_limit: "{{ ansible_facts['memtotal_mb'] }}m"
 ceph_mgr_docker_cpu_limit: 1
 
-ceph_mgr_docker_extra_env:
 ceph_config_keys: [] # DON'T TOUCH ME
+# If you want to add parameters, you should retain the existing ones and include the new ones.
+ceph_mgr_container_params:
+  volumes:
+    - /var/lib/ceph/mgr:/var/lib/ceph/mgr:z,rshared
+    - /var/lib/ceph/bootstrap-mgr:/var/lib/ceph/bootstrap-mgr:z
+  args:
+    - -f
+    - -i={{ ansible_facts['hostname'] }}
 
 
 ###########

roles/ceph-mgr/templates/ceph-mgr.service.j2

@@ -27,23 +27,16 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   --security-opt label=disable \
   --memory={{ ceph_mgr_docker_memory_limit }} \
   --cpus={{ ceph_mgr_docker_cpu_limit }} \
-{% for v in ceph_common_container_params['volumes'] %}
+{% for v in ceph_common_container_params['volumes'] + ceph_mgr_container_params['volumes'] | default([]) %}
   -v {{ v }} \
 {% endfor %}
-  -v /var/lib/ceph/mgr:/var/lib/ceph/mgr:z,rshared \
-  -v /var/lib/ceph/bootstrap-mgr:/var/lib/ceph/bootstrap-mgr:z \
-{% for k, v in ceph_common_container_params['envs'].items() %}
+{% for k, v in (ceph_common_container_params['envs'] | combine(ceph_mgr_container_params['envs'] | default({}))).items() %}
   -e {{ k }}={{ v }} \
 {% endfor %}
-  {{ ceph_mgr_docker_extra_env }} \
   --name=ceph-mgr-{{ ansible_facts['hostname'] }} \
   --entrypoint=/usr/bin/ceph-mgr \
   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-{% for arg in ceph_common_container_params['args'] %}
-  {{ arg }} \
-{% endfor %}
-  -f \
-  -i {{ ansible_facts['hostname'] }}
+  {{ (ceph_common_container_params['args'] + ceph_mgr_container_params['args'] | default([])) | join(' ') }}
 {% if container_binary == 'podman' %}
 ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"
 {% else %}

roles/ceph-mon/defaults/main.yml

@@ -36,18 +36,26 @@ client_admin_ceph_authtool_cap:
 # Resource limitation
 # For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
 # Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
-# These options can be passed using the 'ceph_mon_docker_extra_env' variable.
 ceph_mon_docker_memory_limit: "{{ ansible_facts['memtotal_mb'] }}m"
 ceph_mon_docker_cpu_limit: 1
 ceph_mon_container_listen_port: 3300
 
-# Use this variable to add extra env configuration to run your mon container.
-# If you want to set a custom admin keyring you can set this variable like following:
-# ceph_mon_docker_extra_env: -e ADMIN_SECRET={{ admin_secret }}
-ceph_mon_docker_extra_env:
+# Use this variable to modify the configuration to run your mon container.
 mon_docker_privileged: false
 mon_docker_net_host: true
 ceph_config_keys: [] # DON'T TOUCH ME
+# If you want to add parameters, you should retain the existing ones and include the new ones.
+ceph_mon_container_params:
+  volumes:
+    - /var/lib/ceph/mon:/var/lib/ceph/mon:z,rshared
+  args:
+    - -f
+    - --default-mon-cluster-log-to-file=false
+    - --default-mon-cluster-log-to-stderr=true
+    - -i={{ monitor_name }}
+    - --mon-data=/var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }}
+    - --public-addr={{ _current_monitor_address }}
+    - --mon-initial-members={{ groups[mon_group_name] | join(',') }}
 
 
 ###########

roles/ceph-mon/templates/ceph-mon.service.j2

@@ -28,10 +28,9 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \
   --memory={{ ceph_mon_docker_memory_limit }} \
   --cpus={{ ceph_mon_docker_cpu_limit }} \
   --security-opt label=disable \
-{% for v in ceph_common_container_params['volumes'] %}
+{% for v in ceph_common_container_params['volumes'] + ceph_mon_container_params['volumes'] | default([]) %}
   -v {{ v }} \
 {% endfor %}
-  -v /var/lib/ceph/mon:/var/lib/ceph/mon:z,rshared \
 {% if ansible_facts['os_family'] == 'RedHat' -%}
   -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \
 {% endif -%}
@@ -41,19 +40,12 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \
 {% if mon_docker_net_host | bool -%}
   --net=host \
 {% endif -%}
-{% for k, v in ceph_common_container_params['envs'].items() %}
+{% for k, v in (ceph_common_container_params['envs'] | combine(ceph_mon_container_params['envs'] | default({}))).items() %}
   -e {{ k }}={{ v }} \
 {% endfor %}
-  {{ ceph_mon_docker_extra_env }} \
   --entrypoint=/usr/bin/ceph-mon \
-  {{ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-{% for arg in ceph_common_container_params['args'] %}
-  {{ arg }} \
-{% endfor %}
-  -f \
-  --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true \
-  -i {{ monitor_name }} --mon-data /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }} \
-  --public-addr {{ _current_monitor_address }} --mon-initial-members {{ groups[mon_group_name] | join(',') }}
+  {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
+  {{ (ceph_common_container_params['args'] + ceph_mon_container_params['args'] | default([])) | join(' ') }}
 {% if container_binary == 'podman' %}
 ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"
 {% else %}

roles/ceph-osd/defaults/main.yml

@@ -160,7 +160,6 @@ ceph_config_keys: [] # DON'T TOUCH ME
 # Resource limitation
 # For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
 # Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
-# These options can be passed using the 'ceph_osd_docker_extra_env' variable.
 ceph_osd_docker_memory_limit: "{{ ansible_facts['memtotal_mb'] }}m"
 ceph_osd_docker_cpu_limit: 4
 
@@ -181,9 +180,22 @@ ceph_osd_docker_prepare_env: -e OSD_JOURNAL_SIZE={{ journal_size }}
 
 # ACTIVATE DEVICE
 #
-ceph_osd_docker_extra_env:
 ceph_osd_numactl_opts: ""
 
+# If you want to add parameters, you should retain the existing ones and include the new ones.
+ceph_osd_container_params:
+  volumes:
+    - /dev:/dev
+    - /var/lib/ceph/bootstrap-osd/ceph.keyring:/var/lib/ceph/bootstrap-osd/ceph.keyring:z
+    - /var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":/var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":z
+    - /var/run/udev/:/var/run/udev/
+    - /run/lvm/:/run/lvm/
+  envs:
+    OSD_ID: ${OSD_ID}
+  args:
+    - -f
+    - -i=${OSD_ID}
+
 ###########
 # SYSTEMD #
 ###########

roles/ceph-osd/templates/systemd-run.j2

@@ -43,26 +43,17 @@ numactl \
 {% if ceph_osd_docker_cpuset_mems is defined -%}
 --cpuset-mems='{{ ceph_osd_docker_cpuset_mems }}' \
 {% endif -%}
-{% for v in ceph_common_container_params['volumes'] %}
+{% for v in ceph_common_container_params['volumes'] + ceph_osd_container_params['volumes'] | default([]) %}
   -v {{ v }} \
 {% endfor %}
--v /dev:/dev \
--v /var/lib/ceph/bootstrap-osd/ceph.keyring:/var/lib/ceph/bootstrap-osd/ceph.keyring:z \
--v /var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":/var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":z \
--v /var/run/udev/:/var/run/udev/ \
 {% if ansible_facts['distribution'] == 'Ubuntu' -%}
---security-opt apparmor:unconfined \
+--security-opt apparmor=unconfined \
 {% endif -%}
-{% for k, v in ceph_common_container_params['envs'].items() %}
+{% for k, v in (ceph_common_container_params['envs'] | combine(ceph_osd_container_params['envs'] | default({}))).items() %}
 -e {{ k }}={{ v }} \
 {% endfor %}
--v /run/lvm/:/run/lvm/ \
--e OSD_ID=${OSD_ID} \
 --name=ceph-osd-${OSD_ID} \
 --entrypoint=/usr/bin/ceph-osd \
 {{ ceph_osd_docker_extra_env }} \
 {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-{% for arg in ceph_common_container_params['args'] %}
-  {{ arg }} \
-{% endfor %}
--f -i ${OSD_ID}
+{{ (ceph_common_container_params['args'] + ceph_osd_container_params['args'] | default([])) | join(' ') }}

roles/ceph-rgw/defaults/main.yml

@@ -70,15 +70,21 @@ copy_admin_key: false
 # Resource limitation
 # For the whole list of limits you can apply see: docs.docker.com/engine/admin/resource_constraints
 # Default values are based from: https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/2/html/red_hat_ceph_storage_hardware_guide/minimum_recommendations
-# These options can be passed using the 'ceph_rgw_docker_extra_env' variable.
 ceph_rgw_docker_memory_limit: "4096m"
 ceph_rgw_docker_cpu_limit: 8
 # ceph_rgw_docker_cpuset_cpus: "0,2,4,6,8,10,12,14,16"
 # ceph_rgw_docker_cpuset_mems: "0"
 
-ceph_rgw_docker_extra_env:
 ceph_config_keys: [] # DON'T TOUCH ME
 rgw_config_keys: "/" # DON'T TOUCH ME
+# If you want to add parameters, you should retain the existing ones and include the new ones.
+ceph_rgw_container_params:
+  volumes:
+    - /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:z
+  args:
+    - -f
+    - -n=client.rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}
+    - -k=/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}/keyring
 
 ###########
 # SYSTEMD #