From a9e34575638f5f0ddc45b35b353170a68e6fa37e Mon Sep 17 00:00:00 2001 From: Seena Fallah Date: Sun, 18 Feb 2024 03:41:41 +0100 Subject: [PATCH] container: cleanup container systemd units Signed-off-by: Seena Fallah --- group_vars/all.yml.sample | 19 +++++++++++- .../tasks/create_ceph_initial_dirs.yml | 1 + .../templates/ceph-crash.service.j2 | 9 ++++-- roles/ceph-defaults/defaults/main.yml | 19 +++++++++++- roles/ceph-mds/templates/ceph-mds.service.j2 | 15 ++++++---- roles/ceph-mgr/templates/ceph-mgr.service.j2 | 21 +++++++------- roles/ceph-mon/templates/ceph-mon.service.j2 | 29 +++++++++---------- roles/ceph-osd/templates/systemd-run.j2 | 17 ++++++----- .../templates/ceph-radosgw.service.j2 | 13 ++++++--- 9 files changed, 96 insertions(+), 47 deletions(-) diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample index 2fa9e8265f..a2651614a3 100644 --- a/group_vars/all.yml.sample +++ b/group_vars/all.yml.sample @@ -534,7 +534,24 @@ dummy: #containerized_deployment: false #container_binary: #timeout_command: "{{ 'timeout --foreground -s KILL ' ~ docker_pull_timeout if (docker_pull_timeout != '0') and (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) else '' }}" - +#ceph_common_container_params: +# envs: +# NODE_NAME: "{{ ansible_facts['hostname'] }}" +# CEPH_USE_RANDOM_NONCE: "1" +# CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}" +# TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}" +# args: +# - --setuser=ceph +# - --setgroup=ceph +# - --default-log-to-file=false +# - --default-log-to-stderr=true +# - --default-log-stderr-prefix="debug " +# volumes: +# - /var/lib/ceph/crash:/var/lib/ceph/crash:z +# - /var/run/ceph:/var/run/ceph:z +# - /var/log/ceph:/var/log/ceph:z +# - /etc/ceph:/etc/ceph:z +# - /etc/localtime:/etc/localtime:ro # this is only here for usage with the rolling_update.yml playbook # do not ever change this here diff --git a/roles/ceph-config/tasks/create_ceph_initial_dirs.yml b/roles/ceph-config/tasks/create_ceph_initial_dirs.yml index a131ac2152..64936e171a 100644 --- a/roles/ceph-config/tasks/create_ceph_initial_dirs.yml +++ b/roles/ceph-config/tasks/create_ceph_initial_dirs.yml @@ -13,6 +13,7 @@ - /var/lib/ceph/osd - /var/lib/ceph/mds - /var/lib/ceph/tmp + - /var/lib/ceph/crash - /var/lib/ceph/radosgw - /var/lib/ceph/bootstrap-rgw - /var/lib/ceph/bootstrap-mgr diff --git a/roles/ceph-crash/templates/ceph-crash.service.j2 b/roles/ceph-crash/templates/ceph-crash.service.j2 index 1424eda836..bba5fe7aa1 100644 --- a/roles/ceph-crash/templates/ceph-crash.service.j2 +++ b/roles/ceph-crash/templates/ceph-crash.service.j2 @@ -24,9 +24,12 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-crash-%i \ {% if cluster != 'ceph' %} -e CEPH_ARGS="--cluster {{ cluster }}" \ {% endif %} --v /var/lib/ceph/crash:/var/lib/ceph/crash:z \ --v /etc/localtime:/etc/localtime:ro \ --v /etc/ceph:/etc/ceph:z \ +{% for v in ceph_common_container_params['volumes'] %} + -v {{ v }} \ +{% endfor %} +{% for k, v in ceph_common_container_params['envs'].items() %} + -e {{ k }}={{ v }} \ +{% endfor %} --entrypoint=/usr/bin/ceph-crash {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} {% if container_binary == 'podman' %} ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`" diff --git a/roles/ceph-defaults/defaults/main.yml b/roles/ceph-defaults/defaults/main.yml index 17483e469e..28c68306df 100644 --- a/roles/ceph-defaults/defaults/main.yml +++ b/roles/ceph-defaults/defaults/main.yml @@ -526,7 +526,24 @@ ceph_client_docker_registry: "{{ ceph_docker_registry }}" containerized_deployment: false container_binary: timeout_command: "{{ 'timeout --foreground -s KILL ' ~ docker_pull_timeout if (docker_pull_timeout != '0') and (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) else '' }}" - +ceph_common_container_params: + envs: + NODE_NAME: "{{ ansible_facts['hostname'] }}" + CEPH_USE_RANDOM_NONCE: "1" + CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}" + TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}" + args: + - --setuser=ceph + - --setgroup=ceph + - --default-log-to-file=false + - --default-log-to-stderr=true + - --default-log-stderr-prefix="debug " + volumes: + - /var/lib/ceph/crash:/var/lib/ceph/crash:z + - /var/run/ceph:/var/run/ceph:z + - /var/log/ceph:/var/log/ceph:z + - /etc/ceph:/etc/ceph:z + - /etc/localtime:/etc/localtime:ro # this is only here for usage with the rolling_update.yml playbook # do not ever change this here diff --git a/roles/ceph-mds/templates/ceph-mds.service.j2 b/roles/ceph-mds/templates/ceph-mds.service.j2 index f57b8098f8..9f56ec61b6 100644 --- a/roles/ceph-mds/templates/ceph-mds.service.j2 +++ b/roles/ceph-mds/templates/ceph-mds.service.j2 @@ -30,16 +30,19 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \ --cpus={{ cpu_limit }} \ -v /var/lib/ceph/bootstrap-mds:/var/lib/ceph/bootstrap-mds:z \ -v /var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:/var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:z \ - -v /etc/ceph:/etc/ceph:z \ - -v /var/run/ceph:/var/run/ceph:z \ - -v /etc/localtime:/etc/localtime:ro \ - -v /var/log/ceph:/var/log/ceph:z \ - -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ - -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ +{% for v in ceph_common_container_params['volumes'] %} + -v {{ v }} \ +{% endfor %} +{% for k, v in ceph_common_container_params['envs'].items() %} + -e {{ k }}={{ v }} \ +{% endfor %} {{ ceph_mds_docker_extra_env }} \ --name=ceph-mds-{{ ansible_facts['hostname'] }} \ --entrypoint=/usr/bin/ceph-mds \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ +{% for arg in ceph_common_container_params['args'] %} + {{ arg }} \ +{% endfor %} -f -i {{ ansible_facts['hostname'] }} {% if container_binary == 'podman' %} ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`" diff --git a/roles/ceph-mgr/templates/ceph-mgr.service.j2 b/roles/ceph-mgr/templates/ceph-mgr.service.j2 index 419cdacd43..8f9efc92f9 100644 --- a/roles/ceph-mgr/templates/ceph-mgr.service.j2 +++ b/roles/ceph-mgr/templates/ceph-mgr.service.j2 @@ -27,21 +27,22 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \ --security-opt label=disable \ --memory={{ ceph_mgr_docker_memory_limit }} \ --cpus={{ ceph_mgr_docker_cpu_limit }} \ - -v /var/lib/ceph/mgr:/var/lib/ceph/mgr:z,rshared \ +{% for v in ceph_common_container_params['volumes'] %} + -v {{ v }} \ +{% endfor %} + -v /var/lib/ceph/mgr:/var/lib/ceph/mgr:z \ -v /var/lib/ceph/bootstrap-mgr:/var/lib/ceph/bootstrap-mgr:z \ - -v /etc/ceph:/etc/ceph:z \ - -v /var/run/ceph:/var/run/ceph:z \ - -v /etc/localtime:/etc/localtime:ro \ - -v /var/log/ceph:/var/log/ceph:z \ - -e CLUSTER={{ cluster }} \ - -e CEPH_DAEMON=MGR \ - -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ - -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ +{% for k, v in ceph_common_container_params['envs'].items() %} + -e {{ k }}={{ v }} \ +{% endfor %} {{ ceph_mgr_docker_extra_env }} \ --name=ceph-mgr-{{ ansible_facts['hostname'] }} \ --entrypoint=/usr/bin/ceph-mgr \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ - -f --default-log-to-file=false --default-log-to-stderr=true \ +{% for arg in ceph_common_container_params['args'] %} + {{ arg }} \ +{% endfor %} + -f \ -i {{ ansible_facts['hostname'] }} {% if container_binary == 'podman' %} ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`" diff --git a/roles/ceph-mon/templates/ceph-mon.service.j2 b/roles/ceph-mon/templates/ceph-mon.service.j2 index 1b7f850cf8..4e8e55cff7 100644 --- a/roles/ceph-mon/templates/ceph-mon.service.j2 +++ b/roles/ceph-mon/templates/ceph-mon.service.j2 @@ -28,11 +28,10 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \ --memory={{ ceph_mon_docker_memory_limit }} \ --cpus={{ ceph_mon_docker_cpu_limit }} \ --security-opt label=disable \ - -v /var/lib/ceph/mon:/var/lib/ceph/mon:z,rshared \ - -v /etc/ceph:/etc/ceph:z \ - -v /var/run/ceph:/var/run/ceph:z \ - -v /etc/localtime:/etc/localtime:ro \ - -v /var/log/ceph:/var/log/ceph:z \ +{% for v in ceph_common_container_params['volumes'] %} + -v {{ v }} \ +{% endfor %} + -v /var/lib/ceph/mon:/var/lib/ceph/mon:z \ {% if ansible_facts['os_family'] == 'RedHat' -%} -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \ {% endif -%} @@ -42,19 +41,19 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \ {% if mon_docker_net_host | bool -%} --net=host \ {% endif -%} - -e IP_VERSION={{ ip_version[-1:] }} \ - -e MON_IP={{ _current_monitor_address }} \ - -e CLUSTER={{ cluster }} \ - -e FSID={{ fsid }} \ - -e MON_PORT={{ ceph_mon_container_listen_port }} \ - -e CEPH_PUBLIC_NETWORK={{ public_network | regex_replace(' ', '') }} \ - -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ - -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ +{% for k, v in ceph_common_container_params['envs'].items() %} + -e {{ k }}={{ v }} \ +{% endfor %} {{ ceph_mon_docker_extra_env }} \ --entrypoint=/usr/bin/ceph-mon \ {{ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ - -f --default-log-to-file=false --default-log-to-stderr=true \ - -i {{ monitor_name }} --mon-data /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }} --public-addr {{ _current_monitor_address }} --mon-initial-members {{ groups[mon_group_name][0] }} +{% for arg in ceph_common_container_params['args'] %} + {{ arg }} \ +{% endfor %} + -f \ + --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true \ + -i {{ monitor_name }} --mon-data /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }} \ + --public-addr {{ _current_monitor_address }} --mon-initial-members {{ groups[mon_group_name] | join(',') }} {% if container_binary == 'podman' %} ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`" {% else %} diff --git a/roles/ceph-osd/templates/systemd-run.j2 b/roles/ceph-osd/templates/systemd-run.j2 index 9d5ce3feef..73ba1b556f 100644 --- a/roles/ceph-osd/templates/systemd-run.j2 +++ b/roles/ceph-osd/templates/systemd-run.j2 @@ -43,23 +43,26 @@ numactl \ {% if ceph_osd_docker_cpuset_mems is defined -%} --cpuset-mems='{{ ceph_osd_docker_cpuset_mems }}' \ {% endif -%} +{% for v in ceph_common_container_params['volumes'] %} + -v {{ v }} \ +{% endfor %} -v /dev:/dev \ --v /etc/localtime:/etc/localtime:ro \ -v /var/lib/ceph/bootstrap-osd/ceph.keyring:/var/lib/ceph/bootstrap-osd/ceph.keyring:z \ -v /var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":/var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":z \ --v /etc/ceph:/etc/ceph:z \ --v /var/run/ceph:/var/run/ceph:z \ -v /var/run/udev/:/var/run/udev/ \ --v /var/log/ceph:/var/log/ceph:z \ {% if ansible_facts['distribution'] == 'Ubuntu' -%} --security-opt apparmor:unconfined \ {% endif -%} --e CLUSTER={{ cluster }} \ --e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ +{% for k, v in ceph_common_container_params['envs'].items() %} +-e {{ k }}={{ v }} \ +{% endfor %} -v /run/lvm/:/run/lvm/ \ -e OSD_ID=${OSD_ID} \ --name=ceph-osd-${OSD_ID} \ --entrypoint=/usr/bin/ceph-osd \ {{ ceph_osd_docker_extra_env }} \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ --f -i ${OSD_ID} \ No newline at end of file +{% for arg in ceph_common_container_params['args'] %} + {{ arg }} \ +{% endfor %} +-f -i ${OSD_ID} diff --git a/roles/ceph-rgw/templates/ceph-radosgw.service.j2 b/roles/ceph-rgw/templates/ceph-radosgw.service.j2 index c618437e5a..add58124ba 100644 --- a/roles/ceph-rgw/templates/ceph-radosgw.service.j2 +++ b/roles/ceph-rgw/templates/ceph-radosgw.service.j2 @@ -34,11 +34,13 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \ {% if ceph_rgw_docker_cpuset_mems is defined -%} --cpuset-mems="{{ ceph_rgw_docker_cpuset_mems }}" \ {% endif -%} +{% for v in ceph_common_container_params['volumes'] %} + -v {{ v }} \ +{% endfor %} -v /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:z \ - -v /etc/ceph:/etc/ceph \ - -v /var/run/ceph:/var/run/ceph \ - -v /etc/localtime:/etc/localtime \ - -v /var/log/ceph:/var/log/ceph \ +{% for k, v in ceph_common_container_params['envs'].items() %} + -e {{ k }}={{ v }} \ +{% endfor %} {% if ansible_facts['os_family'] == 'RedHat' -%} -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \ {% endif -%} @@ -50,6 +52,9 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \ --entrypoint=/usr/bin/radosgw \ {{ ceph_rgw_docker_extra_env }} \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ +{% for arg in ceph_common_container_params['args'] %} + {{ arg }} \ +{% endfor %} -f -n client.rgw.{{ ansible_facts['hostname'] }}.${INST_NAME} -k /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}/keyring {% if container_binary == 'podman' %} ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"