container: cleanup container systemd units

group_vars/all.yml.sample

@@ -534,7 +534,24 @@ dummy:
 #containerized_deployment: false
 #container_binary:
 #timeout_command: "{{ 'timeout --foreground -s KILL ' ~ docker_pull_timeout if (docker_pull_timeout != '0') and (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) else '' }}"
-
+#ceph_common_container_params:
+#  envs:
+#    NODE_NAME: "{{ ansible_facts['hostname'] }}"
+#    CEPH_USE_RANDOM_NONCE: "1"
+#    CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
+#    TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}"
+#  args:
+#    - --setuser=ceph
+#    - --setgroup=ceph
+#    - --default-log-to-file=false
+#    - --default-log-to-stderr=true
+#    - --default-log-stderr-prefix="debug "
+#  volumes:
+#    - /var/lib/ceph/crash:/var/lib/ceph/crash:z
+#    - /var/run/ceph:/var/run/ceph:z
+#    - /var/log/ceph:/var/log/ceph:z
+#    - /etc/ceph:/etc/ceph:z
+#    - /etc/localtime:/etc/localtime:ro
 
 # this is only here for usage with the rolling_update.yml playbook
 # do not ever change this here

roles/ceph-config/tasks/create_ceph_initial_dirs.yml

@@ -13,6 +13,7 @@
     - /var/lib/ceph/osd
     - /var/lib/ceph/mds
     - /var/lib/ceph/tmp
+    - /var/lib/ceph/crash
     - /var/lib/ceph/radosgw
     - /var/lib/ceph/bootstrap-rgw
     - /var/lib/ceph/bootstrap-mgr

roles/ceph-crash/templates/ceph-crash.service.j2

@@ -24,9 +24,12 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-crash-%i \
 {% if cluster != 'ceph' %}
 -e CEPH_ARGS="--cluster {{ cluster }}" \
 {% endif %}
--v /var/lib/ceph/crash:/var/lib/ceph/crash:z \
--v /etc/localtime:/etc/localtime:ro \
--v /etc/ceph:/etc/ceph:z \
+{% for v in ceph_common_container_params['volumes'] %}
+  -v {{ v }} \
+{% endfor %}
+{% for k, v in ceph_common_container_params['envs'].items() %}
+  -e {{ k }}={{ v }} \
+{% endfor %}
 --entrypoint=/usr/bin/ceph-crash {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
 {% if container_binary == 'podman' %}
 ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"

roles/ceph-defaults/defaults/main.yml

@@ -526,7 +526,22 @@ ceph_client_docker_registry: "{{ ceph_docker_registry }}"
 containerized_deployment: false
 container_binary:
 timeout_command: "{{ 'timeout --foreground -s KILL ' ~ docker_pull_timeout if (docker_pull_timeout != '0') and (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) else '' }}"
-
+ceph_common_container_params:
+  envs:
+    NODE_NAME: "{{ ansible_facts['hostname'] }}"
+    CEPH_USE_RANDOM_NONCE: "1"
+    CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
+    TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}"
+  args:
+    - --default-log-to-file=false
+    - --default-log-to-stderr=true
+    - --default-log-stderr-prefix="debug "
+  volumes:
+    - /var/lib/ceph/crash:/var/lib/ceph/crash:z
+    - /var/run/ceph:/var/run/ceph:z
+    - /var/log/ceph:/var/log/ceph:z
+    - /etc/ceph:/etc/ceph:z
+    - /etc/localtime:/etc/localtime:ro
 
 # this is only here for usage with the rolling_update.yml playbook
 # do not ever change this here

roles/ceph-facts/tasks/facts.yml

@@ -197,7 +197,7 @@
   block:
     - &read-osd-pool-default-crush-rule
       name: Read osd pool default crush rule
-      ansible.builtin.command: grep 'osd pool default crush rule' /etc/ceph/{{ cluster }}.conf
+      ansible.builtin.command: grep 'osd_pool_default_crush_rule' /etc/ceph/{{ cluster }}.conf
       register: crush_rule_variable
       changed_when: false
       check_mode: false

roles/ceph-mds/templates/ceph-mds.service.j2

@@ -30,16 +30,19 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   --cpus={{ cpu_limit }} \
   -v /var/lib/ceph/bootstrap-mds:/var/lib/ceph/bootstrap-mds:z \
   -v /var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:/var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:z \
-  -v /etc/ceph:/etc/ceph:z \
-  -v /var/run/ceph:/var/run/ceph:z \
-  -v /etc/localtime:/etc/localtime:ro \
-  -v /var/log/ceph:/var/log/ceph:z \
-  -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-  -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
+{% for v in ceph_common_container_params['volumes'] %}
+  -v {{ v }} \
+{% endfor %}
+{% for k, v in ceph_common_container_params['envs'].items() %}
+  -e {{ k }}={{ v }} \
+{% endfor %}
   {{ ceph_mds_docker_extra_env }} \
   --name=ceph-mds-{{ ansible_facts['hostname'] }} \
   --entrypoint=/usr/bin/ceph-mds \
   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
+{% for arg in ceph_common_container_params['args'] %}
+  {{ arg }} \
+{% endfor %}
   -f -i {{ ansible_facts['hostname'] }}
 {% if container_binary == 'podman' %}
 ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"

roles/ceph-mgr/templates/ceph-mgr.service.j2

@@ -27,21 +27,22 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   --security-opt label=disable \
   --memory={{ ceph_mgr_docker_memory_limit }} \
   --cpus={{ ceph_mgr_docker_cpu_limit }} \
+{% for v in ceph_common_container_params['volumes'] %}
+  -v {{ v }} \
+{% endfor %}
   -v /var/lib/ceph/mgr:/var/lib/ceph/mgr:z,rshared \
   -v /var/lib/ceph/bootstrap-mgr:/var/lib/ceph/bootstrap-mgr:z \
-  -v /etc/ceph:/etc/ceph:z \
-  -v /var/run/ceph:/var/run/ceph:z \
-  -v /etc/localtime:/etc/localtime:ro \
-  -v /var/log/ceph:/var/log/ceph:z \
-  -e CLUSTER={{ cluster }} \
-  -e CEPH_DAEMON=MGR \
-  -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-  -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
+{% for k, v in ceph_common_container_params['envs'].items() %}
+  -e {{ k }}={{ v }} \
+{% endfor %}
   {{ ceph_mgr_docker_extra_env }} \
   --name=ceph-mgr-{{ ansible_facts['hostname'] }} \
   --entrypoint=/usr/bin/ceph-mgr \
   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-  -f --default-log-to-file=false --default-log-to-stderr=true \
+{% for arg in ceph_common_container_params['args'] %}
+  {{ arg }} \
+{% endfor %}
+  -f \
   -i {{ ansible_facts['hostname'] }}
 {% if container_binary == 'podman' %}
 ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"

roles/ceph-mon/templates/ceph-mon.service.j2

@@ -28,11 +28,10 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \
   --memory={{ ceph_mon_docker_memory_limit }} \
   --cpus={{ ceph_mon_docker_cpu_limit }} \
   --security-opt label=disable \
+{% for v in ceph_common_container_params['volumes'] %}
+  -v {{ v }} \
+{% endfor %}
   -v /var/lib/ceph/mon:/var/lib/ceph/mon:z,rshared \
-  -v /etc/ceph:/etc/ceph:z \
-  -v /var/run/ceph:/var/run/ceph:z \
-  -v /etc/localtime:/etc/localtime:ro \
-  -v /var/log/ceph:/var/log/ceph:z \
 {% if ansible_facts['os_family'] == 'RedHat' -%}
   -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \
 {% endif -%}
@@ -42,19 +41,19 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \
 {% if mon_docker_net_host | bool -%}
   --net=host \
 {% endif -%}
-  -e IP_VERSION={{ ip_version[-1:] }} \
-  -e MON_IP={{ _current_monitor_address }} \
-  -e CLUSTER={{ cluster }} \
-  -e FSID={{ fsid }} \
-  -e MON_PORT={{ ceph_mon_container_listen_port }} \
-  -e CEPH_PUBLIC_NETWORK={{ public_network | regex_replace(' ', '') }} \
-  -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-  -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
+{% for k, v in ceph_common_container_params['envs'].items() %}
+  -e {{ k }}={{ v }} \
+{% endfor %}
   {{ ceph_mon_docker_extra_env }} \
   --entrypoint=/usr/bin/ceph-mon \
   {{ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
-  -f --default-log-to-file=false --default-log-to-stderr=true \
-  -i {{ monitor_name }} --mon-data /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }} --public-addr {{ _current_monitor_address }} --mon-initial-members {{ groups[mon_group_name][0] }}
+{% for arg in ceph_common_container_params['args'] %}
+  {{ arg }} \
+{% endfor %}
+  -f \
+  --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true \
+  -i {{ monitor_name }} --mon-data /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }} \
+  --public-addr {{ _current_monitor_address }} --mon-initial-members {{ groups[mon_group_name] | join(',') }}
 {% if container_binary == 'podman' %}
 ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"
 {% else %}

roles/ceph-osd/tasks/crush_rules.yml

@@ -63,7 +63,7 @@
   community.general.ini_file:
     dest: "/etc/ceph/{{ cluster }}.conf"
     section: "global"
-    option: "osd pool default crush rule"
+    option: "osd_pool_default_crush_rule"
     value: "{{ info_ceph_default_crush_rule_yaml.rule_id }}"
     mode: "0644"
   delegate_to: "{{ item }}"

roles/ceph-osd/templates/systemd-run.j2

@@ -43,23 +43,26 @@ numactl \
 {% if ceph_osd_docker_cpuset_mems is defined -%}
 --cpuset-mems='{{ ceph_osd_docker_cpuset_mems }}' \
 {% endif -%}
+{% for v in ceph_common_container_params['volumes'] %}
+  -v {{ v }} \
+{% endfor %}
 -v /dev:/dev \
--v /etc/localtime:/etc/localtime:ro \
 -v /var/lib/ceph/bootstrap-osd/ceph.keyring:/var/lib/ceph/bootstrap-osd/ceph.keyring:z \
 -v /var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":/var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":z \
--v /etc/ceph:/etc/ceph:z \
--v /var/run/ceph:/var/run/ceph:z \
 -v /var/run/udev/:/var/run/udev/ \
--v /var/log/ceph:/var/log/ceph:z \
 {% if ansible_facts['distribution'] == 'Ubuntu' -%}
 --security-opt apparmor:unconfined \
 {% endif -%}
--e CLUSTER={{ cluster }} \
--e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \
+{% for k, v in ceph_common_container_params['envs'].items() %}
+-e {{ k }}={{ v }} \
+{% endfor %}
 -v /run/lvm/:/run/lvm/ \
 -e OSD_ID=${OSD_ID} \
 --name=ceph-osd-${OSD_ID} \
 --entrypoint=/usr/bin/ceph-osd \
 {{ ceph_osd_docker_extra_env }} \
 {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
--f -i ${OSD_ID}
+{% for arg in ceph_common_container_params['args'] %}
+  {{ arg }} \
+{% endfor %}
+-f -i ${OSD_ID}

roles/ceph-rgw/templates/ceph-radosgw.service.j2

@@ -34,11 +34,13 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   {% if ceph_rgw_docker_cpuset_mems is defined -%}
   --cpuset-mems="{{ ceph_rgw_docker_cpuset_mems }}" \
   {% endif -%}
+{% for v in ceph_common_container_params['volumes'] %}
+  -v {{ v }} \
+{% endfor %}
   -v /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:z \
-  -v /etc/ceph:/etc/ceph \
-  -v /var/run/ceph:/var/run/ceph \
-  -v /etc/localtime:/etc/localtime \
-  -v /var/log/ceph:/var/log/ceph \
+{% for k, v in ceph_common_container_params['envs'].items() %}
+  -e {{ k }}={{ v }} \
+{% endfor %}
   {% if ansible_facts['os_family'] == 'RedHat' -%}
   -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \
   {% endif -%}
@@ -50,6 +52,9 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   --entrypoint=/usr/bin/radosgw \
   {{ ceph_rgw_docker_extra_env }} \
   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
+{% for arg in ceph_common_container_params['args'] %}
+  {{ arg }} \
+{% endfor %}
   -f -n client.rgw.{{ ansible_facts['hostname'] }}.${INST_NAME} -k /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}/keyring
 {% if container_binary == 'podman' %}
 ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"

tox-cephadm.ini

@@ -32,7 +32,7 @@ commands=
   bash {toxinidir}/tests/scripts/vagrant_up.sh --no-provision {posargs:--provider=virtualbox}
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
 
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/infrastructure-playbooks/cephadm.yml --extra-vars "\
+  ansible-playbook -vv --diff -i {changedir}/hosts {toxinidir}/infrastructure-playbooks/cephadm.yml --extra-vars "\
       delegate_facts_host={env:DELEGATE_FACTS_HOST:True} \
       ceph_docker_registry_auth=True \
       ceph_docker_registry_username={env:DOCKER_HUB_USERNAME} \

tox-docker2podman.ini

@@ -33,18 +33,18 @@ commands=
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
 
   # configure lvm
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/lvm_setup.yml
+  ansible-playbook -vv --diff -i {changedir}/hosts {toxinidir}/tests/functional/lvm_setup.yml
 
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/setup.yml
+  ansible-playbook -vv --diff -i {changedir}/hosts {toxinidir}/tests/functional/setup.yml
 
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/site-container.yml.sample --extra-vars "\
+  ansible-playbook -vv --diff -i {changedir}/hosts {toxinidir}/site-container.yml.sample --extra-vars "\
       delegate_facts_host={env:DELEGATE_FACTS_HOST:True} \
       ceph_docker_registry_auth=True \
       ceph_docker_registry_username={env:DOCKER_HUB_USERNAME} \
       ceph_docker_registry_password={env:DOCKER_HUB_PASSWORD} \
   "
 
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/infrastructure-playbooks/docker-to-podman.yml --extra-vars "\
+  ansible-playbook -vv --diff -i {changedir}/hosts {toxinidir}/infrastructure-playbooks/docker-to-podman.yml --extra-vars "\
       delegate_facts_host={env:DELEGATE_FACTS_HOST:True} \
   "
 

tox-external_clients.ini

@@ -35,13 +35,13 @@ commands=
   bash {toxinidir}/tests/scripts/vagrant_up.sh --no-provision {posargs:--provider=virtualbox}
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
 
-  ansible-playbook -vv -i {changedir}/inventory {toxinidir}/tests/functional/setup.yml
+  ansible-playbook -vv --diff -i {changedir}/inventory {toxinidir}/tests/functional/setup.yml
 
   # configure lvm
-  ansible-playbook -vv -i {changedir}/inventory/hosts {toxinidir}/tests/functional/lvm_setup.yml
+  ansible-playbook -vv --diff -i {changedir}/inventory/hosts {toxinidir}/tests/functional/lvm_setup.yml
 
-  non_container: ansible-playbook -vv -i "localhost," -c local {toxinidir}/tests/functional/dev_setup.yml --extra-vars "dev_setup=True change_dir={changedir} ceph_dev_branch=main ceph_dev_sha1=latest" --tags "vagrant_setup"
-  ansible-playbook -vv -i {changedir}/inventory/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} --limit 'all:!clients' --extra-vars "\
+  non_container: ansible-playbook -vv --diff -i "localhost," -c local {toxinidir}/tests/functional/dev_setup.yml --extra-vars "dev_setup=True change_dir={changedir} ceph_dev_branch=main ceph_dev_sha1=latest" --tags "vagrant_setup"
+  ansible-playbook -vv --diff -i {changedir}/inventory/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} --limit 'all:!clients' --extra-vars "\
       yes_i_know=true \
       delegate_facts_host={env:DELEGATE_FACTS_HOST:True} \
       ceph_dev_branch=main \
@@ -51,9 +51,9 @@ commands=
       ceph_docker_registry_password={env:DOCKER_HUB_PASSWORD} \
   "
 
-  ansible-playbook -vv -i {changedir}/inventory {toxinidir}/tests/functional/external_clients_admin_key.yml
+  ansible-playbook -vv --diff -i {changedir}/inventory {toxinidir}/tests/functional/external_clients_admin_key.yml
 
-  ansible-playbook -vv -i {changedir}/inventory/external_clients-hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars "\
+  ansible-playbook -vv --diff -i {changedir}/inventory/external_clients-hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars "\
       yes_i_know=true \
       ireallymeanit=yes \
       fsid=40358a87-ab6e-4bdc-83db-1d909147861c \
@@ -68,7 +68,7 @@ commands=
 
   py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/inventory/external_clients-hosts --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests/test_install.py::TestCephConf
 
-  ansible-playbook -vv -i {changedir}/inventory/external_clients-hosts {toxinidir}/infrastructure-playbooks/rolling_update.yml --extra-vars "\
+  ansible-playbook -vv --diff -i {changedir}/inventory/external_clients-hosts {toxinidir}/infrastructure-playbooks/rolling_update.yml --extra-vars "\
       ireallymeanit=yes \
       fsid=40358a87-ab6e-4bdc-83db-1d909147861c \
       external_cluster_mon_ips=192.168.31.10,192.168.31.11,192.168.31.12 \

tox-podman.ini

@@ -37,11 +37,11 @@ commands=
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
 
   # configure lvm
-  ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/tests/functional/lvm_setup.yml
+  ansible-playbook -vv --diff -i {changedir}/{env:INVENTORY} {toxinidir}/tests/functional/lvm_setup.yml
 
-  ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/tests/functional/setup.yml
+  ansible-playbook -vv --diff -i {changedir}/{env:INVENTORY} {toxinidir}/tests/functional/setup.yml
 
-  ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars "\
+  ansible-playbook -vv --diff -i {changedir}/{env:INVENTORY} {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars "\
       delegate_facts_host={env:DELEGATE_FACTS_HOST:True} \
       ceph_docker_registry_auth=True \
       ceph_docker_registry_username={env:DOCKER_HUB_USERNAME} \
@@ -52,7 +52,7 @@ commands=
   py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests
 
   # reboot all vms
-  ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/tests/functional/reboot.yml
+  ansible-playbook -vv --diff -i {changedir}/{env:INVENTORY} {toxinidir}/tests/functional/reboot.yml
 
   # retest to ensure cluster came back up correctly after rebooting
   py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests