cephadm: support no huge pages for nvmeof spdk

depends on: ceph/ceph-nvmeof#898

Signed-off-by: Alexander Indenbaum <[email protected]>

src/cephadm/cephadmlib/daemons/nvmeof.py

@@ -73,13 +73,18 @@ def _get_container_mounts(
             os.path.join(data_dir, 'ceph-nvmeof.conf')
         ] = '/src/ceph-nvmeof.conf:z'
         mounts[os.path.join(data_dir, 'configfs')] = '/sys/kernel/config'
-        mounts['/dev/hugepages'] = '/dev/hugepages'
-        mounts['/dev/vfio/vfio'] = '/dev/vfio/vfio'
         mounts[log_dir] = '/var/log/ceph:z'
         if mtls_dir:
             mounts[mtls_dir] = '/src/mtls:z'
         return mounts
 
+    def _get_huge_pages_mounts(self, files: Dict[str, str]) -> -> Dict[str, str]:
+        mounts = dict()
+        if 'spdk_mem_size' not in files:
+            mounts['/dev/hugepages'] = '/dev/hugepages'
+            mounts['/dev/vfio/vfio'] = '/dev/vfio/vfio'
+        return mounts
+
     def _get_tls_cert_key_mounts(
         self, data_dir: str, files: Dict[str, str]
     ) -> Dict[str, str]:
@@ -111,6 +116,7 @@ def customize_container_mounts(
             )
         else:
             mounts.update(self._get_container_mounts(data_dir, log_dir))
+        mounts.update(self._get_huge_pages_mounts(self.files))
         mounts.update(self._get_tls_cert_key_mounts(data_dir, self.files))
 
     def customize_container_binds(
@@ -198,11 +204,13 @@ def configfs_mount_umount(data_dir, mount=True):
             )
         return cmd.split()
 
-    @staticmethod
-    def get_sysctl_settings() -> List[str]:
-        return [
-            'vm.nr_hugepages = 4096',
-        ]
+    def get_sysctl_settings(self) -> List[str]:
+        if 'spdk_mem_size' not in self.files:
+            return [
+                'vm.nr_hugepages = 4096',
+            ]
+        else:
+            return []
 
     def container(self, ctx: CephadmContext) -> CephContainer:
         ctr = daemon_to_container(ctx, self)
@@ -222,4 +230,6 @@ def customize_container_args(
         args.append(ctx.container_engine.unlimited_pids_option)
         args.extend(['--ulimit', 'memlock=-1:-1'])
         args.extend(['--ulimit', 'nofile=10240'])
-        args.extend(['--cap-add=SYS_ADMIN', '--cap-add=CAP_SYS_NICE'])
+        args.extend(['--cap-add=CAP_SYS_NICE'])
+        if 'spdk_mem_size' not in self.files:
+            args.extend(['--cap-add=SYS_ADMIN'])

src/pybind/mgr/cephadm/services/nvmeof.py

@@ -66,6 +66,10 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
         daemon_spec.keyring = keyring
         daemon_spec.extra_files = {'ceph-nvmeof.conf': gw_conf}
 
+        # Indicate to the daemon whether to utilize huge pages
+        if spec.spdk_mem_size:
+            daemon_spec.extra_files['spdk_mem_size'] = str(spec.spdk_mem_size)
+
         if spec.enable_auth:
             if (
                 not spec.client_cert

src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2

@@ -65,6 +65,9 @@ protocol_log_level = {{ spec.spdk_protocol_log_level }}
 log_file_dir = {{ spec.spdk_log_file_dir }}
 {% endif %}
 conn_retries = {{ spec.conn_retries }}
+{% if spec.spdk_mem_size %}
+mem_size = {{ spec.spdk_mem_size }}
+{% endif %}
 transports = {{ spec.transports }}
 {% if transport_tcp_options %}
 transport_tcp_options = {{ transport_tcp_options }}

src/python-common/ceph/deployment/service_spec.py

@@ -1347,7 +1347,8 @@ def __init__(self,
                  client_key: Optional[str] = None,
                  client_cert: Optional[str] = None,
                  root_ca_cert: Optional[str] = None,
-                 spdk_path: Optional[str] = None,
+                 spdk_path: Optional[str] = None, # unused and duplicate of tgt_path below, consider removing
+                 spdk_mem_size: Optional[int] = None,
                  tgt_path: Optional[str] = None,
                  spdk_timeout: Optional[float] = 60.0,
                  spdk_log_level: Optional[str] = '',
@@ -1443,8 +1444,10 @@ def __init__(self,
         self.client_cert = client_cert
         #: ``root_ca_cert`` CA cert for server/client certs
         self.root_ca_cert = root_ca_cert
-        #: ``spdk_path`` path to SPDK
+        #: ``spdk_path`` path is unused and duplicate of tgt_path below, consider removing
         self.spdk_path = spdk_path or '/usr/local/bin/nvmf_tgt'
+        #: ``spdk_mem_size`` memory size in MB for DPDK
+        self.spdk_mem_size = spdk_mem_size
         #: ``tgt_path`` nvmeof target path
         self.tgt_path = tgt_path or '/usr/local/bin/nvmf_tgt'
         #: ``spdk_timeout`` SPDK connectivity timeout