btl/sm: remove emulated RDMA code

This removes the emulated RDMA code provided as a fallback so btl/sm could be used for osc/rdma. This functionality is now provided by ```btl_base_am_rdma_init ()``` so the dedicated version in btl/sm is no longer needed. Signed-off-by: Nathan Hjelm <[email protected]>
open-mpi · Mar 3, 2021 · 59de860 · 59de860
1 parent b644768
commit 59de860
Show file tree

Hide file tree

Showing 7 changed files with 5 additions and 245 deletions.
diff --git a/opal/mca/btl/sm/Makefile.am b/opal/mca/btl/sm/Makefile.am
@@ -42,7 +42,6 @@ libmca_btl_sm_la_sources = \
     btl_sm_xpmem.h \
     btl_sm_knem.c \
     btl_sm_knem.h \
-    btl_sm_sc_emu.c \
     btl_sm_atomic.c
 
 # Make the output library in this directory, and name it either

diff --git a/opal/mca/btl/sm/btl_sm.h b/opal/mca/btl/sm/btl_sm.h
@@ -103,7 +103,6 @@ enum {
       MCA_BTL_SM_CMA   = 1,
       MCA_BTL_SM_KNEM  = 2,
       MCA_BTL_SM_NONE  = 3,
-      MCA_BTL_SM_EMUL  = 4,
 };
 
 /**
@@ -246,11 +245,6 @@ int mca_btl_sm_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
                          int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
 #endif
 
-int mca_btl_sm_put_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                           uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                           mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                           int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-
 /**
  * Initiate an synchronous get.
  *
@@ -281,29 +275,6 @@ int mca_btl_sm_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
 
 ino_t mca_btl_sm_get_user_ns_id(void);
 
-int mca_btl_sm_get_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                           uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                           mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                           int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-
-int mca_btl_sm_emu_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                        uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
-                        mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
-                        mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-
-int mca_btl_sm_emu_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                         void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                         mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
-                         uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
-                         void *cbcontext, void *cbdata);
-
-int mca_btl_sm_emu_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
-                           void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                           mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
-                           int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
-
-void mca_btl_sm_sc_emu_init (void);
-
 /**
  * Allocate a segment.
  *

diff --git a/opal/mca/btl/sm/btl_sm_component.c b/opal/mca/btl/sm/btl_sm_component.c
@@ -77,7 +77,6 @@ static mca_base_var_enum_value_t single_copy_mechanisms[] = {
 #if OPAL_BTL_SM_HAVE_KNEM
                                                              {.value = MCA_BTL_SM_KNEM, .string = "knem"},
 #endif
-                                                             {.value = MCA_BTL_SM_EMUL, .string = "emulated"},
                                                              {.value = MCA_BTL_SM_NONE, .string = "none"},
                                                              {.value = 0, .string = NULL}
 };
@@ -254,28 +253,7 @@ static int mca_btl_sm_component_register (void)
     mca_btl_sm.super.btl_rdma_pipeline_send_length = mca_btl_sm.super.btl_eager_limit;
     mca_btl_sm.super.btl_rdma_pipeline_frag_size   = mca_btl_sm.super.btl_eager_limit;
 
-#if OPAL_HAVE_ATOMIC_MATH_64
-    mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA |
-        MCA_BTL_FLAGS_ATOMIC_OPS | MCA_BTL_FLAGS_ATOMIC_FOPS;
-
-    mca_btl_sm.super.btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_AND |
-        MCA_BTL_ATOMIC_SUPPORTS_OR | MCA_BTL_ATOMIC_SUPPORTS_XOR | MCA_BTL_ATOMIC_SUPPORTS_CSWAP |
-        MCA_BTL_ATOMIC_SUPPORTS_GLOB | MCA_BTL_ATOMIC_SUPPORTS_SWAP;
-#if OPAL_HAVE_ATOMIC_MATH_32
-    mca_btl_sm.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_32BIT;
-#endif /* OPAL_HAVE_ATOMIC_MATH_32 */
-
-#if OPAL_HAVE_ATOMIC_MIN_64
-    mca_btl_sm.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_MIN;
-#endif /* OPAL_HAVE_ATOMIC_MIN_64 */
-
-#if OPAL_HAVE_ATOMIC_MAX_64
-    mca_btl_sm.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_MAX;
-#endif /* OPAL_HAVE_ATOMIC_MAX_64 */
-
-#else
-    mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA;
-#endif /* OPAL_HAVE_ATOMIC_MATH_64 */
+    mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND;
 
     if (MCA_BTL_SM_NONE != mca_btl_sm_component.single_copy_mechanism) {
         /* True single copy mechanisms should provide better bandwidth */
@@ -284,12 +262,6 @@ static int mca_btl_sm_component_register (void)
         mca_btl_sm.super.btl_bandwidth = 10000; /* Mbs */
     }
 
-    mca_btl_sm.super.btl_get = mca_btl_sm_get_sc_emu;
-    mca_btl_sm.super.btl_put = mca_btl_sm_put_sc_emu;
-    mca_btl_sm.super.btl_atomic_op = mca_btl_sm_emu_aop;
-    mca_btl_sm.super.btl_atomic_fop = mca_btl_sm_emu_afop;
-    mca_btl_sm.super.btl_atomic_cswap = mca_btl_sm_emu_acswap;
-
     mca_btl_sm.super.btl_latency   = 1;     /* Microsecs */
 
     /* Call the BTL based to register its MCA params */
@@ -427,9 +399,6 @@ static void mca_btl_sm_check_single_copy (void)
     int initial_mechanism = mca_btl_sm_component.single_copy_mechanism;
 #endif
 
-    /* single-copy emulation is always used to support AMO's right now */
-    mca_btl_sm_sc_emu_init ();
-
 #if OPAL_BTL_SM_HAVE_XPMEM
     if (MCA_BTL_SM_XPMEM == mca_btl_sm_component.single_copy_mechanism) {
         /* try to create an xpmem segment for the entire address space */

diff --git a/opal/mca/btl/sm/btl_sm_get.c b/opal/mca/btl/sm/btl_sm_get.c
@@ -167,16 +167,3 @@ int mca_btl_sm_get_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
     return OPAL_SUCCESS;
 }
 #endif
-
-int mca_btl_sm_get_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                              uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                              mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                              int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
-{
-    if (size > mca_btl_sm.super.btl_get_limit) {
-        return OPAL_ERR_NOT_AVAILABLE;
-    }
-
-    return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_GET, 0, 0, 0, order, flags, size,
-                                       local_address, remote_address, cbfunc, cbcontext, cbdata);
-}
diff --git a/opal/mca/btl/sm/btl_sm_module.c b/opal/mca/btl/sm/btl_sm_module.c
@@ -19,7 +19,7 @@
  *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2018-2019 Triad National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2020      Google, LLC. All rights reserved.
+ * Copyright (c) 2020-2021 Google, LLC. All rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -266,8 +266,6 @@ static int init_sm_endpoint (struct mca_btl_base_endpoint_t **ep_out, struct opa
                      * Currently the kernel does not allow * process_vm_{read,write}v()
                      * for processes running in different user namespaces even if
                      * all involved user IDs are mapped to the same user ID.
-                     *
-                     * Fallback to MCA_BTL_SM_EMUL.
                      */
                     if (MCA_BASE_VAR_SOURCE_DEFAULT != source) {
                         /* If CMA has been explicitly selected we want to error out */
@@ -281,11 +279,9 @@ static int init_sm_endpoint (struct mca_btl_base_endpoint_t **ep_out, struct opa
                      */
                     opal_show_help("help-btl-sm.txt", "cma-different-user-namespace-warning",
                                    true, opal_process_info.nodename);
-                    mca_btl_sm_component.single_copy_mechanism = MCA_BTL_SM_EMUL;
-                    mca_btl_sm.super.btl_get = mca_btl_sm_get_sc_emu;
-                    mca_btl_sm.super.btl_put = mca_btl_sm_put_sc_emu;
-                    mca_btl_sm.super.btl_put_limit = mca_btl_sm.super.btl_max_send_size - sizeof (mca_btl_sm_sc_emu_hdr_t);
-                    mca_btl_sm.super.btl_get_limit = mca_btl_sm.super.btl_max_send_size - sizeof (mca_btl_sm_sc_emu_hdr_t);
+                    mca_btl_sm_component.single_copy_mechanism = MCA_BTL_SM_NONE;
+                    mca_btl_sm.super.btl_get = NULL;
+                    mca_btl_sm.super.btl_put = NULL;
                 }
             }
 #if OPAL_BTL_SM_HAVE_XPMEM

diff --git a/opal/mca/btl/sm/btl_sm_put.c b/opal/mca/btl/sm/btl_sm_put.c
@@ -143,19 +143,3 @@ int mca_btl_sm_put_knem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
     return OPAL_SUCCESS;
 }
 #endif
-
-/**
- * @brief Provides an emulated put path which uses copy-in copy-out with shared memory buffers
- */
-int mca_btl_sm_put_sc_emu (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
-                              uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
-                              mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
-                              int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
-{
-    if (size > mca_btl_sm.super.btl_put_limit) {
-        return OPAL_ERR_NOT_AVAILABLE;
-    }
-
-    return mca_btl_sm_rdma_frag_start (btl, endpoint, MCA_BTL_SM_OP_PUT, 0, 0, 0, order, flags, size,
-                                       local_address, remote_address, cbfunc, cbcontext, cbdata);
-}
diff --git a/opal/mca/btl/sm/btl_sm_sc_emu.c b/opal/mca/btl/sm/btl_sm_sc_emu.c