Merge pull request #14 from yosefe/topic/ib-setup

UCT/IB/RC: Add resource query and setup infrastructure.

config/m4/ib.m4

@@ -5,21 +5,78 @@
 # $HEADER$
 #
 
+
+#
+# Select IB transports
+#
 with_ib=no
 
+
 #
 # RC Support
 #
 AC_ARG_WITH([rc],
-            [AC_HELP_STRING([--with-rc], [Compile with RC support])],
+            [AC_HELP_STRING([--with-rc], [Compile with IB Reliable Connection support])],
             [],
             [with_rc=yes])
-AM_CONDITIONAL([HAVE_TL_RC], [test "x$with_rc" != xno])
 AS_IF([test "x$with_rc" != xno], 
       [AC_DEFINE([HAVE_TL_RC], 1, [RC transport support])
        with_ib=yes
        transports="${transports},rc"])
 
-AM_CONDITIONAL([HAVE_IB], [test "x$with_ib" != xno])
+
+AC_ARG_WITH([ud],
+            [AC_HELP_STRING([--with-ud], [Compile with IB Unreliable Datagram support])],
+            [],
+            [with_ud=yes;with_ib=yes])
+AS_IF([test "x$with_ud" != xno],
+      [AC_DEFINE([HAVE_TL_UD], 1, [UD transport support])
+       with_ib=yes
+       transports="${transports},ud"])
+
+
+AC_ARG_WITH([dc],
+            [AC_HELP_STRING([--with-dc], [Compile with IB Dynamic Connection support])],
+            [],
+            [with_dc=yes;with_ib=yes])
+AS_IF([test "x$with_dc" != xno],
+      [AC_CHECK_DECLS(IBV_EXP_QPT_DC_INI, [], [with_dc=no], [[#include <infiniband/verbs.h>]])
+       AC_CHECK_MEMBERS([struct ibv_exp_dct_init_attr.inline_size], [] , [with_dc=no], [[#include <infiniband/verbs.h>]])
+      ])
+AS_IF([test "x$with_dc" != xno],
+      [AC_DEFINE([HAVE_TL_DC], 1, [DC transport support])
+       with_ib=yes
+       transports="${transports},dc"])
+
+
+#
+# Check basic IB support: User wanted at least one IB transport, and we found
+# verbs header file and library.
+#
+AS_IF([test "x$with_ib" != xno],
+      [AC_CHECK_HEADER([infiniband/verbs.h], [],
+                       [AC_MSG_ERROR([ibverbs header files not found]);with_ib=no])
+       save_LDFLAGS="$LDFLAGS"
+       AC_CHECK_LIB([ibverbs], [ibv_get_device_list],
+                    [AC_SUBST(IBVERBS_LDFLAGS, [-libverbs])],
+                    [AC_MSG_ERROR([libibverbs not found]);with_ib=no])
+       LDFLAGS="$save_LDFLAGS"
+      ])
 AS_IF([test "x$with_ib" != xno],
       [AC_DEFINE([HAVE_IB], 1, [IB support])])
+
+
+#
+# Check for experimental verbs support
+#
+AC_CHECK_HEADER([infiniband/verbs_exp.h],
+                [AC_DEFINE([HAVE_VERBS_EXP_H], 1, [IB experimental verbs])
+                 verbs_exp=yes],
+                [verbs_exp=no])
+
+
+#
+# For automake
+#
+AM_CONDITIONAL([HAVE_IB], [test "x$with_ib" != xno])
+AM_CONDITIONAL([HAVE_TL_RC], [test "x$with_rc" != xno])

doc/CodeStyle

@@ -3,12 +3,18 @@
 * Style
  - 4 spaces, no tabs
  - up to 80 columns
- - names must begin with ucp/uct/ucs
  - single space around operators
  - indent function arguments on column
  - no spaces in the end-of-line
 
-* C++
+ * Naming convention:
+ - lower case, underscores
+ - names must begin with ucp/uct/ucs
+ - an output argument which is a pointer to a user variable has _p suffix
+ - value types (e.g struct types, integer types) have _t suffix
+ - pointer to structs, which are used as API handles, have _h suffix
+
+ * C++
  - used only for unit testing
  - lower-case class names (same as stl/boost)
 

src/ucs/sys/sys.c

@@ -338,6 +338,15 @@ uint64_t ucs_string_to_id(const char* str)
     return id;
 }
 
+void ucs_snprintf_zero(char *buf, size_t size, const char *fmt, ...)
+{
+    va_list ap;
+
+    memset(buf, 0, size);
+    va_start(ap, fmt);
+    vsnprintf(buf, size - 1, fmt, ap);
+    va_end(ap);
+}
 
 ssize_t ucs_read_file(char *buffer, size_t max, int silent,
                       const char *filename_fmt, ...)

src/ucs/sys/sys.h

@@ -172,6 +172,18 @@ ucs_open_output_stream(const char *config_str, FILE **p_fstream, int *p_need_clo
 uint64_t ucs_string_to_id(const char *str);
 
 
+/**
+ * Format a string to a buffer of given size, and fill the rest of the buffer
+ * with '\0'. Also, guarantee that the last char in the buffer is '\0'.
+ *
+ * @param buf  Buffer to format the string to.
+ * @param size Buffer size.
+ * @param fmt  Format string.
+ */
+void ucs_snprintf_zero(char *buf, size_t size, const char *fmt, ...)
+    UCS_F_PRINTF(3, 4);
+
+
 /**
  * Read file contents into a string. If the size of the data is smaller than the
  * supplied upper limit (max), a null terminator is appended to the data.

src/uct/Makefile.am

@@ -8,7 +8,7 @@
 lib_LTLIBRARIES     = libuct.la
 
 libuct_la_CPPFLAGS = -I$(abs_top_srcdir)/src -I$(abs_top_builddir)/src
-libuct_la_LDFLAGS  = -ldl -version-info $(SOVERSION)
+libuct_la_LDFLAGS  = -ldl -version-info $(SOVERSION) $(IBVERBS_LDFLAGS)
 libuct_la_LIBADD   = $(LIBM) ../ucs/libucs.la
 libuct_ladir       = $(includedir)
 
@@ -25,11 +25,24 @@ libuct_la_SOURCES = \
 	tl/context.c \
 	tl/tl.c
 
-
 if HAVE_IB
 noinst_HEADERS += \
-	ib/ib_context.h
+	ib/base/ib_context.h \
+	ib/base/ib_device.h \
+	ib/base/ib_iface.h
+	ib/base/ib_verbs.h
+
+libuct_la_SOURCES += \
+	ib/base/ib_context.c \
+	ib/base/ib_device.c \
+	ib/base/ib_iface.c
+endif
+
+if HAVE_TL_RC
+noinst_HEADERS += \
+	ib/rc/rc_iface.h
 
 libuct_la_SOURCES += \
-	ib/ib_context.c
+	ib/rc/rc_iface.c \
+	ib/rc/rc_tl.c
 endif

src/uct/api/tl.h

@@ -15,21 +15,24 @@
 #include "uct_def.h"
 
 #include <ucs/type/status.h>
+#include <sys/socket.h>
 #include <stddef.h>
+#include <sched.h>
 
 
 /**
  * Communication interface context
  */
 typedef struct uct_iface {
+    uct_tl_ops_t             *ops;
 } uct_iface_t;
 
 
 /**
  * Remote endpoint
  */
 typedef struct uct_ep {
-    uct_ops_t           *ops;
+    uct_tl_ops_t             *ops;
 } uct_ep_t;
 
 
@@ -52,12 +55,32 @@ typedef struct uct_iface_attr {
 } uct_iface_attr_t;
 
 
+/**
+ * Communication resource.
+ */
+typedef struct uct_resource_desc {
+    char                     tl_name[UCT_MAX_NAME_LEN];  /* Transport name */
+    char                     hw_name[UCT_MAX_NAME_LEN];  /* Hardware resource name */
+    uint64_t                 latency;      /* Latency, nanoseconds */
+    size_t                   bandwidth;    /* Bandwidth, bytes/second */
+    cpu_set_t                local_cpus;   /* Mask of CPUs near the resource */
+    socklen_t                addrlen;      /* Size of address */
+    struct sockaddr_storage  subnet_addr;  /* Subnet address. Devices which can
+                                              reach each other have same address */
+} uct_resource_desc_t;
+
+
 /**
  * Transport operations.
  */
-struct uct_ops {
+struct uct_tl_ops {
+
+    ucs_status_t (*query_resources)(uct_context_h context,
+                                    uct_resource_desc_t **resources_p,
+                                    unsigned *num_resources_p);
 
-    ucs_status_t (*iface_open)(uct_context_h *context, uct_iface_h *iface_p);
+    ucs_status_t (*iface_open)(uct_context_h context, const char *hw_name,
+                               uct_iface_h *iface_p);
     void         (*iface_close)(uct_iface_h iface);
 
     ucs_status_t (*iface_query)(uct_iface_h iface,
@@ -80,7 +103,6 @@ struct uct_ops {
 
     ucs_status_t (*iface_flush)(uct_iface_h iface, uct_req_h *req_p,
                                   uct_completion_cb_t cb);
-
 };
 
 

src/uct/api/uct.h

@@ -32,4 +32,51 @@ ucs_status_t uct_init(uct_context_h *context_p);
 void uct_cleanup(uct_context_h context);
 
 
+/**
+ * @ingroup CONTEXT
+ * @brief Query for transport resources.
+ *
+ * @param [in]  context         Handle to context.
+ * @param [out] resources_p     Filled with a pointer to an array of resource descriptors.
+ * @param [out] num_resources_p Filled with the number of resources in the array.
+ *
+ * @return Error code.
+ */
+ucs_status_t uct_query_resources(uct_context_h context,
+                                 uct_resource_desc_t **resources_p,
+                                 unsigned *num_resources_p);
+
+
+/**
+ * @ingroup CONTEXT
+ * @brief Release the list of resources returned from uct_query_resources.
+ *
+ * @param [in] resources  Array of resource descriptors to release.
+ *
+ */
+void uct_release_resource_list(uct_resource_desc_t *resources);
+
+
+/**
+ * @brief Open a communication interface.
+ *
+ * @param [in]  context       Handle to context.
+ * @param [in]  tl_name       Transport name.
+ * @param [in]  hw_name       Hardware resource name,
+ * @param [out] iface_p       Filled with a handle to opened communication interface.
+ *
+ * @return Error code.
+ */
+ucs_status_t uct_iface_open(uct_context_h context, const char *tl_name,
+                            const char *hw_name, uct_iface_h *iface_p);
+
+
+/**
+ * @brief Close a communication interface.
+ *
+ * @param [in]  iface         Interface to close.
+ */
+void uct_iface_close(uct_iface_h iface);
+
+
 #endif

src/uct/api/uct_def.h

@@ -10,13 +10,14 @@
 
 #include <stdint.h>
 
+#define UCT_MAX_NAME_LEN         64
 
 typedef struct uct_context       *uct_context_h;
 typedef struct uct_iface         *uct_iface_h;
 typedef struct uct_iface_addr    uct_iface_addr_t;
 typedef struct uct_ep            *uct_ep_h;
 typedef struct uct_ep_addr       uct_ep_addr_t;
-typedef struct uct_ops           uct_ops_t;
+typedef struct uct_tl_ops        uct_tl_ops_t;
 typedef uint64_t                 uct_lkey_t;
 typedef uint64_t                 uct_rkey_t;
 typedef struct uct_req           *uct_req_h;