Merge pull request ofiwg#17 from bturrubiates/cray-master

Cray master

Makefile.am

@@ -284,6 +284,9 @@ _gni_files = \
 	prov/gni/src/gnix_fabric.c \
 	prov/gni/src/gnix_dom.c \
 	prov/gni/src/gnix_ep.c \
+	prov/gni/src/gnix_cq.c \
+	prov/gni/src/gnix_av.c \
+	prov/gni/src/gnix_mr.c \
 	prov/gni/src/gnix_ep_rdm.c \
 	prov/gni/src/gnix_nameserver.c \
 	prov/gni/src/gnix_util.c 

configure.ac

@@ -74,6 +74,9 @@ AC_PROG_CC_C99
 dnl Checks for header files.
 AC_HEADER_STDC
 
+dnl Check for compiler features
+AC_C_TYPEOF
+
 LT_INIT
 
 dnl dlopen support is optional

prov/gni/configure.m4

@@ -26,7 +26,20 @@ m4_include([config/pkg.m4])
                                   LDFLAGS="$CRAY_GNI_HEADER_LIBS $LDFLAGS"
                                  ],
                                  [gni_header_happy=0])
+               PKG_CHECK_MODULES([CRAY_ALPS_LLI], [cray-alpslli],
+                                 [alps_lli_happy=1
+                                  CPPFLAGS="$CRAY_ALPS_LLI_CFLAGS $CPPFLAGS"
+                                  LDFLAGS="$CRAY_ALPS_LLI_LIBS $LDFLAGS"
+                                 ],
+                                 [alps_lli_happy=0])
+               PKG_CHECK_MODULES([CRAY_ALPS_UTIL], [cray-alpsutil],
+                                 [alps_util_happy=1
+                                  CPPFLAGS="$CRAY_ALPS_UTIL_CFLAGS $CPPFLAGS"
+                                  LDFLAGS="$CRAY_ALPS_UTIL_LIBS $LDFLAGS"
+                                 ],
+                                 [alps_util_happy=0])
 	       ])
-	AS_IF([test $gni_header_happy -eq 1 -a $ugni_lib_happy -eq 1], [$1], [$2])
+	AS_IF([test $gni_header_happy -eq 1 -a $ugni_lib_happy -eq 1 \
+               -a $alps_lli_happy -eq 1 -a $alps_util_happy -eq 1], [$1], [$2])
 ])
 

prov/gni/src/gnix.h

@@ -165,6 +165,7 @@ struct gnix_ep_name {
 	struct {
 		uint32_t name_type : 8;
 		uint32_t unused : 24;
+		uint32_t cookie;
 	};
 	uint64_t reserved[4];
 };
@@ -182,7 +183,8 @@ enum gnix_progress_type {
  */
 struct gnix_fabric {
 	struct fid_fabric fab_fid;
-	atomic_t ref;
+	/* llist of cdm's opened from fabric */
+	struct list_head cdm_list;
 };
 
 /*
@@ -192,11 +194,17 @@ struct gnix_fabric {
  */
 struct gnix_domain {
 	struct fid_domain domain_fid;
-	struct gnix_cdm *cdm;
-	struct gnix_nic *nic;
+	/* used for fabric object llist of domains*/
+	struct list_node list;
+	/* list nics this domain is attached to, TODO: thread safety */
+	struct list_head nic_list;
+	/* cm nic bound to this domain */
+	struct gnix_cm_nic *cm_nic;
+	uint8_t ptag;
+	uint32_t cookie;
+	/* work queue for domain */
 	struct list_head domain_wq;
-	uint32_t device_id;
-	uint32_t device_addr;
+	int ref_cnt;
 };
 
 struct gnix_cdm {
@@ -212,8 +220,32 @@ struct gnix_cdm {
 	int ref_cnt;
 };
 
+/*
+ * gnix cm nic struct - to be used only for GNI_EpPostData, etc.
+ */
+
+struct gnix_cm_nic {
+	struct list_node list;
+	gni_cdm_handle_t gni_cdm_hndl;
+	gni_nic_handle_t gni_nic_hndl;
+	/* free list of datagrams   */
+	struct list_head datagram_free_list;
+	/* list of active wc datagrams   */
+	struct list_head wc_datagram_active_list;
+	/* free list of wc datagrams   */
+	struct list_head wc_datagram_free_list;
+	/* pointer to domain this nic is attached to */
+	struct gnix_domain *domain;
+	struct gnix_datagram *datagram_base;
+	uint32_t inst_id;
+	uint32_t device_id;
+	uint32_t device_addr;
+	int ref_cnt;
+};
+
 struct gnix_nic {
 	struct list_node list;
+	gni_cdm_handle_t gni_cdm_hndl;
 	gni_nic_handle_t gni_nic_hndl;
 	/* receive completion queue for hndl */
 	gni_cq_handle_t rx_cq;
@@ -227,7 +259,8 @@ struct gnix_nic {
 	struct list_head wqe_active_list;
 	/* list for managing wqe's */
 	struct gnix_wqe_list *wqe_list;
-	/* list of active smsg req's */
+	/* pointer to domain this nic is attached to */
+	struct gnix_domain *domain;
 	struct list_head smsg_active_req_list;
 	/* list for managing smsg req's */
 	struct gnix_smsg_req_list *smsg_req_list;
@@ -301,18 +334,40 @@ struct gnix_rdm_ep {
 /*
  * globals
  */
-extern const char const gnix_fab_name[];
+extern const char gnix_fab_name[];
+extern const char gnix_dom_name[];
+extern uint32_t gnix_cdm_modes;
+
+/*
+ * linked list helpers
+ */
+
+static inline void gnix_list_node_init(struct list_node *node)
+{
+	node->prev = node->next = NULL;
+}
+
+static inline void gnix_list_del_init(struct list_node *node)
+{
+	list_del(node);
+	node->prev = node->next = node;
+}
 
 /*
  * prototypes
  */
 int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info,
 		     struct fid_domain **domain, void *context);
-int gnix_rdm_getinfo(uint32_t version, const char *node, const char *service,
-		     uint64_t flags, struct fi_info *hints,
-		     struct fi_info **info);
-struct fi_info *gnix_fi_info(enum fi_ep_type ep_type, struct fi_info *hints);
-int gnix_verify_domain_attr(struct fi_domain_attr *attr);
+int gnix_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
+		 struct fid_av **av, void *context);
+int gnix_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
+		 struct fid_cq **cq, void *context);
+int gnix_ep_open(struct fid_domain *domain, struct fi_info *info,
+		 struct fid_ep **ep, void *context);
+
+int gnix_mr_reg(struct fid_domain *domain, const void *buf, size_t len,
+		uint64_t access, uint64_t offset, uint64_t requested_key,
+		uint64_t flags, struct fid_mr **mr, void *context);
 
 #ifdef __cplusplus
 } /* extern "C" */

prov/gni/src/gnix_av.c

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015 Los Alamos National Security, LLC. Allrights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+//
+// Address vector common code
+//
+#include <stdlib.h>
+#include <string.h>
+
+#include "gnix.h"
+#include "gnix_util.h"
+
+int gnix_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
+		 struct fid_av **av, void *context)
+{
+	/* TODO: implement this puppy */
+	return -FI_ENOSYS;
+}

prov/gni/src/gnix_cq.c

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2015 Cray Inc. All rights reserved.
+ * Copyright (c) 2015 Los Alamos National Security, LLC. Allrights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+//
+// CQ common code
+//
+#include <stdlib.h>
+#include <string.h>
+
+#include "gnix.h"
+#include "gnix_util.h"
+
+int gnix_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
+		 struct fid_cq **cq, void *context)
+{
+	/* TODO: implement this puppy */
+	return -FI_ENOSYS;
+}