diff --git a/Makefile.am b/Makefile.am index 4d0723f5b17..5bee470f035 100644 --- a/Makefile.am +++ b/Makefile.am @@ -284,6 +284,9 @@ _gni_files = \ prov/gni/src/gnix_fabric.c \ prov/gni/src/gnix_dom.c \ prov/gni/src/gnix_ep.c \ + prov/gni/src/gnix_cq.c \ + prov/gni/src/gnix_av.c \ + prov/gni/src/gnix_mr.c \ prov/gni/src/gnix_ep_rdm.c \ prov/gni/src/gnix_nameserver.c \ prov/gni/src/gnix_util.c diff --git a/configure.ac b/configure.ac index 77ed9303e92..ac060602ecb 100644 --- a/configure.ac +++ b/configure.ac @@ -74,6 +74,9 @@ AC_PROG_CC_C99 dnl Checks for header files. AC_HEADER_STDC +dnl Check for compiler features +AC_C_TYPEOF + LT_INIT dnl dlopen support is optional diff --git a/prov/gni/configure.m4 b/prov/gni/configure.m4 index 1c48a9ba69f..b109215cf64 100644 --- a/prov/gni/configure.m4 +++ b/prov/gni/configure.m4 @@ -26,7 +26,20 @@ m4_include([config/pkg.m4]) LDFLAGS="$CRAY_GNI_HEADER_LIBS $LDFLAGS" ], [gni_header_happy=0]) + PKG_CHECK_MODULES([CRAY_ALPS_LLI], [cray-alpslli], + [alps_lli_happy=1 + CPPFLAGS="$CRAY_ALPS_LLI_CFLAGS $CPPFLAGS" + LDFLAGS="$CRAY_ALPS_LLI_LIBS $LDFLAGS" + ], + [alps_lli_happy=0]) + PKG_CHECK_MODULES([CRAY_ALPS_UTIL], [cray-alpsutil], + [alps_util_happy=1 + CPPFLAGS="$CRAY_ALPS_UTIL_CFLAGS $CPPFLAGS" + LDFLAGS="$CRAY_ALPS_UTIL_LIBS $LDFLAGS" + ], + [alps_util_happy=0]) ]) - AS_IF([test $gni_header_happy -eq 1 -a $ugni_lib_happy -eq 1], [$1], [$2]) + AS_IF([test $gni_header_happy -eq 1 -a $ugni_lib_happy -eq 1 \ + -a $alps_lli_happy -eq 1 -a $alps_util_happy -eq 1], [$1], [$2]) ]) diff --git a/prov/gni/src/gnix.h b/prov/gni/src/gnix.h index 930cb5d7621..b773a4aa344 100644 --- a/prov/gni/src/gnix.h +++ b/prov/gni/src/gnix.h @@ -165,6 +165,7 @@ struct gnix_ep_name { struct { uint32_t name_type : 8; uint32_t unused : 24; + uint32_t cookie; }; uint64_t reserved[4]; }; @@ -182,7 +183,8 @@ enum gnix_progress_type { */ struct gnix_fabric { struct fid_fabric fab_fid; - atomic_t ref; + /* llist of cdm's opened from fabric */ + struct list_head cdm_list; }; /* @@ -192,11 +194,17 @@ struct gnix_fabric { */ struct gnix_domain { struct fid_domain domain_fid; - struct gnix_cdm *cdm; - struct gnix_nic *nic; + /* used for fabric object llist of domains*/ + struct list_node list; + /* list nics this domain is attached to, TODO: thread safety */ + struct list_head nic_list; + /* cm nic bound to this domain */ + struct gnix_cm_nic *cm_nic; + uint8_t ptag; + uint32_t cookie; + /* work queue for domain */ struct list_head domain_wq; - uint32_t device_id; - uint32_t device_addr; + int ref_cnt; }; struct gnix_cdm { @@ -212,8 +220,32 @@ struct gnix_cdm { int ref_cnt; }; +/* + * gnix cm nic struct - to be used only for GNI_EpPostData, etc. + */ + +struct gnix_cm_nic { + struct list_node list; + gni_cdm_handle_t gni_cdm_hndl; + gni_nic_handle_t gni_nic_hndl; + /* free list of datagrams */ + struct list_head datagram_free_list; + /* list of active wc datagrams */ + struct list_head wc_datagram_active_list; + /* free list of wc datagrams */ + struct list_head wc_datagram_free_list; + /* pointer to domain this nic is attached to */ + struct gnix_domain *domain; + struct gnix_datagram *datagram_base; + uint32_t inst_id; + uint32_t device_id; + uint32_t device_addr; + int ref_cnt; +}; + struct gnix_nic { struct list_node list; + gni_cdm_handle_t gni_cdm_hndl; gni_nic_handle_t gni_nic_hndl; /* receive completion queue for hndl */ gni_cq_handle_t rx_cq; @@ -227,7 +259,8 @@ struct gnix_nic { struct list_head wqe_active_list; /* list for managing wqe's */ struct gnix_wqe_list *wqe_list; - /* list of active smsg req's */ + /* pointer to domain this nic is attached to */ + struct gnix_domain *domain; struct list_head smsg_active_req_list; /* list for managing smsg req's */ struct gnix_smsg_req_list *smsg_req_list; @@ -301,18 +334,40 @@ struct gnix_rdm_ep { /* * globals */ -extern const char const gnix_fab_name[]; +extern const char gnix_fab_name[]; +extern const char gnix_dom_name[]; +extern uint32_t gnix_cdm_modes; + +/* + * linked list helpers + */ + +static inline void gnix_list_node_init(struct list_node *node) +{ + node->prev = node->next = NULL; +} + +static inline void gnix_list_del_init(struct list_node *node) +{ + list_del(node); + node->prev = node->next = node; +} /* * prototypes */ int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info, struct fid_domain **domain, void *context); -int gnix_rdm_getinfo(uint32_t version, const char *node, const char *service, - uint64_t flags, struct fi_info *hints, - struct fi_info **info); -struct fi_info *gnix_fi_info(enum fi_ep_type ep_type, struct fi_info *hints); -int gnix_verify_domain_attr(struct fi_domain_attr *attr); +int gnix_av_open(struct fid_domain *domain, struct fi_av_attr *attr, + struct fid_av **av, void *context); +int gnix_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, + struct fid_cq **cq, void *context); +int gnix_ep_open(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, void *context); + +int gnix_mr_reg(struct fid_domain *domain, const void *buf, size_t len, + uint64_t access, uint64_t offset, uint64_t requested_key, + uint64_t flags, struct fid_mr **mr, void *context); #ifdef __cplusplus } /* extern "C" */ diff --git a/prov/gni/src/gnix_av.c b/prov/gni/src/gnix_av.c new file mode 100644 index 00000000000..d61fc0000e1 --- /dev/null +++ b/prov/gni/src/gnix_av.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2015 Cray Inc. All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. Allrights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// +// Address vector common code +// +#include +#include + +#include "gnix.h" +#include "gnix_util.h" + +int gnix_av_open(struct fid_domain *domain, struct fi_av_attr *attr, + struct fid_av **av, void *context) +{ + /* TODO: implement this puppy */ + return -FI_ENOSYS; +} diff --git a/prov/gni/src/gnix_cq.c b/prov/gni/src/gnix_cq.c new file mode 100644 index 00000000000..6ecabe36d8e --- /dev/null +++ b/prov/gni/src/gnix_cq.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2015 Cray Inc. All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. Allrights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// +// CQ common code +// +#include +#include + +#include "gnix.h" +#include "gnix_util.h" + +int gnix_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, + struct fid_cq **cq, void *context) +{ + /* TODO: implement this puppy */ + return -FI_ENOSYS; +} diff --git a/prov/gni/src/gnix_dom.c b/prov/gni/src/gnix_dom.c index b652180eb3b..05a14e49845 100644 --- a/prov/gni/src/gnix_dom.c +++ b/prov/gni/src/gnix_dom.c @@ -1,5 +1,4 @@ /* - * Copyright (c) 2014 Intel Corporation, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. Allrights reserved. * Copyright (c) 2015 Cray Inc. All rights reserved. * @@ -37,14 +36,196 @@ #endif /* HAVE_CONFIG_H */ #include -#include #include "gnix.h" #include "gnix_util.h" +static int gnix_domain_close(fid_t fid) +{ + int ret = FI_SUCCESS; + struct gnix_domain *domain; + struct gnix_cm_nic *cm_nic; + struct gnix_nic *p, *next; + gni_return_t status; + + domain = container_of(fid, struct gnix_domain, domain_fid.fid); + if (domain->ref_cnt) { + --domain->ref_cnt; + } + + GNIX_LOG_INFO("gnix_domain_close invoked.\n"); + + if (domain->ref_cnt == 0) { + if (domain->cm_nic) { + cm_nic = domain->cm_nic; + if (cm_nic->gni_cdm_hndl) { + status = GNI_CdmDestroy(cm_nic->gni_cdm_hndl); + if (status != GNI_RC_SUCCESS) { + GNIX_LOG_ERROR("oops, cdm destroy" + "failed\n"); + } + } + free(domain->cm_nic); + } + + list_for_each_safe(&domain->nic_list, p, next, list) + { + list_del(&p->list); + gnix_list_node_init(&p->list); + /* TODO: free nic here */ + } + + /* + * remove from the list of cdms attached to fabric + */ + gnix_list_del_init(&domain->list); + + memset(domain, 0, sizeof *domain); + free(domain); + } + + GNIX_LOG_INFO("gnix_domain_close invoked returning %d\n", ret); + return ret; +} + +static struct fi_ops gnix_fi_ops = { + .size = sizeof(struct fi_ops), + .close = gnix_domain_close, + .bind = fi_no_bind, + .control = fi_no_control +}; + +static struct fi_ops_domain gnix_domain_ops = { + .size = sizeof(struct fi_ops_domain), + .av_open = gnix_av_open, + .cq_open = gnix_cq_open, + .endpoint = gnix_ep_open, + /* TODO: no cntrs for now in gnix */ + .cntr_open = fi_no_cntr_open, + .poll_open = fi_no_poll_open, + .stx_ctx = fi_no_stx_context, + .srx_ctx = fi_no_srx_context +}; + +static struct fi_ops_mr gnix_domain_mr_ops = { + .size = sizeof(struct fi_ops_mr), + .reg = gnix_mr_reg +}; + int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info, struct fid_domain **dom, void *context) { - /* TODO: need to implement */ - return -FI_ENOSYS; + struct gnix_domain *domain = NULL; + int ret = FI_SUCCESS; + uint8_t ptag; + uint32_t cookie, device_addr; + struct gnix_cm_nic *cm_nic = NULL; + struct gnix_fabric *fabric_priv; + gni_return_t status; + + GNIX_LOG_INFO("%s\n", __func__); + + fabric_priv = container_of(fabric, struct gnix_fabric, fab_fid); + if (!info->domain_attr->name || + strncmp(info->domain_attr->name, gnix_dom_name, + strlen(gnix_dom_name))) { + ret = -FI_EINVAL; + goto fn_err; + } + + /* + * check cookie/ptag credentials - for FI_EP_MSG we may be creating a + * domain + * using a cookie supplied being used by the server. Otherwise, we use + * use the cookie/ptag supplied by the job launch system. + */ + if (info->dest_addr) { + ret = + gnixu_get_rdma_credentials(info->dest_addr, &ptag, &cookie); + if (ret) { + GNIX_LOG_ERROR("gnixu_get_rdma_credentials returned" + "ptag %d cookie 0x%x\n", ptag, cookie); + goto fn_err; + } + } else { + ret = gnixu_get_rdma_credentials(NULL, &ptag, &cookie); + } + + GNIX_LOG_INFO("gnix rdma credentials returned ptag %d cookie 0x%x\n", + ptag, cookie); + domain = calloc(1, sizeof *domain); + if (domain == NULL) { + ret = -FI_ENOMEM; + goto fn_err; + } + + list_head_init(&domain->nic_list); + gnix_list_node_init(&domain->list); + + list_add_tail(&fabric_priv->cdm_list, &domain->list); + + list_head_init(&domain->domain_wq); + + /* + * Set up the connection management nic for this domain + */ + cm_nic = (struct gnix_cm_nic *)calloc(1, sizeof *cm_nic); + if (cm_nic == NULL) { + ret = -FI_ENOMEM; + goto fn_err; + } + + gnix_list_node_init(&cm_nic->list); + list_head_init(&cm_nic->datagram_free_list); + list_head_init(&cm_nic->wc_datagram_active_list); + list_head_init(&cm_nic->wc_datagram_free_list); + + status = GNI_CdmCreate(getpid(), ptag, cookie, gnix_cdm_modes, + &cm_nic->gni_cdm_hndl); + if (status != GNI_RC_SUCCESS) { + GNIX_LOG_ERROR("GNI_CdmCreate returned %s\n", + gni_err_str[status]); + /* TODO: need a translater from gni to fi errors */ + ret = -FI_EACCES; + goto fn_err; + } + + /* + * Okay, now go for the attach + */ + status = GNI_CdmAttach(cm_nic->gni_cdm_hndl, 0, &device_addr, + &cm_nic->gni_nic_hndl); + if (status != GNI_RC_SUCCESS) { + GNIX_LOG_ERROR("GNI_CdmAttach returned %s\n", + gni_err_str[status]); + ret = -FI_EACCES; + goto fn_err; + } + + domain->cm_nic = cm_nic; + domain->ptag = ptag; + domain->cookie = cookie; + domain->ref_cnt = 1; + cm_nic->domain = domain; + + domain->domain_fid.fid.fclass = FI_CLASS_DOMAIN; + domain->domain_fid.fid.context = context; + domain->domain_fid.fid.ops = &gnix_fi_ops; + domain->domain_fid.ops = &gnix_domain_ops; + domain->domain_fid.mr = &gnix_domain_mr_ops; + + *dom = &domain->domain_fid; + return FI_SUCCESS; + +fn_err: + if (cm_nic && cm_nic->gni_cdm_hndl) { + GNI_CdmDestroy(cm_nic->gni_cdm_hndl); + } + if (domain != NULL) { + free(domain); + } + if (cm_nic != NULL) { + free(cm_nic); + } + return ret; } diff --git a/prov/gni/src/gnix_ep.c b/prov/gni/src/gnix_ep.c index 3d23392e2e3..f687a35771f 100644 --- a/prov/gni/src/gnix_ep.c +++ b/prov/gni/src/gnix_ep.c @@ -38,3 +38,10 @@ #include "gnix.h" #include "gnix_util.h" + +int gnix_ep_open(struct fid_domain *domain, struct fi_info *info, + struct fid_ep **ep, void *context) +{ + /* TODO: implement this puppy */ + return -FI_ENOSYS; +} diff --git a/prov/gni/src/gnix_fabric.c b/prov/gni/src/gnix_fabric.c index ab122d17562..b868bf4e9cc 100644 --- a/prov/gni/src/gnix_fabric.c +++ b/prov/gni/src/gnix_fabric.c @@ -54,8 +54,13 @@ #include "gnix_util.h" #include "gnix_nameserver.h" -const const char gnix_fab_name[] = "gni"; -const const char gnix_dom_name[] = "/sys/class/gni/kgni0"; +const char gnix_fab_name[] = "gni"; +const char gnix_dom_name[] = "/sys/class/gni/kgni0"; + +uint32_t gnix_cdm_modes = + (GNI_CDM_MODE_FAST_DATAGRAM_POLL | GNI_CDM_MODE_FMA_SHARED | + GNI_CDM_MODE_FMA_SMALL_WINDOW | GNI_CDM_MODE_FORK_PARTCOPY | + GNI_CDM_MODE_ERR_NO_KILL); const struct fi_fabric_attr gnix_fabric_attr = { .fabric = NULL, @@ -80,12 +85,12 @@ static int gnix_fabric_close(fid_t fid) struct gnix_fabric *fab; fab = container_of(fid, struct gnix_fabric, fab_fid); - if (atomic_get(&fab->ref)) { + if (!list_empty(&fab->cdm_list)) { return -FI_EBUSY; } free(fab); - return 0; + return FI_SUCCESS; } static struct fi_ops gnix_fab_fi_ops = { @@ -117,11 +122,10 @@ static int gnix_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, fab->fab_fid.fid.context = context; fab->fab_fid.fid.ops = &gnix_fab_fi_ops; fab->fab_fid.ops = &gnix_fab_ops; + list_head_init(&fab->cdm_list); *fabric = &fab->fab_fid; -#if 0 - atomic_init(&fab->ref, 0); -#endif - return 0; + + return FI_SUCCESS; } static int gnix_getinfo(uint32_t version, const char *node, const char *service, @@ -131,7 +135,9 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service, int ret = 0; int mode = GNIX_FAB_MODES; struct fi_info *gnix_info; - struct gnix_ep_name *dest_addr = NULL, *src_addr = NULL, *addr = NULL; + struct gnix_ep_name *dest_addr = NULL; + struct gnix_ep_name *src_addr = NULL; + struct gnix_ep_name *addr = NULL; /* * the code below for resolving a node/service to what @@ -296,7 +302,7 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service, gnix_info->tx_attr->caps = gnix_info->caps; gnix_info->tx_attr->mode = gnix_info->mode; - if(hints && hints->tx_attr && hints->tx_attr->op_flags) { + if (hints && hints->tx_attr && hints->tx_attr->op_flags) { gnix_info->tx_attr->op_flags = hints->tx_attr->op_flags; } else { gnix_info->tx_attr->op_flags = GNIX_EP_OP_FLAGS; @@ -312,7 +318,7 @@ static int gnix_getinfo(uint32_t version, const char *node, const char *service, gnix_info->rx_attr->caps = gnix_info->caps; gnix_info->rx_attr->mode = gnix_info->mode; - if(hints && hints->rx_attr && hints->rx_attr->op_flags) { + if (hints && hints->rx_attr && hints->rx_attr->op_flags) { gnix_info->rx_attr->op_flags = hints->rx_attr->op_flags; } else { gnix_info->rx_attr->op_flags = GNIX_EP_OP_FLAGS; diff --git a/prov/gni/src/gnix_mr.c b/prov/gni/src/gnix_mr.c new file mode 100644 index 00000000000..bf86bd33ca2 --- /dev/null +++ b/prov/gni/src/gnix_mr.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2015 Cray Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// +// memory registration common code +// +#include +#include + +#include "gnix.h" +#include "gnix_util.h" + +int gnix_mr_reg(struct fid_domain *domain, const void *buf, size_t len, + uint64_t access, uint64_t offset, uint64_t requested_key, + uint64_t flags, struct fid_mr **mr, void *context) +{ + /* TODO: implement this puppy */ + return -FI_ENOSYS; +} diff --git a/prov/gni/src/gnix_util.c b/prov/gni/src/gnix_util.c index 915937822e4..7478a855461 100644 --- a/prov/gni/src/gnix_util.c +++ b/prov/gni/src/gnix_util.c @@ -53,5 +53,108 @@ #include #include +#include "alps/alps.h" +#include "alps/alps_toolAssist.h" +#include "alps/libalpsutil.h" +#include "alps/libalpslli.h" + #include "gnix.h" #include "gnix_util.h" + +int gnixu_get_rdma_credentials(void *addr, uint8_t *ptag, uint32_t *cookie) +{ + int ret = FI_SUCCESS; + int alps_status = 0; + uint64_t apid; + size_t alps_count; + alpsAppLLIGni_t *rdmacred_rsp = NULL; + alpsAppGni_t *rdmacred_buf = NULL; + + if ((ptag == NULL) || (cookie == NULL)) { + ret = -FI_EINVAL; + goto err; + } + + /* + * TODO: need to handle non null addr differently at some point, + * a non-NULL addr can be used to acquire RDMA credentials other than + * those assigned by ALPS/nativized slurm. + */ + /* lli_lock doesn't return anything useful */ + ret = alps_app_lli_lock(); + + /* + * First get our apid + */ + ret = alps_app_lli_put_request(ALPS_APP_LLI_ALPS_REQ_APID, NULL, 0); + if (ret != ALPS_APP_LLI_ALPS_STAT_OK) { + GNIX_LOG_ERROR("lli put failed, ret=%s\n", strerror(ret)); + ret = -FI_EIO; + goto err; + } + + ret = alps_app_lli_get_response(&alps_status, &alps_count); + if (alps_status != ALPS_APP_LLI_ALPS_STAT_OK) { + GNIX_LOG_ERROR("lli get response failed, ret=%s\n", + strerror(ret)); + ret = -FI_EIO; + goto err; + } + + ret = alps_app_lli_get_response_bytes(&apid, sizeof(apid)); + if (ret != ALPS_APP_LLI_ALPS_STAT_OK) { + GNIX_LOG_ERROR("lli get response failed, ret=%s\n", + strerror(ret)); + ret = -FI_EIO; + goto err; + } + + /* + * now get the GNI rdma credentials info + */ + ret = alps_app_lli_put_request(ALPS_APP_LLI_ALPS_REQ_GNI, NULL, 0); + if (ret != ALPS_APP_LLI_ALPS_STAT_OK) { + GNIX_LOG_ERROR("lli put failed, ret=%s\n", strerror(ret)); + ret = -FI_EIO; + goto err; + } + + ret = alps_app_lli_get_response(&alps_status, &alps_count); + if (alps_status != ALPS_APP_LLI_ALPS_STAT_OK) { + GNIX_LOG_ERROR("lli get response failed, ret=%s\n", + strerror(ret)); + ret = -FI_EIO; + goto err; + } + + rdmacred_rsp = malloc(alps_count); + if (rdmacred_rsp == NULL) { + ret = -FI_ENOMEM; + goto err; + } + + memset(rdmacred_rsp, 0, alps_count); + + ret = alps_app_lli_get_response_bytes(rdmacred_rsp, alps_count); + if (ret != ALPS_APP_LLI_ALPS_STAT_OK) { + GNIX_LOG_ERROR("lli get response failed, ret=%s\n", + strerror(ret)); + ret = -FI_EIO; + goto err; + } + + rdmacred_buf = (alpsAppGni_t *) rdmacred_rsp->u.buf; + + /* + * just use the first ptag/cookie for now + */ + + *ptag = rdmacred_buf[0].ptag; + *cookie = rdmacred_buf[0].cookie; +err: + alps_app_lli_unlock(); + if (rdmacred_rsp != NULL) { + free(rdmacred_rsp); + } + return ret; +} diff --git a/prov/gni/src/gnix_util.h b/prov/gni/src/gnix_util.h index e0ddae0e7ee..c115f03e075 100644 --- a/prov/gni/src/gnix_util.h +++ b/prov/gni/src/gnix_util.h @@ -49,4 +49,9 @@ extern const char gnix_dom_name[]; #define GNIX_LOG_WARN(...) FI_WARN(gnix_fab_name, __VA_ARGS__) #define GNIX_LOG_ERROR(...) FI_WARN(gnix_fab_name, __VA_ARGS__) +/* + * prototypes + */ +int gnixu_get_rdma_credentials(void *addr, uint8_t *ptag, uint32_t *cookie); + #endif