diff --git a/prov/efa/src/rxr/rxr.h b/prov/efa/src/rxr/rxr.h index 71f3ce77e71..167db56b160 100644 --- a/prov/efa/src/rxr/rxr.h +++ b/prov/efa/src/rxr/rxr.h @@ -535,6 +535,7 @@ struct rxr_ep { /* rx/tx queue size of core provider */ size_t core_rx_size; size_t max_outstanding_tx; + size_t core_inject_size; size_t max_data_payload_size; /* Resource management flag */ diff --git a/prov/efa/src/rxr/rxr_ep.c b/prov/efa/src/rxr/rxr_ep.c index 5fa8c94d923..a1b1b4422f8 100644 --- a/prov/efa/src/rxr/rxr_ep.c +++ b/prov/efa/src/rxr/rxr_ep.c @@ -1673,6 +1673,7 @@ int rxr_endpoint(struct fid_domain *domain, struct fi_info *info, assert(info->tx_attr->msg_order == info->rx_attr->msg_order); rxr_ep->msg_order = info->rx_attr->msg_order; rxr_ep->core_msg_order = rdm_info->rx_attr->msg_order; + rxr_ep->core_inject_size = rdm_info->tx_attr->inject_size; rxr_ep->mtu_size = rdm_info->ep_attr->max_msg_size; fi_freeinfo(rdm_info); diff --git a/prov/efa/src/rxr/rxr_init.c b/prov/efa/src/rxr/rxr_init.c index 3ac020b8e85..c85431c0b29 100644 --- a/prov/efa/src/rxr/rxr_init.c +++ b/prov/efa/src/rxr/rxr_init.c @@ -317,6 +317,10 @@ static int rxr_info_to_rxr(uint32_t version, const struct fi_info *core_info, *info->ep_attr = *rxr_info.ep_attr; *info->domain_attr = *rxr_info.domain_attr; + /* TODO: update inject_size when we implement inject */ + info->tx_attr->inject_size = 0; + rxr_info.tx_attr->inject_size = info->tx_attr->inject_size; + info->addr_format = core_info->addr_format; info->domain_attr->ep_cnt = core_info->domain_attr->ep_cnt; info->domain_attr->cq_cnt = core_info->domain_attr->cq_cnt; @@ -354,20 +358,6 @@ static int rxr_info_to_rxr(uint32_t version, const struct fi_info *core_info, } } - if (hints->caps & FI_TAGGED) { - info->tx_attr->inject_size = - core_info->tx_attr->inject_size > sizeof(struct rxr_eager_tagrtm_hdr) ? - core_info->tx_attr->inject_size - sizeof(struct rxr_eager_tagrtm_hdr) - : 0; - } else if (hints->caps & FI_MSG) { - info->tx_attr->inject_size = - core_info->tx_attr->inject_size > sizeof(struct rxr_eager_msgrtm_hdr) ? - core_info->tx_attr->inject_size - sizeof(struct rxr_eager_msgrtm_hdr) - : 0; - } - - rxr_info.tx_attr->inject_size = info->tx_attr->inject_size; - /* We only support manual progress for RMA operations */ if (hints->caps & FI_RMA) { info->domain_attr->control_progress = FI_PROGRESS_MANUAL; diff --git a/prov/efa/src/rxr/rxr_msg.c b/prov/efa/src/rxr/rxr_msg.c index fc2727fe922..6fd08fe08b5 100644 --- a/prov/efa/src/rxr/rxr_msg.c +++ b/prov/efa/src/rxr/rxr_msg.c @@ -263,6 +263,8 @@ ssize_t rxr_msg_inject(struct fid_ep *ep, const void *buf, size_t len, msg.addr = dest_addr; rxr_ep = container_of(ep, struct rxr_ep, util_ep.ep_fid.fid); + assert(len <= rxr_ep->core_inject_size - sizeof(struct rxr_eager_msgrtm_hdr)); + return rxr_msg_generic_send(ep, &msg, 0, ofi_op_msg, rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION | FI_INJECT); } @@ -291,6 +293,7 @@ ssize_t rxr_msg_injectdata(struct fid_ep *ep, const void *buf, * source address. This means that we may end up not using the core * providers inject for this send. */ + assert(len <= rxr_ep->core_inject_size - sizeof(struct rxr_eager_msgrtm_hdr)); return rxr_msg_generic_send(ep, &msg, 0, ofi_op_msg, rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION | FI_REMOTE_CQ_DATA | FI_INJECT); @@ -388,6 +391,8 @@ ssize_t rxr_msg_tinject(struct fid_ep *ep_fid, const void *buf, size_t len, msg.addr = dest_addr; rxr_ep = container_of(ep_fid, struct rxr_ep, util_ep.ep_fid.fid); + assert(len <= rxr_ep->core_inject_size - sizeof(struct rxr_eager_tagrtm_hdr)); + return rxr_msg_generic_send(ep_fid, &msg, tag, ofi_op_tagged, rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION | FI_INJECT); } @@ -415,6 +420,8 @@ ssize_t rxr_msg_tinjectdata(struct fid_ep *ep_fid, const void *buf, size_t len, * source address. This means that we may end up not using the core * providers inject for this send. */ + assert(len <= rxr_ep->core_inject_size - sizeof(struct rxr_eager_tagrtm_hdr)); + return rxr_msg_generic_send(ep_fid, &msg, tag, ofi_op_tagged, rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION | FI_REMOTE_CQ_DATA | FI_INJECT);