Revert "prov/efa: Advertise the correct inject_size"

This change is causing performance regressions with certain codes. Reverting it till we can root cause and fix it. This reverts commit 42cb642. Signed-off-by: Raghu Raja <[email protected]>
shefty · May 26, 2020 · 0ecc833 · 0ecc833
1 parent 6ec0bae
commit 0ecc833
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 14 deletions.
diff --git a/prov/efa/src/rxr/rxr.h b/prov/efa/src/rxr/rxr.h
@@ -535,6 +535,7 @@ struct rxr_ep {
 	/* rx/tx queue size of core provider */
 	size_t core_rx_size;
 	size_t max_outstanding_tx;
+	size_t core_inject_size;
 	size_t max_data_payload_size;
 
 	/* Resource management flag */

diff --git a/prov/efa/src/rxr/rxr_ep.c b/prov/efa/src/rxr/rxr_ep.c
@@ -1673,6 +1673,7 @@ int rxr_endpoint(struct fid_domain *domain, struct fi_info *info,
 	assert(info->tx_attr->msg_order == info->rx_attr->msg_order);
 	rxr_ep->msg_order = info->rx_attr->msg_order;
 	rxr_ep->core_msg_order = rdm_info->rx_attr->msg_order;
+	rxr_ep->core_inject_size = rdm_info->tx_attr->inject_size;
 	rxr_ep->mtu_size = rdm_info->ep_attr->max_msg_size;
 	fi_freeinfo(rdm_info);
 

diff --git a/prov/efa/src/rxr/rxr_init.c b/prov/efa/src/rxr/rxr_init.c
@@ -317,6 +317,10 @@ static int rxr_info_to_rxr(uint32_t version, const struct fi_info *core_info,
 	*info->ep_attr = *rxr_info.ep_attr;
 	*info->domain_attr = *rxr_info.domain_attr;
 
+	/* TODO: update inject_size when we implement inject */
+	info->tx_attr->inject_size = 0;
+	rxr_info.tx_attr->inject_size = info->tx_attr->inject_size;
+
 	info->addr_format = core_info->addr_format;
 	info->domain_attr->ep_cnt = core_info->domain_attr->ep_cnt;
 	info->domain_attr->cq_cnt = core_info->domain_attr->cq_cnt;
@@ -354,20 +358,6 @@ static int rxr_info_to_rxr(uint32_t version, const struct fi_info *core_info,
 			}
 		}
 
-		if (hints->caps & FI_TAGGED) {
-			info->tx_attr->inject_size =
-				core_info->tx_attr->inject_size > sizeof(struct rxr_eager_tagrtm_hdr) ?
-				core_info->tx_attr->inject_size - sizeof(struct rxr_eager_tagrtm_hdr)
-				: 0;
-		} else if (hints->caps & FI_MSG) {
-			info->tx_attr->inject_size =
-				core_info->tx_attr->inject_size > sizeof(struct rxr_eager_msgrtm_hdr) ?
-				core_info->tx_attr->inject_size - sizeof(struct rxr_eager_msgrtm_hdr)
-				: 0;
-		}
-
-		rxr_info.tx_attr->inject_size = info->tx_attr->inject_size;
-
 		/* We only support manual progress for RMA operations */
 		if (hints->caps & FI_RMA) {
 			info->domain_attr->control_progress = FI_PROGRESS_MANUAL;

diff --git a/prov/efa/src/rxr/rxr_msg.c b/prov/efa/src/rxr/rxr_msg.c
@@ -263,6 +263,8 @@ ssize_t rxr_msg_inject(struct fid_ep *ep, const void *buf, size_t len,
 	msg.addr = dest_addr;
 
 	rxr_ep = container_of(ep, struct rxr_ep, util_ep.ep_fid.fid);
+	assert(len <= rxr_ep->core_inject_size - sizeof(struct rxr_eager_msgrtm_hdr));
+
 	return rxr_msg_generic_send(ep, &msg, 0, ofi_op_msg,
 				    rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION | FI_INJECT);
 }
@@ -291,6 +293,7 @@ ssize_t rxr_msg_injectdata(struct fid_ep *ep, const void *buf,
 	 * source address. This means that we may end up not using the core
 	 * providers inject for this send.
 	 */
+	assert(len <= rxr_ep->core_inject_size - sizeof(struct rxr_eager_msgrtm_hdr));
 	return rxr_msg_generic_send(ep, &msg, 0, ofi_op_msg,
 				    rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION |
 				    FI_REMOTE_CQ_DATA | FI_INJECT);
@@ -388,6 +391,8 @@ ssize_t rxr_msg_tinject(struct fid_ep *ep_fid, const void *buf, size_t len,
 	msg.addr = dest_addr;
 
 	rxr_ep = container_of(ep_fid, struct rxr_ep, util_ep.ep_fid.fid);
+	assert(len <= rxr_ep->core_inject_size - sizeof(struct rxr_eager_tagrtm_hdr));
+
 	return rxr_msg_generic_send(ep_fid, &msg, tag, ofi_op_tagged,
 				    rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION | FI_INJECT);
 }
@@ -415,6 +420,8 @@ ssize_t rxr_msg_tinjectdata(struct fid_ep *ep_fid, const void *buf, size_t len,
 	 * source address. This means that we may end up not using the core
 	 * providers inject for this send.
 	 */
+	assert(len <= rxr_ep->core_inject_size - sizeof(struct rxr_eager_tagrtm_hdr));
+
 	return rxr_msg_generic_send(ep_fid, &msg, tag, ofi_op_tagged,
 				    rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION |
 				    FI_REMOTE_CQ_DATA | FI_INJECT);