prov/efa: Advertise the correct inject_size

Commit 717f14b retired the RTS packet and set the inject_size to zero. Commmit 5aa0fae started inlining sends based on the inline_buf_size we get from efadv_query_device(). With the completions already squashed for fi_(t)inject(), we can advertise the correct size based on the capabilities requested and the corresponding largest header type for eager messages. Signed-off-by: Raghu Raja <[email protected]>
shefty · May 19, 2020 · 42cb642 · 42cb642
1 parent 4f4704c
commit 42cb642
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 13 deletions.
diff --git a/prov/efa/src/rxr/rxr.h b/prov/efa/src/rxr/rxr.h
@@ -535,7 +535,6 @@ struct rxr_ep {
 	/* rx/tx queue size of core provider */
 	size_t core_rx_size;
 	size_t max_outstanding_tx;
-	size_t core_inject_size;
 	size_t max_data_payload_size;
 
 	/* Resource management flag */

diff --git a/prov/efa/src/rxr/rxr_ep.c b/prov/efa/src/rxr/rxr_ep.c
@@ -1668,7 +1668,6 @@ int rxr_endpoint(struct fid_domain *domain, struct fi_info *info,
 	assert(info->tx_attr->msg_order == info->rx_attr->msg_order);
 	rxr_ep->msg_order = info->rx_attr->msg_order;
 	rxr_ep->core_msg_order = rdm_info->rx_attr->msg_order;
-	rxr_ep->core_inject_size = rdm_info->tx_attr->inject_size;
 	rxr_ep->mtu_size = rdm_info->ep_attr->max_msg_size;
 	if (rxr_env.mtu_size > 0 && rxr_env.mtu_size < rxr_ep->mtu_size)
 		rxr_ep->mtu_size = rxr_env.mtu_size;

diff --git a/prov/efa/src/rxr/rxr_init.c b/prov/efa/src/rxr/rxr_init.c
@@ -317,10 +317,6 @@ static int rxr_info_to_rxr(uint32_t version, const struct fi_info *core_info,
 	*info->ep_attr = *rxr_info.ep_attr;
 	*info->domain_attr = *rxr_info.domain_attr;
 
-	/* TODO: update inject_size when we implement inject */
-	info->tx_attr->inject_size = 0;
-	rxr_info.tx_attr->inject_size = info->tx_attr->inject_size;
-
 	info->addr_format = core_info->addr_format;
 	info->domain_attr->ep_cnt = core_info->domain_attr->ep_cnt;
 	info->domain_attr->cq_cnt = core_info->domain_attr->cq_cnt;
@@ -358,6 +354,20 @@ static int rxr_info_to_rxr(uint32_t version, const struct fi_info *core_info,
 			}
 		}
 
+		if (hints->caps & FI_TAGGED) {
+			info->tx_attr->inject_size =
+				core_info->tx_attr->inject_size > sizeof(struct rxr_eager_tagrtm_hdr) ?
+				core_info->tx_attr->inject_size - sizeof(struct rxr_eager_tagrtm_hdr)
+				: 0;
+		} else if (hints->caps & FI_MSG) {
+			info->tx_attr->inject_size =
+				core_info->tx_attr->inject_size > sizeof(struct rxr_eager_msgrtm_hdr) ?
+				core_info->tx_attr->inject_size - sizeof(struct rxr_eager_msgrtm_hdr)
+				: 0;
+		}
+
+		rxr_info.tx_attr->inject_size = info->tx_attr->inject_size;
+
 		/* We only support manual progress for RMA operations */
 		if (hints->caps & FI_RMA) {
 			info->domain_attr->control_progress = FI_PROGRESS_MANUAL;

diff --git a/prov/efa/src/rxr/rxr_msg.c b/prov/efa/src/rxr/rxr_msg.c
@@ -263,8 +263,6 @@ ssize_t rxr_msg_inject(struct fid_ep *ep, const void *buf, size_t len,
 	msg.addr = dest_addr;
 
 	rxr_ep = container_of(ep, struct rxr_ep, util_ep.ep_fid.fid);
-	assert(len <= rxr_ep->core_inject_size - sizeof(struct rxr_eager_msgrtm_hdr));
-
 	return rxr_msg_generic_send(ep, &msg, 0, ofi_op_msg,
 				    rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION | FI_INJECT);
 }
@@ -293,7 +291,6 @@ ssize_t rxr_msg_injectdata(struct fid_ep *ep, const void *buf,
 	 * source address. This means that we may end up not using the core
 	 * providers inject for this send.
 	 */
-	assert(len <= rxr_ep->core_inject_size - sizeof(struct rxr_eager_msgrtm_hdr));
 	return rxr_msg_generic_send(ep, &msg, 0, ofi_op_msg,
 				    rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION |
 				    FI_REMOTE_CQ_DATA | FI_INJECT);
@@ -391,8 +388,6 @@ ssize_t rxr_msg_tinject(struct fid_ep *ep_fid, const void *buf, size_t len,
 	msg.addr = dest_addr;
 
 	rxr_ep = container_of(ep_fid, struct rxr_ep, util_ep.ep_fid.fid);
-	assert(len <= rxr_ep->core_inject_size - sizeof(struct rxr_eager_tagrtm_hdr));
-
 	return rxr_msg_generic_send(ep_fid, &msg, tag, ofi_op_tagged,
 				    rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION | FI_INJECT);
 }
@@ -420,8 +415,6 @@ ssize_t rxr_msg_tinjectdata(struct fid_ep *ep_fid, const void *buf, size_t len,
 	 * source address. This means that we may end up not using the core
 	 * providers inject for this send.
 	 */
-	assert(len <= rxr_ep->core_inject_size - sizeof(struct rxr_eager_tagrtm_hdr));
-
 	return rxr_msg_generic_send(ep_fid, &msg, tag, ofi_op_tagged,
 				    rxr_tx_flags(rxr_ep) | RXR_NO_COMPLETION |
 				    FI_REMOTE_CQ_DATA | FI_INJECT);