wip debug

fabtests/functional/rdm_tagged_peek.c

@@ -38,6 +38,10 @@
 
 #include <shared.h>
 
+
+#define BASE_TAG 0x900d
+#define SEND_CNT 8
+
 static struct fi_context fi_context;
 
 static int wait_for_send_comp(int count)
@@ -103,91 +107,177 @@ static int trecv_op(uint64_t tag, uint64_t flags, bool ignore_nomsg)
 	return ret;
 }
 
-static int run(void)
+static int test_bad(void)
 {
-	int i, ret;
+	int ret;
 
-	ret = ft_init_fabric();
-	if (ret)
+	printf("Peek for a bad msg\n");
+	ret = trecv_op(0xbad, FI_PEEK, false);
+	if (ret != -FI_ENOMSG) {
+		FT_PRINTERR("FI_PEEK - bad msg", ret);
 		return ret;
+	}
 
-	if (opts.dst_addr) {
-		printf("Searching for a bad msg\n");
-		ret = trecv_op(0xbad, FI_PEEK, false);
-		if (ret != -FI_ENOMSG) {
-			FT_PRINTERR("FI_PEEK", ret);
-			return ret;
-		}
+	printf("Peek w/ claim for a bad msg\n");
+	ret = trecv_op(0xbad, FI_PEEK | FI_CLAIM, false);
+	if (ret != -FI_ENOMSG) {
+		FT_PRINTERR("FI_PEEK - claim bad msg", ret);
+		return ret;
+	}
 
-		printf("Searching for a bad msg with claim\n");
-		ret = trecv_op(0xbad, FI_PEEK | FI_CLAIM, false);
-		if (ret != -FI_ENOMSG) {
-			FT_PRINTERR("FI_PEEK", ret);
-			return ret;
-		}
+	return 0;
+}
 
-		printf("Searching for first msg\n");
-		ret = trecv_op(0x900d, FI_PEEK, true);
-		if (ret != 1) {
-			FT_PRINTERR("FI_PEEK", ret);
-			return ret;
-		}
+static int test_peek(void)
+{
+	int ret;
 
-		printf("Receiving first msg\n");
-		ret = trecv_op(0x900d, 0, false);
-		if (ret != 1) {
-			FT_PRINTERR("Receive after peek", ret);
-			return ret;
-		}
+	printf("Peek msg 1\n");
+	ret = trecv_op(BASE_TAG + 1, FI_PEEK, true);
+	if (ret != 1) {
+		FT_PRINTERR("FI_PEEK", ret);
+		return ret;
+	}
 
-		printf("Searching for second msg to claim\n");
-		ret = trecv_op(0x900d + 1, FI_PEEK | FI_CLAIM, true);
-		if (ret != 1) {
-			FT_PRINTERR("FI_PEEK | FI_CLAIM", ret);
-			return ret;
-		}
+	printf("Receive msg 1\n");
+	ret = trecv_op(BASE_TAG + 1, 0, false);
+	if (ret != 1) {
+		FT_PRINTERR("Receive after peek", ret);
+		return ret;
+	}
 
-		printf("Receiving second msg\n");
-		ret = trecv_op(0x900d + 1, FI_CLAIM, false);
-		if (ret != 1) {
-			FT_PRINTERR("FI_CLAIM", ret);
-			return ret;
-		}
+	return 0;
+}
 
-		printf("Searching for third msg to peek and discard\n");
-		ret = trecv_op(0x900d + 2, FI_PEEK | FI_DISCARD, true);
-		if (ret != 1) {
-			FT_PRINTERR("FI_PEEK | FI_DISCARD", ret);
-			return ret;
-		}
+static int test_claim(void)
+{
+	int ret;
 
-		printf("Checking to see if third msg was discarded\n");
-		ret = trecv_op(0x900d + 2, FI_PEEK, false);
-		if (ret != -FI_ENOMSG) {
-			FT_PRINTERR("FI_PEEK", ret);
-			return ret;
-		}
+	printf("Peek w/ claim msg 2\n");
+	ret = trecv_op(BASE_TAG + 2, FI_PEEK | FI_CLAIM, true);
+	if (ret != 1) {
+		FT_PRINTERR("FI_PEEK | FI_CLAIM", ret);
+		return ret;
+	}
+
+	printf("Receive claimed msg 2\n");
+	ret = trecv_op(BASE_TAG + 2, FI_CLAIM, false);
+	if (ret != 1) {
+		FT_PRINTERR("FI_CLAIM", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int test_discard(void)
+{
+	int ret;
+
+	printf("Peek & discard msg 3\n");
+	ret = trecv_op(BASE_TAG + 3, FI_PEEK | FI_DISCARD, true);
+	if (ret != 1) {
+		FT_PRINTERR("FI_PEEK | FI_DISCARD", ret);
+		return ret;
+	}
+
+	printf("Checking to see if msg 3 was discarded\n");
+	ret = trecv_op(BASE_TAG + 3, FI_PEEK, false);
+	if (ret != -FI_ENOMSG) {
+		FT_PRINTERR("FI_PEEK", ret);
+		return ret;
+	}
 
-		printf("Searching for fourth msg to claim and discard\n");
-		ret = trecv_op(0x900d + 3, FI_PEEK | FI_CLAIM, true);
+	printf("Peek w/ claim msg 4\n");
+	ret = trecv_op(BASE_TAG + 4, FI_PEEK | FI_CLAIM, true);
+	if (ret != 1) {
+		FT_PRINTERR("FI_DISCARD", ret);
+		return ret;
+	}
+
+	printf("Claim and discard msg 4\n");
+	ret = trecv_op(BASE_TAG + 4, FI_CLAIM | FI_DISCARD, false);
+	if (ret != 1) {
+		FT_PRINTERR("FI_CLAIM", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int test_ooo(void)
+{
+	int i, ret;
+
+	for (i = SEND_CNT; i >= 5; i--) {
+		printf("Receive msg %d\n", i);
+		ret = trecv_op(BASE_TAG + i, 0, false);
 		if (ret != 1) {
-			FT_PRINTERR("FI_DISCARD", ret);
+			FT_PRINTERR("trecv", ret);
 			return ret;
 		}
+	}
 
-		printf("Discarding fourth msg\n");
-		ret = trecv_op(0x900d + 3, FI_CLAIM | FI_DISCARD, false);
-		if (ret != 1) {
-			FT_PRINTERR("FI_CLAIM", ret);
+	return 0;
+}
+
+static int do_recvs(void)
+{
+	int ret;
+
+	ret = test_bad();
+	if (ret)
+		return ret;
+
+	ret = test_peek();
+	if (ret)
+		return ret;
+
+	ret = test_claim();
+	if (ret)
+		return ret;
+
+	ret = test_discard();
+	if (ret)
+		return ret;
+
+	ret = test_ooo();
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int do_sends(void)
+{
+	int i, ret;
+
+	printf("Sending %d tagged messages\n", SEND_CNT);
+	for(i = 1; i <= SEND_CNT; i++) {
+		ret = fi_tsend(ep, tx_buf, tx_size, mr_desc,
+				remote_fi_addr, BASE_TAG + i,
+				&tx_ctx_arr[i].context);
+		if (ret)
 			return ret;
-		}
+	}
 
-		printf("Retrieving fifth message\n");
-		ret = trecv_op(0x900d + 4, 0, false);
-		if (ret != 1) {
-			FT_PRINTERR("Receive after peek", ret);
+	printf("Waiting for messages to complete\n");
+	ret = wait_for_send_comp(SEND_CNT);
+	return ret;
+}
+
+static int run(void)
+{
+	int ret;
+
+	ret = ft_init_fabric();
+	if (ret)
+		return ret;
+
+	if (opts.dst_addr) {
+		ret = do_recvs();
+		if (ret)
 			return ret;
-		}
 
 		/* sync with sender before ft_finalize, since we sent
 		 * and received messages outside of the sequence numbers
@@ -202,16 +292,7 @@ static int run(void)
 		if (ret)
 			return ret;
 	} else {
-		printf("Sending five tagged messages\n");
-		for(i = 0; i < 5; i++) {
-			ret = fi_tsend(ep, tx_buf, tx_size, mr_desc,
-				       remote_fi_addr, 0x900d+i,
-				       &tx_ctx_arr[i].context);
-			if (ret)
-				return ret;
-		}
-		printf("Waiting for messages to complete\n");
-		ret = wait_for_send_comp(5);
+		ret = do_sends();
 		if (ret)
 			return ret;
 
@@ -232,8 +313,9 @@ int main(int argc, char **argv)
 
 	opts = INIT_OPTS;
 	opts.options |= FT_OPT_SIZE;
+	opts.transfer_size = 64;  /* Don't expect receiver buffering */
 	opts.comp_method = FT_COMP_SREAD;
-	opts.window_size = 5;
+	opts.window_size = SEND_CNT;
 
 	hints = fi_allocinfo();
 	if (!hints) {

include/ofi_mem.h

@@ -39,6 +39,7 @@
 #include <assert.h>
 #include <stdlib.h>
 #include <string.h>
+#include <stdio.h>
 #include <ofi.h>
 #include <ofi_list.h>
 #include <ofi_osd.h>
@@ -97,6 +98,44 @@ static inline int ofi_str_dup(const char *src, char **dst)
 	return 0;
 }
 
+static inline void add_char(char **buf, uint8_t c, uint8_t space) {
+    if ( space ) {
+	    sprintf(*buf, " ");
+        *buf += 1;
+    }
+    sprintf (*buf, "%02x", c);
+    *buf += 2;
+}
+
+static inline void ofi_mem_dump(const struct fi_provider *prov,
+	enum fi_log_level level, enum fi_log_subsys subsys, const char *func, int line,
+	const void *src, size_t size)
+{
+	unsigned int width = 32;
+	char temp[128] = {0};
+	char *offset = temp;
+	const char *orig = src;
+	unsigned int i;
+	for (i = 0; i <= size; i++) {
+		if ((i % width) == 0 || i == size) {
+			if (i != 0) {
+				while ((i % width) != 0) {
+					add_char(&offset, 0, (i % 8) == 0);
+					i++;
+				}
+				snprintf(offset++, width, " %s", &orig[i - width]);
+				fi_log(prov, level, subsys, func, line, "%s\n", temp);
+			}
+			offset = &temp[i % width];
+			sprintf(offset, "%08x ", i);
+			offset += 9;
+		}
+		add_char(&offset, orig[i], (i % 8) == 0);
+	}
+}
+
+#define FI_LOG_BUF(prov, level, subsys, buf, len) ofi_mem_dump(prov, level, subsys, __func__, __LINE__, buf, len)
+
 /* Dynamic array -- see ofi_indexer.h */
 
 /*

man/fi_peer.3.md

@@ -10,9 +10,18 @@ tagline: Libfabric Programmer's Manual
 fi_export_fid / fi_import_fid
 : Share a fabric object between different providers or resources
 
+struct fid_peer_av
+: An address vector sharable between independent providers
+
+struct fid_peer_av_set
+: An AV set sharable between independent providers
+
 struct fid_peer_cq
 : A completion queue that may be shared between independent providers
 
+struct fid_peer_srx
+: A shared receive context that may be shared between independent providers
+
 # SYNOPSIS
 
 ```c
@@ -41,6 +50,11 @@ int fi_import_fid(struct fid *fid, struct fid *expfid, uint64_t flags);
 
 # DESCRIPTION
 
+NOTICE: The peer APIs describe by this man page are developmental and may
+change between libfabric versions.  The data structures and API definitions
+should not be considered stable between versions.  Providers being used
+as peers must target the same libfabric version.
+
 Functions defined in this man page are typically used by providers to
 communicate with other providers, known as peer providers, or by other
 libraries to communicate with the libfabric core, known as peer libraries.