rpma: add rpma_mr_advise()

doc/manuals_3.txt

@@ -38,6 +38,7 @@ rpma_flush.3
 rpma_log_get_threshold.3
 rpma_log_set_function.3
 rpma_log_set_threshold.3
+rpma_mr_advise.3
 rpma_mr_dereg.3
 rpma_mr_get_descriptor.3
 rpma_mr_get_descriptor_size.3

src/include/librpma.h

@@ -1146,6 +1146,45 @@ int rpma_mr_remote_get_flush_type(const struct rpma_mr_remote *mr,
  */
 int rpma_mr_remote_delete(struct rpma_mr_remote **mr_ptr);
 
+/** 3
+ * rpma_mr_advise - give advice about an address range in a memory registration
+ *
+ * SYNOPSIS
+ *
+ *	#include <librpma.h>
+ *
+ *	struct rpma_mr_local *mr;
+ *	int rpma_mr_advise(struct rpma_mr_local *mr, size_t offset, size_t len,
+ *		enum ibv_advise_mr_advice advice, uint32_t flags);
+ *
+ * DESCRIPTION
+ * rpma_mr_advise() give advice about an address range in a memory
+ * registration. The usage parameter specifies the operations that can be
+ * performed on the given memory address range.
+ * the advice
+ * - IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT - prefetching without prefaulting
+ * - IBV_ADVISE_MR_ADVICE_PREFETCH - read-only access
+ * - IBV_ADVISE_MR_ADVICE_PREFETCH_WRITE - read and write access
+ *
+ * the flags
+ * - IBV_ADVISE_MR_FLAG_FLUSH - the underlying pages are guaranteed to be
+ *   updated in the HCA before returning SUCCESS
+ *
+ * RETURN VALUE
+ * The rpma_mr_advise() function returns 0 on success or a negative error code
+ * on failure.
+ *
+ * ERRORS
+ * rpma_mr_advise() can fail with the following errors:
+ *
+ * - RPMA_E_PROVIDER - memory registration failed
+ *
+ * SEE ALSO
+ * rpma_mr_reg(3), librpma(7) and https://pmem.io/rpma/
+ */
+int rpma_mr_advise(struct rpma_mr_local *mr, size_t offset, size_t len,
+		enum ibv_advise_mr_advice advice, uint32_t flags);
+
 /* connection configuration */
 
 struct rpma_conn_cfg;

src/mr.c

@@ -510,3 +510,24 @@ rpma_mr_remote_get_flush_type(const struct rpma_mr_remote *mr, int *flush_type)
 
 	return 0;
 }
+
+/*
+ * rpma_mr_advise -- give advice about an address range in a memory registration
+ */
+int
+rpma_mr_advise(struct rpma_mr_local *mr, size_t offset, size_t len,
+		enum ibv_advise_mr_advice advice, uint32_t flags)
+{
+	struct ibv_sge sg_list;
+	sg_list.lkey = mr->ibv_mr->lkey;
+	sg_list.addr = (uint64_t)((uintptr_t)mr->ibv_mr->addr + offset);
+	sg_list.length = len;
+
+	int ret = ibv_advise_mr(mr->ibv_mr->pd, advice, flags, &sg_list, 1);
+	if (ret) {
+		RPMA_LOG_ERROR_WITH_ERRNO(ret, "ibv_advise_mr");
+		return RPMA_E_PROVIDER;
+	}
+
+	return 0;
+}

tests/unit/common/mocks-ibverbs.c

@@ -347,3 +347,31 @@ ibv_wc_status_str(enum ibv_wc_status status)
 {
 	return "";
 }
+
+/*
+ * ibv_post_send_mock -- mock of ibv_advise_mr()
+ */
+int
+ibv_advise_mr_mock(struct ibv_pd *pd,
+				enum ibv_advise_mr_advice advice,
+				uint32_t flags,
+				struct ibv_sge *sg_list,
+				uint32_t num_sge)
+{
+	struct ibv_advise_mr_mock_args *args =
+		mock_type(struct ibv_advise_mr_mock_args *);
+
+	assert_non_null(pd);
+	assert_non_null(sg_list);
+
+	assert_int_equal(pd, args->pd);
+	assert_int_equal(advice, args->advice);
+	assert_int_equal(flags, args->flags);
+
+	assert_int_equal(sg_list->lkey, args->lkey);
+	assert_int_equal(sg_list->addr, args->local_addr);
+	assert_int_equal(sg_list->length, args->length);
+	assert_int_equal(num_sge, args->num_sge);
+
+	return args->ret;
+}

tests/unit/common/mocks-ibverbs.h

@@ -58,6 +58,17 @@ struct ibv_post_recv_mock_args {
 	int ret;
 };
 
+struct ibv_advise_mr_mock_args {
+	struct ibv_pd *pd;
+	enum ibv_advise_mr_advice advice;
+	uint32_t flags;
+	uint64_t local_addr;
+	size_t length;
+	uint32_t lkey;
+	uint32_t num_sge;
+	int ret;
+};
+
 #ifdef ON_DEMAND_PAGING_SUPPORTED
 int ibv_query_device_ex_mock(struct ibv_context *context,
 		const struct ibv_query_device_ex_input *input,
@@ -73,4 +84,7 @@ int ibv_post_recv_mock(struct ibv_qp *qp, struct ibv_recv_wr *wr,
 
 int ibv_req_notify_cq_mock(struct ibv_cq *cq, int solicited_only);
 
+int ibv_advise_mr_mock(struct ibv_pd *pd, enum ibv_advise_mr_advice advice,
+		uint32_t flags, struct ibv_sge *sg_list, uint32_t num_sge);
+
 #endif /* MOCKS_IBVERBS_H */

tests/unit/mr/CMakeLists.txt

@@ -1,6 +1,6 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2020, Intel Corporation
+# Copyright 2020-2021, Intel Corporation
 #
 
 include(../../cmake/ctest_helpers.cmake)
@@ -26,6 +26,7 @@ function(add_test_mr name)
 	add_test_generic(NAME ${name} TRACERS none)
 endfunction()
 
+add_test_mr(advise)
 add_test_mr(descriptor)
 add_test_mr(get_flush_type)
 add_test_mr(local)

tests/unit/mr/mr-advise.c

@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/* Copyright 2021, Intel Corporation */
+
+/*
+ * mr-advise.c -- rpma_mr_advise() unit tests
+ */
+
+#include <infiniband/verbs.h>
+#include <stdlib.h>
+
+#include "cmocka_headers.h"
+#include "mr.h"
+#include "librpma.h"
+
+#include "mr-common.h"
+#include "mocks-ibverbs.h"
+#include "test-common.h"
+
+/*
+ * advise__failed_E_PROVIDER - rpma_mr_advise failed
+ * with RPMA_E_PROVIDER when length > ibv_mr->length
+ */
+static void
+advise__failed_E_PROVIDER(void **mrs_ptr)
+{
+	struct mrs *mrs = (struct mrs *)*mrs_ptr;
+
+	/* configure mocks */
+	struct ibv_advise_mr_mock_args args;
+	args.pd = MOCK_IBV_PD;
+	args.advice = IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE;
+	args.flags = IB_UVERBS_ADVISE_MR_FLAG_FLUSH;
+	args.local_addr = MOCK_LADDR + MOCK_SRC_OFFSET;
+	args.length = MOCK_LEN;
+	args.lkey = MOCK_LKEY;
+	args.num_sge = 1;
+	args.ret = RPMA_E_PROVIDER;
+	will_return(ibv_advise_mr_mock, &args);
+
+	/* run test */
+	int ret = rpma_mr_advise(mrs->local, MOCK_SRC_OFFSET, MOCK_LEN,
+				IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE,
+				IB_UVERBS_ADVISE_MR_FLAG_FLUSH);
+
+
+	/* verify the results */
+	assert_int_equal(ret, RPMA_E_PROVIDER);
+}
+
+/*
+ * read__success - happy day scenario
+ */
+static void
+advise__success(void **mrs_ptr)
+{
+	struct mrs *mrs = (struct mrs *)*mrs_ptr;
+
+	/* configure mocks */
+	struct ibv_advise_mr_mock_args args;
+	args.pd = MOCK_IBV_PD;
+	args.advice = IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE;
+	args.flags = IB_UVERBS_ADVISE_MR_FLAG_FLUSH;
+	args.local_addr = MOCK_LADDR + MOCK_SRC_OFFSET;
+	args.length = MOCK_LEN;
+	args.lkey = MOCK_LKEY;
+	args.num_sge = 1;
+	args.ret = MOCK_OK;
+	will_return(ibv_advise_mr_mock, &args);
+
+	/* run test */
+	int ret = rpma_mr_advise(mrs->local, MOCK_SRC_OFFSET, MOCK_LEN,
+				IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE,
+				IB_UVERBS_ADVISE_MR_FLAG_FLUSH);
+
+	/* verify the results */
+	assert_int_equal(ret, MOCK_OK);
+}
+
+/*
+ * group_setup_mr_advise -- prepare resources for all tests in the group
+ */
+static int
+group_setup_mr_advise(void **unused)
+{
+	/* configure global mocks */
+
+	/*
+	 * ibv_advise_mr() is defined as a static inline function
+	 * in the included header <infiniband/verbs.h>,
+	 * so we cannot define it again. It is defined as:
+	 * {
+	 *		return (struct verbs_context *)(((uint8_t *)ctx)
+	 *		- offsetof(struct verbs_context, context))
+	 *		->advise_mr(pd, advice, flags, sg_list, num_sge);
+	 * }
+	 * so we can set the advise_mr function pointer to our mock function.
+	 */
+	Verbs_context.advise_mr = ibv_advise_mr_mock;
+	Verbs_context.sz = sizeof(Verbs_context);
+
+	Ibv_pd.context = (struct ibv_context *)((uint8_t *)&Verbs_context +
+		offsetof(struct verbs_context, context));
+	Ibv_pd.context->abi_compat = __VERBS_ABI_IS_EXTENDED;
+
+	Ibv_mr.lkey = MOCK_LKEY;
+	Ibv_mr.pd = MOCK_IBV_PD;
+
+	return 0;
+}
+
+static const struct CMUnitTest test_mr_advise[] = {
+	/* rpma_mr_adivse() unit tests */
+	cmocka_unit_test_setup_teardown(advise__failed_E_PROVIDER,
+			setup__mr_local_and_remote,
+			teardown__mr_local_and_remote),
+	cmocka_unit_test_setup_teardown(advise__success,
+			setup__mr_local_and_remote,
+			teardown__mr_local_and_remote),
+	cmocka_unit_test(NULL)
+};
+
+int
+main(int argc, char *argv[])
+{
+	return cmocka_run_group_tests(test_mr_advise,
+		group_setup_mr_advise, NULL);
+}

tests/unit/mr/mr-common.h

@@ -48,6 +48,9 @@
 #define MOCK_RADDR		(uint64_t)0x0001020304050607
 #define MOCK_RKEY		(uint32_t)0x10111213
 
+#define MOCK_LADDR		(uint64_t)0x0001020304050607
+#define MOCK_LKEY		(uint32_t)0x20212223
+
 /* a state used for rpma_mr_read/_write tests */
 struct mrs {
 	struct rpma_mr_local *local;