RFC: ABD chunk iterator

include/sys/abd_impl.h

@@ -104,6 +104,7 @@ void abd_free_linear_page(abd_t *);
 void abd_iter_init(struct abd_iter  *, abd_t *);
 boolean_t abd_iter_at_end(struct abd_iter *);
 void abd_iter_advance(struct abd_iter *, size_t);
+size_t abd_iter_size(struct abd_iter *aiter);
 void abd_iter_map(struct abd_iter *);
 void abd_iter_unmap(struct abd_iter *);
 void abd_iter_page(struct abd_iter *);
@@ -120,6 +121,187 @@ void abd_iter_page(struct abd_iter *);
 #define	ABD_LINEAR_BUF(abd)	((abd)->abd_u.abd_linear.abd_buf)
 #define	ABD_GANG(abd)		((abd)->abd_u.abd_gang)
 
+/*
+ * Chunk iterators.
+ *
+ * This is a new type of ABD iterator that iterates over data chunks. The idea
+ * is that since ABDs are effectively a wrapper over a set of memory regions,
+ * an iterator that yields data chunks can be smaller and simpler.
+ *
+ * The iterator object abd_chunk_t can be thought of as a pointer to a chunk
+ * within an array of chunks. There are three main functions involved with
+ * setting up and using an iterator:
+ *
+ * - abd_chunk_t ch = abd_chunk_start(abd_t *abd, size_t off, size_t size)
+ *
+ *   Create a new iterator over the given ABD, starting at off bytes, for size
+ *   bytes. If off and size would fall outside of the ABD, the returned
+ *   iterator will be unusable (already "done").
+ *
+ * - void abd_chunk_advance(abd_chunk_t *ch)
+ *
+ *   Move the iterator to the next chunk. If there is no next chunk, the
+ *   iterator is changed to the "done" state and is no longer useable.
+ *
+ * - boolean_t abd_chunk_done(abd_chunk_t *ch)
+ *
+ *   If true, the iterator is pointing to a valid chunk, and the underlying
+ *   memory can be accessed with the access functions. If false, the iterator
+ *   is exhausted and no longer useable.
+ *
+ * Together, these allow an ABD to be processed in a for loop:
+ *
+ *   for (abd_chunk_t ch = abd_chunk_start(abd, off, size);
+ *       !abd_chunk_done(&ch); abd_chunk_advance(&ch))
+ *
+ * With a valid chunk iterator, the following functions can be used to work
+ * with the underlying data or memory:
+ *
+ * - size_t abd_chunk_size(abd_chunk_t *ch)
+ *
+ *   The number of data bytes within the chunk.
+ *
+ * - void *data = abd_chunk_map(abd_chunk_t *ch)
+ *
+ *   Map the memory within the chunk into the address space, and return a
+ *   pointer to the start of the data.
+ *
+ * - void abd_chunk_unmap(abd_chunk_t *ch)
+ *
+ *   Unmap previously-mapped chunk memory. For convenience, if there is nothing
+ *   mapped, nothing happens.
+ */
+
+/* XXX temp exposing old iterator control functions for use in chunk iters */
+abd_t *abd_init_abd_iter(abd_t *abd, struct abd_iter *aiter, size_t off);
+abd_t *abd_advance_abd_iter(abd_t *abd, abd_t *cabd, struct abd_iter *aiter,
+    size_t len);
+
+typedef struct {
+	abd_t		*ch_abd;	/* ABD being iterated over */
+
+	size_t		ch_coff;	/* chunk offset within ABD */
+	size_t		ch_csize;	/* size of chunk within ABD */
+	size_t		ch_doff;	/* data offset within chunk */
+	size_t		ch_dsize;	/* size of data remaining in iter */
+
+	struct abd_iter	ch_iter;	/* XXX old-style iterator */
+	abd_t		*ch_cabd;	/* XXX child abd, for gang iter */
+} abd_chunk_t;
+
+static inline abd_chunk_t
+abd_chunk_start(abd_t *abd, size_t off, size_t size)
+{
+	abd_chunk_t ch = {
+		.ch_abd = abd,
+	};
+
+	if (size == 0 || (off + size > abd_get_size(abd))) {
+		ch.ch_dsize = 0;
+		ch.ch_doff = 0;
+		return (ch);
+	}
+
+	abd_verify(abd);
+	ASSERT3U(off + size, <=, abd->abd_size);
+
+	/* start of data, size of data */
+	ch.ch_doff = off;
+	ch.ch_dsize = size;
+
+	ch.ch_cabd = abd_init_abd_iter(abd, &ch.ch_iter, 0);
+
+	/* size of first chunk */
+	ch.ch_coff = 0;
+	ch.ch_csize = abd_iter_size(&ch.ch_iter);
+
+	/* roll chunks forward until we reach the one with the data start */
+	while (ch.ch_doff >= ch.ch_csize) {
+		ch.ch_doff -= ch.ch_csize;
+		ch.ch_coff += ch.ch_csize;
+
+		ch.ch_cabd = abd_advance_abd_iter(ch.ch_abd, ch.ch_cabd,
+		    &ch.ch_iter, ch.ch_csize);
+
+		ch.ch_csize = abd_iter_size(&ch.ch_iter);
+	}
+
+	return (ch);
+}
+
+static inline void
+abd_chunk_advance(abd_chunk_t *ch)
+{
+	ASSERT3U(ch->ch_dsize, >, 0);
+
+	/* consume data up to the end of the chunk */
+	ch->ch_dsize -= MIN(ch->ch_dsize, ch->ch_csize - ch->ch_doff);
+
+	/* next data will be at the start of the next chunk */
+	ch->ch_doff = 0;
+
+	/* no more data, so return */
+	if (ch->ch_dsize == 0)
+		return;
+
+	ch->ch_cabd = abd_advance_abd_iter(ch->ch_abd, ch->ch_cabd,
+	    &ch->ch_iter, ch->ch_csize);
+
+	ch->ch_coff += ch->ch_csize;
+	ch->ch_csize = abd_iter_size(&ch->ch_iter);
+}
+
+static inline boolean_t
+abd_chunk_done(abd_chunk_t *ch)
+{
+	return (ch->ch_dsize == 0);
+}
+
+static inline size_t
+abd_chunk_size(abd_chunk_t *ch)
+{
+	return (MIN(ch->ch_dsize, ch->ch_csize - ch->ch_doff));
+}
+
+static inline void *
+abd_chunk_map(abd_chunk_t *ch) {
+	abd_iter_map(&ch->ch_iter);
+	ASSERT3U(ch->ch_iter.iter_mapsize, ==, ch->ch_csize);
+	return (ch->ch_iter.iter_mapaddr + ch->ch_doff);
+}
+
+static inline void
+abd_chunk_unmap(abd_chunk_t *ch) {
+	if (ch->ch_iter.iter_mapaddr == NULL)
+		return;
+	abd_iter_unmap(&ch->ch_iter);
+}
+
+/*
+ * Macro to iterate over an ABD in the most common case, where you want to
+ * do some operation on the actual data. Safe to break out early.
+ *
+ * data and dsize are the name of the buffer and size variables to use inside
+ * the loop.
+ *
+ *     abd_for_each_chunk(abd, off, size, data, dsize, {
+ *         // do work on data an dsize
+ *     });
+ */
+#define	abd_for_each_chunk(abd, off, size, __data, __dsize, __code)	\
+	do {								\
+		void *__data;						\
+		size_t __dsize;						\
+		abd_chunk_t __ch = abd_chunk_start(abd, off, size);	\
+		for (; !abd_chunk_done(&__ch) &&			\
+		    (__data = abd_chunk_map(&__ch)) &&			\
+		    (__dsize = abd_chunk_size(&__ch));			\
+		    abd_chunk_unmap(&__ch), abd_chunk_advance(&__ch)) {	\
+			__code;						\
+		}							\
+		abd_chunk_unmap(&__ch);					\
+	} while (0)
+
 #ifdef __cplusplus
 }
 #endif

lib/libzpool/abd_os.c

@@ -311,6 +311,38 @@ abd_iter_advance(struct abd_iter *aiter, size_t amount)
 	ASSERT3U(aiter->iter_pos, <=, aiter->iter_abd->abd_size);
 }
 
+/* Data in current linear chunk is always just distance to end of ABD */
+#define	ABD_ITER_LINEAR_SIZE(aiter) \
+	((aiter)->iter_abd->abd_size - (aiter)->iter_pos)
+
+/*
+ * Data in current scatter chunk is always distance to end of chunk, or in
+ * the last chunk, distance to end of ABD
+ */
+#define	ABD_ITER_SCATTER_SIZE(aiter)					\
+	(MIN(ABD_PAGESIZE - (((aiter)->iter_pos +			\
+	    ABD_SCATTER((aiter)->iter_abd).abd_offset) & ABD_PAGEMASK),	\
+	    (aiter)->iter_abd->abd_size - (aiter)->iter_pos))		\
+
+/*
+ * Return size of data in current chunk (_not_ underlying memory size).
+ */
+size_t
+abd_iter_size(struct abd_iter *aiter)
+{
+	/* If it's already mapped, just return that size. */
+	if (aiter->iter_mapsize > 0)
+		return (aiter->iter_mapsize);
+
+	/* There's no current chunk if the iterator is exhausted. */
+	if (abd_iter_at_end(aiter))
+		return (0);
+
+	if (abd_is_linear(aiter->iter_abd))
+		return (ABD_ITER_LINEAR_SIZE(aiter));
+	return (ABD_ITER_SCATTER_SIZE(aiter));
+}
+
 void
 abd_iter_map(struct abd_iter *aiter)
 {
@@ -323,8 +355,7 @@ abd_iter_map(struct abd_iter *aiter)
 	if (abd_is_linear(aiter->iter_abd)) {
 		aiter->iter_mapaddr =
 		    ABD_LINEAR_BUF(aiter->iter_abd) + aiter->iter_pos;
-		aiter->iter_mapsize =
-		    aiter->iter_abd->abd_size - aiter->iter_pos;
+		aiter->iter_mapsize = ABD_ITER_LINEAR_SIZE(aiter);
 		return;
 	}
 
@@ -342,6 +373,7 @@ abd_iter_map(struct abd_iter *aiter)
 	aiter->iter_mapsize = MIN(ABD_PAGESIZE - (poff & ABD_PAGEMASK),
 	    aiter->iter_abd->abd_size - aiter->iter_pos);
 	ASSERT3U(aiter->iter_mapsize, <=, ABD_PAGESIZE);
+	ASSERT3U(aiter->iter_mapsize, ==, ABD_ITER_SCATTER_SIZE(aiter));
 
 	aiter->iter_mapaddr = iov->iov_base + (poff & ABD_PAGEMASK);
 }

module/os/linux/zfs/abd_os.c

@@ -890,6 +890,36 @@ abd_iter_advance(struct abd_iter *aiter, size_t amount)
 	}
 }
 
+/* Data in current linear chunk is always just offset to end of ABD */
+#define	ABD_ITER_LINEAR_SIZE(aiter) \
+	((aiter)->iter_abd->abd_size - (aiter)->iter_offset)
+
+/*
+ * Data in current scatter chunk is always distance to end of chunk, or in
+ * the last chunk, distance to end of ABD
+ */
+#define	ABD_ITER_SCATTER_SIZE(aiter)				\
+	(MIN((aiter)->iter_sg->length - (aiter)->iter_offset,	\
+	    (aiter)->iter_abd->abd_size - (aiter)->iter_pos))
+/*
+ * Return size of data in current chunk (_not_ underlying memory size).
+ */
+size_t
+abd_iter_size(struct abd_iter *aiter)
+{
+	/* If it's already mapped, just return that size. */
+	if (aiter->iter_mapsize > 0)
+		return (aiter->iter_mapsize);
+
+	/* There's no current chunk if the iterator is exhausted. */
+	if (abd_iter_at_end(aiter))
+		return (0);
+
+	if (abd_is_linear(aiter->iter_abd))
+		return (ABD_ITER_LINEAR_SIZE(aiter));
+	return (ABD_ITER_SCATTER_SIZE(aiter));
+}
+
 /*
  * Map the current chunk into aiter. This can be safely called when the aiter
  * has already exhausted, in which case this does nothing.
@@ -898,7 +928,6 @@ void
 abd_iter_map(struct abd_iter *aiter)
 {
 	void *paddr;
-	size_t offset = 0;
 
 	ASSERT3P(aiter->iter_mapaddr, ==, NULL);
 	ASSERT0(aiter->iter_mapsize);
@@ -909,18 +938,14 @@ abd_iter_map(struct abd_iter *aiter)
 
 	if (abd_is_linear(aiter->iter_abd)) {
 		ASSERT3U(aiter->iter_pos, ==, aiter->iter_offset);
-		offset = aiter->iter_offset;
-		aiter->iter_mapsize = aiter->iter_abd->abd_size - offset;
+		aiter->iter_mapsize = ABD_ITER_LINEAR_SIZE(aiter);
 		paddr = ABD_LINEAR_BUF(aiter->iter_abd);
 	} else {
-		offset = aiter->iter_offset;
-		aiter->iter_mapsize = MIN(aiter->iter_sg->length - offset,
-		    aiter->iter_abd->abd_size - aiter->iter_pos);
-
+		aiter->iter_mapsize = ABD_ITER_SCATTER_SIZE(aiter);
 		paddr = zfs_kmap_local(sg_page(aiter->iter_sg));
 	}
 
-	aiter->iter_mapaddr = (char *)paddr + offset;
+	aiter->iter_mapaddr = (char *)paddr + aiter->iter_offset;
 }
 
 /*