Merge lookasidelist and kstat

include/os/windows/spl/sys/lookasidelist.h

@@ -0,0 +1,27 @@
+#ifndef _SPL_LOOKASIDELIST_H
+#define	_SPL_LOOKASIDELIST_H
+
+extern void* osif_malloc(uint64_t);
+extern void osif_free(void*, uint64_t);
+
+#define	ZFS_LookAsideList_DRV_TAG '!SFZ'
+
+#define	LOOKASIDELIST_CACHE_NAMELEN 31
+
+typedef struct lookasidelist_cache {
+    uint64_t cache_active_allocations;
+    uint64_t total_alloc;
+    uint64_t total_free;
+    size_t cache_chunksize; // cache object size
+    kstat_t *cache_kstat;
+    char    cache_name[LOOKASIDELIST_CACHE_NAMELEN + 1];
+    LOOKASIDE_LIST_EX lookasideField;
+} lookasidelist_cache_t;
+
+lookasidelist_cache_t *lookasidelist_cache_create(char *name, size_t size);
+void lookasidelist_cache_destroy(lookasidelist_cache_t *pLookasidelist_cache);
+void* lookasidelist_cache_alloc(lookasidelist_cache_t *pLookasidelist_cache);
+void lookasidelist_cache_free(lookasidelist_cache_t *pLookasidelist_cache,
+    void *buf);
+
+#endif

module/os/windows/spl/CMakeLists.txt

@@ -37,6 +37,7 @@ wdk_add_library(splkern
   spl-vnode.c
   spl-windows.c
   spl-xdr.c
+  spl-lookasidelist.c
   ${TMH_FILE_LIST}
 )
 

module/os/windows/spl/spl-lookasidelist.c

@@ -0,0 +1,165 @@
+#include <sys/zfs_context.h>
+#include <sys/lookasidelist.h>
+
+typedef struct lookasidelist_stats {
+    kstat_named_t lookasidestat_active_alloc;
+    kstat_named_t lookasidestat_total_alloc;
+    kstat_named_t lookasidestat_total_free;
+    kstat_named_t lookasidestat_chunk_size;
+} lookasidelist_stats_t;
+
+static lookasidelist_stats_t lookasidelist_stats = {
+	/* Number of active allocations */
+	{ "active_alloc",   KSTAT_DATA_UINT64 },
+	/* The total number of allocations */
+	{ "total_alloc",    KSTAT_DATA_UINT64 },
+	/* The total number of frees */
+	{ "total_free",	    KSTAT_DATA_UINT64 },
+	/* The size of each object/chunk in lookaside list */
+	{ "chunk_size",	    KSTAT_DATA_UINT64 },
+};
+
+static int
+lookaside_kstat_update(kstat_t *ksp, int rw)
+{
+	lookasidelist_stats_t *ks = ksp->ks_data;
+	lookasidelist_cache_t *cp = ksp->ks_private;
+
+	if (rw == KSTAT_WRITE) {
+		return (EACCES);
+	}
+
+	ks->lookasidestat_active_alloc.value.ui64 =
+	    cp->cache_active_allocations;
+	ks->lookasidestat_total_alloc.value.ui64 =
+	    cp->total_alloc;
+	ks->lookasidestat_total_free.value.ui64 =
+	    cp->total_free;
+	ks->lookasidestat_chunk_size.value.ui64 =
+	    cp->cache_chunksize;
+
+	return (0);
+}
+
+void *
+allocate_func(
+    __in POOL_TYPE  PoolType,
+    __in SIZE_T  NumberOfBytes,
+    __in ULONG  Tag,
+    __inout PLOOKASIDE_LIST_EX  Lookaside)
+{
+	lookasidelist_cache_t *pLookasidelist_cache;
+	pLookasidelist_cache = CONTAINING_RECORD(Lookaside,
+	    lookasidelist_cache_t, lookasideField);
+	void *buf = osif_malloc(NumberOfBytes);
+	ASSERT(buf != NULL);
+
+	if (buf != NULL) {
+		atomic_inc_64(&pLookasidelist_cache->cache_active_allocations);
+		atomic_inc_64(&pLookasidelist_cache->total_alloc);
+	}
+
+	return (buf);
+}
+
+void free_func(
+    __in PVOID  Buffer,
+    __inout PLOOKASIDE_LIST_EX  Lookaside
+) {
+	lookasidelist_cache_t *pLookasidelist_cache;
+	pLookasidelist_cache = CONTAINING_RECORD(Lookaside,
+	    lookasidelist_cache_t, lookasideField);
+	osif_free(Buffer, pLookasidelist_cache->cache_chunksize);
+	atomic_dec_64(&pLookasidelist_cache->cache_active_allocations);
+	atomic_inc_64(&pLookasidelist_cache->total_free);
+}
+
+lookasidelist_cache_t *
+lookasidelist_cache_create(char *name,	/* descriptive name for this cache */
+    size_t size) /* size of the objects it manages */
+{
+	lookasidelist_cache_t *pLookasidelist_cache;
+	pLookasidelist_cache = ExAllocatePoolWithTag(NonPagedPool,
+	    sizeof (lookasidelist_cache_t), ZFS_LookAsideList_DRV_TAG);
+
+	if (pLookasidelist_cache != NULL) {
+		bzero(pLookasidelist_cache, sizeof (lookasidelist_cache_t));
+		pLookasidelist_cache->cache_chunksize = size;
+
+		if (name != NULL) {
+			strncpy(pLookasidelist_cache->cache_name, name,
+			    LOOKASIDELIST_CACHE_NAMELEN);
+		}
+
+		NTSTATUS retval = ExInitializeLookasideListEx(
+		    &pLookasidelist_cache->lookasideField,
+		    allocate_func,
+		    free_func,
+		    NonPagedPoolNx,
+		    0,
+		    size,
+		    ZFS_LookAsideList_DRV_TAG,
+		    0);
+
+		ASSERT(retval == STATUS_SUCCESS);
+
+		if (retval != STATUS_SUCCESS) {
+			ExFreePoolWithTag(pLookasidelist_cache,
+			    ZFS_LookAsideList_DRV_TAG);
+			pLookasidelist_cache = NULL;
+		}
+	}
+
+	if (pLookasidelist_cache != NULL) {
+		kstat_t *ksp = kstat_create("spl", 0,
+		    pLookasidelist_cache->cache_name,
+		    "lookasidelist_cache", KSTAT_TYPE_NAMED,
+		    sizeof (lookasidelist_stats) / sizeof (kstat_named_t),
+		    KSTAT_FLAG_VIRTUAL);
+
+		if (ksp != NULL) {
+			ksp->ks_data = &lookasidelist_stats;
+			ksp->ks_update = lookaside_kstat_update;
+			ksp->ks_private = pLookasidelist_cache;
+			pLookasidelist_cache->cache_kstat = ksp;
+			kstat_install(ksp);
+		}
+	}
+
+	return (pLookasidelist_cache);
+}
+
+void
+lookasidelist_cache_destroy(lookasidelist_cache_t *pLookasidelist_cache)
+{
+	if (pLookasidelist_cache != NULL) {
+		ExFlushLookasideListEx(&pLookasidelist_cache->lookasideField);
+		ExDeleteLookasideListEx(&pLookasidelist_cache->lookasideField);
+		ExFreePoolWithTag(pLookasidelist_cache,
+		    ZFS_LookAsideList_DRV_TAG);
+
+		if (pLookasidelist_cache->cache_kstat != NULL) {
+			kstat_delete(pLookasidelist_cache->cache_kstat);
+			pLookasidelist_cache->cache_kstat = NULL;
+		}
+	}
+}
+
+void *
+lookasidelist_cache_alloc(lookasidelist_cache_t *pLookasidelist_cache)
+{
+	void *buf = ExAllocateFromLookasideListEx(
+	    &pLookasidelist_cache->lookasideField);
+	ASSERT(buf != NULL);
+	return (buf);
+}
+
+void
+lookasidelist_cache_free(lookasidelist_cache_t *pLookasidelist_cache, void *buf)
+{
+	ASSERT(buf != NULL);
+	if (buf != NULL) {
+		ExFreeToLookasideListEx(&pLookasidelist_cache->lookasideField,
+		    buf);
+	}
+}

module/os/windows/zfs/abd_os.c

@@ -32,6 +32,7 @@
 #include <sys/zio.h>
 #include <sys/zfs_context.h>
 #include <sys/zfs_znode.h>
+#include <sys/lookasidelist.h>
 
 
 typedef struct abd_stats {
@@ -89,7 +90,7 @@ struct {
  */
 size_t zfs_abd_chunk_size = 4096;
 
-kmem_cache_t *abd_chunk_cache;
+lookasidelist_cache_t *abd_chunk_cache;
 static kstat_t *abd_ksp;
 
 
@@ -105,7 +106,7 @@ static char *abd_zero_buf = NULL;
 static void
 abd_free_chunk(void *c)
 {
-	kmem_cache_free(abd_chunk_cache, c);
+	lookasidelist_cache_free(abd_chunk_cache, c);
 }
 
 static size_t
@@ -182,7 +183,7 @@ abd_alloc_chunks(abd_t *abd, size_t size)
 {
 	size_t n = abd_chunkcnt_for_bytes(size);
 	for (int i = 0; i < n; i++) {
-		void *c = kmem_cache_alloc(abd_chunk_cache, KM_SLEEP);
+		void *c = lookasidelist_cache_alloc(abd_chunk_cache);
 		ABD_SCATTER(abd).abd_chunks[i] = c;
 	}
 	ABD_SCATTER(abd).abd_chunk_size = zfs_abd_chunk_size;
@@ -236,7 +237,7 @@ static void
 abd_alloc_zero_scatter(void)
 {
 	size_t n = abd_chunkcnt_for_bytes(SPA_MAXBLOCKSIZE);
-	abd_zero_buf = kmem_cache_alloc(abd_chunk_cache, KM_SLEEP);
+	abd_zero_buf = lookasidelist_cache_alloc(abd_chunk_cache);
 	bzero(abd_zero_buf, zfs_abd_chunk_size);
 	abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE);
 
@@ -264,15 +265,14 @@ abd_free_zero_scatter(void)
 
 	abd_free_struct(abd_zero_scatter);
 	abd_zero_scatter = NULL;
-	kmem_cache_free(abd_chunk_cache, abd_zero_buf);
+	lookasidelist_cache_free(abd_chunk_cache, abd_zero_buf);
 }
 
 void
 abd_init(void)
 {
-	abd_chunk_cache = kmem_cache_create("abd_chunk", zfs_abd_chunk_size,
-	    MIN(PAGE_SIZE, 4096),
-	    NULL, NULL, NULL, NULL, abd_arena, KMC_NOTOUCH);
+	abd_chunk_cache = lookasidelist_cache_create("abd_chunk",
+	    zfs_abd_chunk_size);
 
 	abd_ksp = kstat_create("zfs", 0, "abdstats", "misc", KSTAT_TYPE_NAMED,
 	    sizeof (abd_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
@@ -294,7 +294,7 @@ abd_fini(void)
 		abd_ksp = NULL;
 	}
 
-	kmem_cache_destroy(abd_chunk_cache);
+	lookasidelist_cache_destroy(abd_chunk_cache);
 	abd_chunk_cache = NULL;
 }
 
@@ -487,5 +487,5 @@ abd_iter_unmap(struct abd_iter *aiter)
 void
 abd_cache_reap_now(void)
 {
-	kmem_cache_reap_now(abd_chunk_cache);
+	// do nothing
 }

module/os/windows/zfs/arc_os.c

@@ -253,8 +253,7 @@ arc_reclaim_thread(void *unused)
 		 * it is worth reaping the abd_chunk_cache
 		 */
 		if (d_adj >= 64LL*1024LL*1024LL) {
-			extern kmem_cache_t *abd_chunk_cache;
-			kmem_cache_reap_now(abd_chunk_cache);
+			abd_cache_reap_now();
 		}
 
 		free_memory = post_adjust_free_memory;