From 1638f4f7e519d363d9ea3d56bcb93cd48bfc5815 Mon Sep 17 00:00:00 2001 From: dkgupta Date: Mon, 20 Dec 2021 19:11:39 -0500 Subject: [PATCH] slab allocator: fixes/enhancements in slab allocator Fixes: When we first implemented slab allocator for KTF, we never cared for freeing memory when entire slab is freed up. Reason being we thought of it as short- running kernel. However we're into a situation where we're allocating a lot of memory, thanks for integration of external library doing malloc and free. This change ensures that if a slab got completly freed up, we free up the memory and put that meta slab back into free list. Added a spinlock around allocation, free and initialization Enhancement: slab allocator was limited in the sense that it wouldn't allocate more if meta_slab_t entries contained in globally allocated page at initializaiton were all exhausted. This change makes a slight design change as follows:- - During allocation if meta_slab_t is not available then allocator will do - Allocate a fresh page - First entry is special meta_slab_t in this page - First entry will treat rest of meta_slab_t entries in page as normal meta_slab_t entry to allocate user requested memory allocations. - Link special meta_slab_t entry in a global list so that all such allocated pages are linked together Note about special meta_slab_t entry:- Special meta_slab_t is basically to manage meta_slab_t entries itself. It's metadata of metadata of user allocations. Signed-off-by: dkgupta --- include/mm/slab.h | 34 ++++++++++- mm/slab.c | 150 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 145 insertions(+), 39 deletions(-) diff --git a/include/mm/slab.h b/include/mm/slab.h index aad2815d..2dc25f3e 100644 --- a/include/mm/slab.h +++ b/include/mm/slab.h @@ -59,6 +59,10 @@ typedef enum slab_size slab_size_t; #define SLAB_SIZE_FULL_MASK ((SLAB_SIZE_MAX << 1) - 1) +#define MAX_SLAB_ALLOC_COUNT (PAGE_SIZE / SLAB_SIZE_MIN) + +#define META_SLAB_PAGE_ENTRY(meta_slab) ((meta_slab_t *) (_ul(meta_slab) & PAGE_MASK)) + /* * SLAB sizes >= 4K should directly allocate pages */ @@ -74,11 +78,39 @@ struct meta_slab { list_head_t slab_head; void *slab_base; unsigned int slab_len; - unsigned int slab_size; + /* + * Don't need more than 12 bits. Currently max slab size is 2048 bytes = 2^11 + */ + unsigned int slab_size : 12; + /* + * slab_allocs is tracking number of allocations currently in this slab. + * At max this can go 4096/16 = 256 slabs. Thus 10 bits are enough + */ + unsigned int slab_allocs : 10; + unsigned int reserved : 10; }; typedef struct meta_slab meta_slab_t; +static inline void increment_slab_allocs(meta_slab_t *slab) { + BUG_ON(slab == NULL); + BUG_ON((slab->slab_allocs >= (slab->slab_len / slab->slab_size))); + + slab->slab_allocs++; +} + +static inline void decrement_slab_allocs(meta_slab_t *slab) { + BUG_ON(slab == NULL); + BUG_ON((slab->slab_allocs == 0)); + + slab->slab_allocs--; +} + +static inline bool slab_is_empty(meta_slab_t *slab) { + BUG_ON(slab == NULL); + return (slab->slab_allocs == 0); +} + int init_slab(void); extern void *kmalloc(size_t size); extern void *kzalloc(size_t size); diff --git a/mm/slab.c b/mm/slab.c index b32a05d6..12d7edd3 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -35,12 +35,14 @@ #include #include #include +#include #include /* List of meta slab pointers of each order */ static list_head_t meta_slab_list[SLAB_ORDER_MAX]; -meta_slab_t global_meta_slab; +static list_head_t meta_slab_page_list; +static spinlock_t slab_mm_lock = SPINLOCK_INIT; static int initialize_slab(meta_slab_t *slab) { int ret = 0; @@ -97,7 +99,7 @@ static void *slab_alloc(meta_slab_t *slab) { /* TODO: Below should be done in thread-safe manner */ next_free = list_first_entry(&slab->slab_head, slab_t, list); list_unlink(&next_free->list); - + increment_slab_allocs(slab); return next_free; } @@ -111,21 +113,75 @@ static void slab_free(meta_slab_t *slab, void *ptr) { new_slab = (slab_t *) ptr; /* TODO: eventually below should be done in thread-safe manner */ list_add_tail(&new_slab->list, &slab->slab_head); + decrement_slab_allocs(slab); } +meta_slab_t *slab_meta_alloc() { + meta_slab_t *meta_slab_page = NULL; + meta_slab_t *meta_slab = NULL; + void *free_page = NULL; + size_t meta_slab_size = 0; + int ret = 0; + + list_for_each_entry (meta_slab_page, &meta_slab_page_list, list) { + meta_slab = slab_alloc(meta_slab_page); + if (meta_slab != NULL) { + dprintk("Allocating meta_slab from meta slab page %p\n", meta_slab_page); + return meta_slab; + } + } + + /* + * If we're here then we've ran out of meta slab pages + * Allocate a 4K page + */ + free_page = get_free_pages(PAGE_ORDER_4K, GFP_KERNEL); + if (free_page == NULL) { + dprintk("slab_meta_alloc failed, not enough free pages\n"); + return NULL; + } + memset(free_page, 0, PAGE_SIZE); + + /* + * First entry in free page is special meta_slab + * pointing to rest of meta_slabs + */ + meta_slab_size = next_power_of_two(sizeof(meta_slab_t)); + meta_slab_page = (meta_slab_t *) free_page; + meta_slab_page->slab_base = (void *) (_ul(meta_slab_page) + meta_slab_size); + meta_slab_page->slab_len = PAGE_SIZE - meta_slab_size; + meta_slab_page->slab_size = meta_slab_size; + meta_slab_page->slab_allocs = 0; + ret = initialize_slab(meta_slab_page); + if (ret != ESUCCESS) { + dprintk("initialize_slab in slab_meta_alloc failed\n"); + put_pages(free_page, PAGE_ORDER_4K); + return NULL; + } + + dprintk("Allocated a new meta page slab %p\n", meta_slab_page); + /* + * add meta_slab_page to global list of meta slab pages + */ + list_add(&meta_slab_page->list, &meta_slab_page_list); + + /* + * Now allocate a meta slab from meta slab page + */ + meta_slab = slab_alloc(meta_slab_page); + + return meta_slab; +} /* * Round up to nearest power of 2 * If greater than max size or less than min size, return null */ -static void *ktf_alloc(unsigned int size) { - unsigned int size_power2 = 0, temp = 0, order_index = 0; +static void *ktf_alloc(size_t size) { + size_t size_power2 = 0, temp = 0, order_index = 0; meta_slab_t *slab = NULL, *meta_slab = NULL; void *alloc = NULL, *free_page = NULL; int ret = 0; - if (size == 0) - return NULL; - if (size < SLAB_SIZE_MIN) size = SLAB_SIZE_MIN; @@ -145,25 +201,25 @@ static void *ktf_alloc(unsigned int size) { dprintk("Alloc size %u, powerof 2 size %u, order %u\n", size, size_power2, order_index); + spin_lock(&slab_mm_lock); /* Go through list of meta_slab_t and try to allocate a free slab */ list_for_each_entry (slab, &meta_slab_list[order_index], list) { alloc = slab_alloc(slab); - if (alloc != NULL) - return alloc; - /* - * TODO: add debug prints for diaganostics that we didn't get free slab - * and trying next page in the list - */ + if (alloc != NULL) { + dprintk("Allocating from %p\n", slab); + goto out; + } } /* * If we reached here that means it's time to allocate a new meta_slab_t entry * and a new page to hold base address */ - meta_slab = slab_alloc(&global_meta_slab); + meta_slab = slab_meta_alloc(); if (meta_slab == NULL) { dprintk("failed, not enough free pages\n"); - return NULL; + alloc = NULL; + goto out; } dprintk("meta_slab allocated %p\n", meta_slab); @@ -176,25 +232,30 @@ static void *ktf_alloc(unsigned int size) { free_page = get_free_pages(PAGE_ORDER_4K, GFP_KERNEL); if (free_page == NULL) { dprintk("ktf_alloc failed, not enough free pages\n"); - slab_free(&global_meta_slab, meta_slab); - return NULL; + slab_free(META_SLAB_PAGE_ENTRY(meta_slab), meta_slab); + alloc = NULL; + goto out; } memset(free_page, 0, PAGE_SIZE); meta_slab->slab_base = free_page; meta_slab->slab_len = PAGE_SIZE; meta_slab->slab_size = size_power2; + meta_slab->slab_allocs = 0; ret = initialize_slab(meta_slab); if (ret != ESUCCESS) { dprintk("initialize_slab failed\n"); put_pages(free_page, PAGE_ORDER_4K); - return NULL; + alloc = NULL; + goto out; } list_add(&meta_slab->list, &meta_slab_list[order_index]); alloc = slab_alloc(meta_slab); +out: + spin_unlock(&slab_mm_lock); return alloc; } @@ -216,13 +277,42 @@ void *kzalloc(size_t size) { static void ktf_free(void *ptr) { int alloc_order; meta_slab_t *slab = NULL; + meta_slab_t *meta_slab_page = NULL; + spin_lock(&slab_mm_lock); for (alloc_order = SLAB_ORDER_16; alloc_order < SLAB_ORDER_MAX; alloc_order++) { /* Go through list of meta_slab_t and try to allocate a free slab */ list_for_each_entry (slab, &meta_slab_list[alloc_order], list) { if ((_ul(ptr) >= (_ul(slab->slab_base))) && (_ul(ptr) < (_ul(slab->slab_base) + _ul(slab->slab_len)))) { slab_free(slab, ptr); + if (slab_is_empty(slab)) { + dprintk("freeing slab %p of slab size %d and base address %p\n", slab, + slab->slab_size, slab->slab_base); + /* + * Order is important here. First unlink from order list + * Then only slab_free because list will be used to link back into + * meta slab free + */ + list_unlink(&slab->list); + put_pages(slab->slab_base, PAGE_ORDER_4K); + meta_slab_page = META_SLAB_PAGE_ENTRY(slab); + slab_free(meta_slab_page, slab); + /* + * If page holding meta slabs is empty due to this operation, we + * should free up meta slab page entirely + */ + if (slab_is_empty(meta_slab_page)) { + dprintk("freeing meta page slab %p of slab size %d and base " + "address %p\n", + meta_slab_page, meta_slab_page->slab_size, + meta_slab_page->slab_base); + list_unlink(&meta_slab_page->list); + memset(meta_slab_page, 0, PAGE_SIZE); + put_pages(meta_slab_page, PAGE_ORDER_4K); + } + } + spin_unlock(&slab_mm_lock); return; } } @@ -238,33 +328,17 @@ void kfree(void *ptr) { ktf_free(ptr); } int init_slab(void) { int ret = 0; int i = 0; - void *alloc_pages = NULL; printk("Initialize SLAB\n"); + spin_lock(&slab_mm_lock); memset(&meta_slab_list, 0, sizeof(meta_slab_list)); - memset(&global_meta_slab, 0, sizeof(global_meta_slab)); - - alloc_pages = get_free_pages(PAGE_ORDER_2M, GFP_KERNEL); - if (NULL == alloc_pages) { - dprintk("get_free_pages failed\n"); - return -ENOMEM; - } - memset(alloc_pages, 0, PAGE_SIZE_2M); - - global_meta_slab.slab_base = alloc_pages; - global_meta_slab.slab_len = PAGE_SIZE_2M; - global_meta_slab.slab_size = next_power_of_two(sizeof(meta_slab_t)); - ret = initialize_slab(&global_meta_slab); - if (ret != ESUCCESS) { - dprintk("initialize_slab failed\n"); - put_pages(alloc_pages, PAGE_ORDER_2M); - return ret; - } + memset(&meta_slab_page_list, 0, sizeof(meta_slab_page_list)); + list_init(&meta_slab_page_list); for (i = SLAB_ORDER_16; i < SLAB_ORDER_MAX; i++) { list_init(&meta_slab_list[i]); } - + spin_unlock(&slab_mm_lock); dprintk("After initializing slab module\n"); return ret; }