From e34271c50632da5eb1295e410f09e2c31dcca654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20R=C3=BCegge?= Date: Mon, 2 Dec 2024 15:01:39 +0100 Subject: [PATCH 1/8] cheap and ugly first attempt --- arch/x86/pagetables.c | 62 +++++++++++++++++++++++------------- include/arch/x86/pagetable.h | 7 ++++ include/mm/pmm.h | 3 ++ mm/pmm.c | 23 +++++++++++-- 4 files changed, 70 insertions(+), 25 deletions(-) diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c index e095146f..5ea646c0 100644 --- a/arch/x86/pagetables.c +++ b/arch/x86/pagetables.c @@ -196,11 +196,17 @@ static inline void clean_pagetable(void *tab) { set_pgentry(e, MFN_INVALID, PT_NO_FLAGS); } -static mfn_t get_cr3_mfn(cr3_t *cr3_entry) { +static mfn_t get_cr3_mfn(cr3_t *cr3_entry, bool special_path) { void *cr3_mapped = NULL; if (mfn_invalid(cr3_entry->mfn)) { - frame_t *frame = get_free_frame(); + frame_t *frame; + if (special_path) { + frame = _get_backup_frame(); + } + else { + frame = get_backup_frame(); + } BUG_ON(!frame); frame->flags.pagetable = 1; @@ -232,7 +238,8 @@ static inline void pgentry_fixup_flags(pgentry_t *entry, unsigned long flags) { } } -static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flags) { +static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flags, + bool special_path) { pgentry_t *tab, *entry; mfn_t mfn; @@ -243,7 +250,13 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag mfn = mfn_from_pgentry(*entry); if (mfn_invalid(mfn)) { - frame_t *frame = get_free_frame(); + frame_t *frame; + if (special_path) { + frame = _get_backup_frame(); + } + else { + frame = get_backup_frame(); + } BUG_ON(!frame); frame->flags.pagetable = 1; @@ -264,12 +277,12 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag * MAP_FAILED when failed to map a NULL (0x0) virtual address and otherwise * it returns the same virtual address passed as argument. */ -static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, +void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, #if defined(__x86_64__) - unsigned long l4_flags, + unsigned long l4_flags, #endif - unsigned long l3_flags, unsigned long l2_flags, - unsigned long l1_flags) { + unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags, + bool special_path) { mfn_t l1t_mfn, l2t_mfn, l3t_mfn; pgentry_t *tab, *entry; @@ -277,7 +290,8 @@ static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, return va ? NULL : MAP_FAILED; #if defined(__x86_64__) - l3t_mfn = get_pgentry_mfn(get_cr3_mfn(cr3_ptr), l4_table_index(va), l4_flags); + l3t_mfn = get_pgentry_mfn(get_cr3_mfn(cr3_ptr, special_path), l4_table_index(va), + l4_flags, special_path); #else l3t_mfn = get_cr3_mfn(cr3_ptr); #endif @@ -290,7 +304,7 @@ static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, goto done; } - l2t_mfn = get_pgentry_mfn(l3t_mfn, l3_table_index(va), l3_flags); + l2t_mfn = get_pgentry_mfn(l3t_mfn, l3_table_index(va), l3_flags, special_path); if (order == PAGE_ORDER_2M) { tab = tmp_map_mfn(l2t_mfn); @@ -300,7 +314,7 @@ static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, goto done; } - l1t_mfn = get_pgentry_mfn(l2t_mfn, l2_table_index(va), l2_flags); + l1t_mfn = get_pgentry_mfn(l2t_mfn, l2_table_index(va), l2_flags, special_path); tab = tmp_map_mfn(l1t_mfn); entry = &tab[l1_table_index(va)]; @@ -314,19 +328,20 @@ static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, static inline void *__vmap_1g(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned long l4_flags, unsigned long l3_flags) { return _vmap(cr3_ptr, va, mfn, PAGE_ORDER_1G, l4_flags, l3_flags | _PAGE_PSE, - PT_NO_FLAGS, PT_NO_FLAGS); + PT_NO_FLAGS, PT_NO_FLAGS, false); } static inline void *__vmap_2m(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned long l4_flags, unsigned long l3_flags, unsigned long l2_flags) { return _vmap(cr3_ptr, va, mfn, PAGE_ORDER_2M, l4_flags, l3_flags, - l2_flags | _PAGE_PSE, PT_NO_FLAGS); + l2_flags | _PAGE_PSE, PT_NO_FLAGS, false); } static inline void *__vmap_4k(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned long l4_flags, unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) { - return _vmap(cr3_ptr, va, mfn, PAGE_ORDER_4K, l4_flags, l3_flags, l2_flags, l1_flags); + return _vmap(cr3_ptr, va, mfn, PAGE_ORDER_4K, l4_flags, l3_flags, l2_flags, l1_flags, + false); } static inline void *_vmap_1g(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned long l3_flags, @@ -375,7 +390,7 @@ void *vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, dprintk("%s: va: 0x%p mfn: 0x%lx (order: %u)\n", __func__, va, mfn, order); spin_lock(&vmap_lock); - va = _vmap(cr3_ptr, va, mfn, order, l4_flags, l3_flags, l2_flags, l1_flags); + va = _vmap(cr3_ptr, va, mfn, order, l4_flags, l3_flags, l2_flags, l1_flags, false); spin_unlock(&vmap_lock); return va; @@ -872,7 +887,8 @@ static void map_pagetable(cr3_t *cr3_ptr, mfn_t table, int level) { void *va = mfn_to_virt_kern(table); pte_t *pt; - pt = _vmap(cr3_ptr, va, table, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, L1_PROT); + pt = _vmap(cr3_ptr, va, table, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, L1_PROT, + false); BUG_ON(!pt); for (int i = 0; i < level_to_entries(level) && level > 1; i++) { @@ -968,7 +984,7 @@ int map_pagetables_va(cr3_t *cr3_ptr, void *va) { err = -EFAULT; spin_lock(&vmap_lock); tab = _vmap(cr3_ptr, mfn_to_virt_kern(cr3_ptr->mfn), cr3_ptr->mfn, PAGE_ORDER_4K, - L4_PROT, L3_PROT, L2_PROT, L1_PROT); + L4_PROT, L3_PROT, L2_PROT, L1_PROT, false); if (!tab) goto unlock; @@ -980,7 +996,7 @@ int map_pagetables_va(cr3_t *cr3_ptr, void *va) { } tab = _vmap(cr3_ptr, mfn_to_virt_kern(l4e->mfn), l4e->mfn, PAGE_ORDER_4K, L4_PROT, - L3_PROT, L2_PROT, L1_PROT); + L3_PROT, L2_PROT, L1_PROT, false); if (!tab) goto unlock; #endif @@ -995,7 +1011,7 @@ int map_pagetables_va(cr3_t *cr3_ptr, void *va) { goto done; tab = _vmap(cr3_ptr, mfn_to_virt_kern(l3e->mfn), l3e->mfn, PAGE_ORDER_4K, L4_PROT, - L3_PROT, L2_PROT, L1_PROT); + L3_PROT, L2_PROT, L1_PROT, false); if (!tab) goto unlock; @@ -1009,7 +1025,7 @@ int map_pagetables_va(cr3_t *cr3_ptr, void *va) { goto done; tab = _vmap(cr3_ptr, mfn_to_virt_kern(l2e->mfn), l2e->mfn, PAGE_ORDER_4K, L4_PROT, - L3_PROT, L2_PROT, L1_PROT); + L3_PROT, L2_PROT, L1_PROT, false); if (!tab) goto unlock; @@ -1040,7 +1056,7 @@ int unmap_pagetables_va(cr3_t *cr3_ptr, void *va) { err = -EFAULT; spin_lock(&vmap_lock); tab = _vmap(cr3_ptr, mfn_to_virt_kern(cr3_ptr->mfn), cr3_ptr->mfn, PAGE_ORDER_4K, - L4_PROT, L3_PROT, L2_PROT, L1_PROT); + L4_PROT, L3_PROT, L2_PROT, L1_PROT, false); if (!tab) goto cleanup; tables[level++] = tab; @@ -1053,7 +1069,7 @@ int unmap_pagetables_va(cr3_t *cr3_ptr, void *va) { } tab = _vmap(cr3_ptr, mfn_to_virt_kern(l4e->mfn), l4e->mfn, PAGE_ORDER_4K, L4_PROT, - L3_PROT, L2_PROT, L1_PROT); + L3_PROT, L2_PROT, L1_PROT, false); if (!tab) goto cleanup; tables[level++] = tab; @@ -1069,7 +1085,7 @@ int unmap_pagetables_va(cr3_t *cr3_ptr, void *va) { goto done; tab = _vmap(cr3_ptr, mfn_to_virt_kern(l3e->mfn), l3e->mfn, PAGE_ORDER_4K, L4_PROT, - L3_PROT, L2_PROT, L1_PROT); + L3_PROT, L2_PROT, L1_PROT, false); if (!tab) goto cleanup; tables[level++] = tab; diff --git a/include/arch/x86/pagetable.h b/include/arch/x86/pagetable.h index b0dbd7e1..03bb79b0 100644 --- a/include/arch/x86/pagetable.h +++ b/include/arch/x86/pagetable.h @@ -386,6 +386,13 @@ static inline void setup_tlb_global(void) { write_cr4(opt_tlb_global ? (cr4 | X86_CR4_PGE) : (cr4 & ~X86_CR4_PGE)); } +void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, +#if defined(__x86_64__) + unsigned long l4_flags, +#endif + unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags, + bool special_path); + #endif /* __ASSEMBLY__ */ #endif /* KTF_PAGETABLE_H */ diff --git a/include/mm/pmm.h b/include/mm/pmm.h index 246c225c..454dcdca 100644 --- a/include/mm/pmm.h +++ b/include/mm/pmm.h @@ -142,6 +142,9 @@ static inline bool is_frame_free(const frame_t *frame) { return frame->flags.free; } +frame_t *get_backup_frame(void); +frame_t *_get_backup_frame(void); + #endif /* __ASSEMBLY__ */ #endif /* KTF_PMM_H */ diff --git a/mm/pmm.c b/mm/pmm.c index 72620d35..50a0be40 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -41,7 +41,7 @@ static frames_array_t early_frames; static list_head_t free_frames[MAX_PAGE_ORDER + 1]; static list_head_t busy_frames[MAX_PAGE_ORDER + 1]; -#define MIN_NUM_4K_FRAMES 2 +#define MIN_NUM_4K_FRAMES 16 static size_t frames_count[MAX_PAGE_ORDER + 1]; static spinlock_t lock = SPINLOCK_INIT; @@ -108,7 +108,10 @@ static frames_array_t *new_frames_array(void) { if (!boot_flags.virt) array = (frames_array_t *) mfn_to_virt_kern(frame->mfn); else { - array = vmap_kern_4k(mfn_to_virt_map(frame->mfn), frame->mfn, L1_PROT); + array = _vmap( + &cr3, + _ptr(_ul(mfn_to_virt_map(frame->mfn)) & PAGE_ORDER_TO_MASK(PAGE_ORDER_4K)), + frame->mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, L1_PROT, true); if (!array) goto error; } @@ -663,6 +666,22 @@ void put_free_frames(mfn_t mfn, unsigned int order) { spin_unlock(&lock); } +frame_t *get_backup_frame(void) { + // return get_free_frame(); + frame_t *frame; + + spin_lock(&lock); + frame = reserve_frame(get_first_frame(free_frames, PAGE_ORDER_4K)); + spin_unlock(&lock); + + return frame; +} + +frame_t *_get_backup_frame(void) { + // return get_free_frame(); + return reserve_frame(get_first_frame(free_frames, PAGE_ORDER_4K)); +} + void map_frames_array(void) { frames_array_t *array; From dd6e8bb81ecb982cb9b33d970367a0d06d81e4df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20R=C3=BCegge?= Date: Mon, 2 Dec 2024 15:34:43 +0100 Subject: [PATCH 2/8] frame refill needs paging lock --- arch/x86/pagetables.c | 54 ++++++++++++++++++++++++------------ include/arch/x86/pagetable.h | 12 ++++---- mm/pmm.c | 4 +-- 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c index 5ea646c0..1d7b1a9e 100644 --- a/arch/x86/pagetables.c +++ b/arch/x86/pagetables.c @@ -277,12 +277,12 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag * MAP_FAILED when failed to map a NULL (0x0) virtual address and otherwise * it returns the same virtual address passed as argument. */ -void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, +void *__vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, #if defined(__x86_64__) - unsigned long l4_flags, + unsigned long l4_flags, #endif - unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags, - bool special_path) { + unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags, + bool special_path) { mfn_t l1t_mfn, l2t_mfn, l3t_mfn; pgentry_t *tab, *entry; @@ -324,24 +324,43 @@ void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, done: return va; } +static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, +#if defined(__x86_64__) + unsigned long l4_flags, +#endif + unsigned long l3_flags, unsigned long l2_flags, + unsigned long l1_flags) { + return __vmap(cr3_ptr, va, mfn, order, l4_flags, l3_flags, l2_flags, l1_flags, false); +} +void *__vmap_paging(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, +#if defined(__x86_64__) + unsigned long l4_flags, +#endif + unsigned long l3_flags, unsigned long l2_flags, + unsigned long l1_flags, bool special_path) { + spin_lock(&vmap_lock); + void *res = __vmap(cr3_ptr, va, mfn, order, l4_flags, l3_flags, l2_flags, l1_flags, + special_path); + spin_unlock(&vmap_lock); + return res; +} static inline void *__vmap_1g(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned long l4_flags, unsigned long l3_flags) { return _vmap(cr3_ptr, va, mfn, PAGE_ORDER_1G, l4_flags, l3_flags | _PAGE_PSE, - PT_NO_FLAGS, PT_NO_FLAGS, false); + PT_NO_FLAGS, PT_NO_FLAGS); } static inline void *__vmap_2m(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned long l4_flags, unsigned long l3_flags, unsigned long l2_flags) { return _vmap(cr3_ptr, va, mfn, PAGE_ORDER_2M, l4_flags, l3_flags, - l2_flags | _PAGE_PSE, PT_NO_FLAGS, false); + l2_flags | _PAGE_PSE, PT_NO_FLAGS); } static inline void *__vmap_4k(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned long l4_flags, unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) { - return _vmap(cr3_ptr, va, mfn, PAGE_ORDER_4K, l4_flags, l3_flags, l2_flags, l1_flags, - false); + return _vmap(cr3_ptr, va, mfn, PAGE_ORDER_4K, l4_flags, l3_flags, l2_flags, l1_flags); } static inline void *_vmap_1g(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned long l3_flags, @@ -390,7 +409,7 @@ void *vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, dprintk("%s: va: 0x%p mfn: 0x%lx (order: %u)\n", __func__, va, mfn, order); spin_lock(&vmap_lock); - va = _vmap(cr3_ptr, va, mfn, order, l4_flags, l3_flags, l2_flags, l1_flags, false); + va = _vmap(cr3_ptr, va, mfn, order, l4_flags, l3_flags, l2_flags, l1_flags); spin_unlock(&vmap_lock); return va; @@ -887,8 +906,7 @@ static void map_pagetable(cr3_t *cr3_ptr, mfn_t table, int level) { void *va = mfn_to_virt_kern(table); pte_t *pt; - pt = _vmap(cr3_ptr, va, table, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, L1_PROT, - false); + pt = _vmap(cr3_ptr, va, table, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, L1_PROT); BUG_ON(!pt); for (int i = 0; i < level_to_entries(level) && level > 1; i++) { @@ -984,7 +1002,7 @@ int map_pagetables_va(cr3_t *cr3_ptr, void *va) { err = -EFAULT; spin_lock(&vmap_lock); tab = _vmap(cr3_ptr, mfn_to_virt_kern(cr3_ptr->mfn), cr3_ptr->mfn, PAGE_ORDER_4K, - L4_PROT, L3_PROT, L2_PROT, L1_PROT, false); + L4_PROT, L3_PROT, L2_PROT, L1_PROT); if (!tab) goto unlock; @@ -996,7 +1014,7 @@ int map_pagetables_va(cr3_t *cr3_ptr, void *va) { } tab = _vmap(cr3_ptr, mfn_to_virt_kern(l4e->mfn), l4e->mfn, PAGE_ORDER_4K, L4_PROT, - L3_PROT, L2_PROT, L1_PROT, false); + L3_PROT, L2_PROT, L1_PROT); if (!tab) goto unlock; #endif @@ -1011,7 +1029,7 @@ int map_pagetables_va(cr3_t *cr3_ptr, void *va) { goto done; tab = _vmap(cr3_ptr, mfn_to_virt_kern(l3e->mfn), l3e->mfn, PAGE_ORDER_4K, L4_PROT, - L3_PROT, L2_PROT, L1_PROT, false); + L3_PROT, L2_PROT, L1_PROT); if (!tab) goto unlock; @@ -1025,7 +1043,7 @@ int map_pagetables_va(cr3_t *cr3_ptr, void *va) { goto done; tab = _vmap(cr3_ptr, mfn_to_virt_kern(l2e->mfn), l2e->mfn, PAGE_ORDER_4K, L4_PROT, - L3_PROT, L2_PROT, L1_PROT, false); + L3_PROT, L2_PROT, L1_PROT); if (!tab) goto unlock; @@ -1056,7 +1074,7 @@ int unmap_pagetables_va(cr3_t *cr3_ptr, void *va) { err = -EFAULT; spin_lock(&vmap_lock); tab = _vmap(cr3_ptr, mfn_to_virt_kern(cr3_ptr->mfn), cr3_ptr->mfn, PAGE_ORDER_4K, - L4_PROT, L3_PROT, L2_PROT, L1_PROT, false); + L4_PROT, L3_PROT, L2_PROT, L1_PROT); if (!tab) goto cleanup; tables[level++] = tab; @@ -1069,7 +1087,7 @@ int unmap_pagetables_va(cr3_t *cr3_ptr, void *va) { } tab = _vmap(cr3_ptr, mfn_to_virt_kern(l4e->mfn), l4e->mfn, PAGE_ORDER_4K, L4_PROT, - L3_PROT, L2_PROT, L1_PROT, false); + L3_PROT, L2_PROT, L1_PROT); if (!tab) goto cleanup; tables[level++] = tab; @@ -1085,7 +1103,7 @@ int unmap_pagetables_va(cr3_t *cr3_ptr, void *va) { goto done; tab = _vmap(cr3_ptr, mfn_to_virt_kern(l3e->mfn), l3e->mfn, PAGE_ORDER_4K, L4_PROT, - L3_PROT, L2_PROT, L1_PROT, false); + L3_PROT, L2_PROT, L1_PROT); if (!tab) goto cleanup; tables[level++] = tab; diff --git a/include/arch/x86/pagetable.h b/include/arch/x86/pagetable.h index 03bb79b0..5b6399a6 100644 --- a/include/arch/x86/pagetable.h +++ b/include/arch/x86/pagetable.h @@ -240,11 +240,11 @@ static inline pte_t *l1_table_entry(pte_t *tab, const void *va) { } static inline pgentry_t pgentry_from_paddr(paddr_t pa, unsigned long flags) { - return (pgentry_t)((pa & ~(PADDR_MASK & PAGE_MASK)) | (flags & _PAGE_ALL_FLAGS)); + return (pgentry_t) ((pa & ~(PADDR_MASK & PAGE_MASK)) | (flags & _PAGE_ALL_FLAGS)); } static inline paddr_t paddr_from_pgentry(pgentry_t pgentry) { - return (paddr_t)(pgentry & ~PADDR_MASK) & PAGE_MASK; + return (paddr_t) (pgentry & ~PADDR_MASK) & PAGE_MASK; } static inline pgentry_t pgentry_from_mfn(mfn_t mfn, unsigned long flags) { @@ -386,12 +386,12 @@ static inline void setup_tlb_global(void) { write_cr4(opt_tlb_global ? (cr4 | X86_CR4_PGE) : (cr4 & ~X86_CR4_PGE)); } -void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, +void *__vmap_paging(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, #if defined(__x86_64__) - unsigned long l4_flags, + unsigned long l4_flags, #endif - unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags, - bool special_path); + unsigned long l3_flags, unsigned long l2_flags, + unsigned long l1_flags, bool special_path); #endif /* __ASSEMBLY__ */ diff --git a/mm/pmm.c b/mm/pmm.c index 50a0be40..33c1dabc 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -41,7 +41,7 @@ static frames_array_t early_frames; static list_head_t free_frames[MAX_PAGE_ORDER + 1]; static list_head_t busy_frames[MAX_PAGE_ORDER + 1]; -#define MIN_NUM_4K_FRAMES 16 +#define MIN_NUM_4K_FRAMES 2 + 4 + (MAX_PAGE_ORDER - PAGE_ORDER_4K) static size_t frames_count[MAX_PAGE_ORDER + 1]; static spinlock_t lock = SPINLOCK_INIT; @@ -108,7 +108,7 @@ static frames_array_t *new_frames_array(void) { if (!boot_flags.virt) array = (frames_array_t *) mfn_to_virt_kern(frame->mfn); else { - array = _vmap( + array = __vmap_paging( &cr3, _ptr(_ul(mfn_to_virt_map(frame->mfn)) & PAGE_ORDER_TO_MASK(PAGE_ORDER_4K)), frame->mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, L1_PROT, true); From 718df325d8f0e62b9eaef22d157471b680dce7ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20R=C3=BCegge?= Date: Mon, 2 Dec 2024 15:52:39 +0100 Subject: [PATCH 3/8] allow frame refill from paging --- arch/x86/pagetables.c | 9 ++++++--- include/arch/x86/pagetable.h | 6 +++--- include/mm/pmm.h | 1 + mm/pmm.c | 34 +++++++++++++++++++++++++++++----- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c index 1d7b1a9e..6c99ac42 100644 --- a/arch/x86/pagetables.c +++ b/arch/x86/pagetables.c @@ -322,6 +322,7 @@ void *__vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, invlpg(va); done: + refill_from_paging(); return va; } static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, @@ -337,11 +338,13 @@ void *__vmap_paging(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, unsigned long l4_flags, #endif unsigned long l3_flags, unsigned long l2_flags, - unsigned long l1_flags, bool special_path) { - spin_lock(&vmap_lock); + unsigned long l1_flags, bool special_path, bool take_lock) { + if (take_lock) + spin_lock(&vmap_lock); void *res = __vmap(cr3_ptr, va, mfn, order, l4_flags, l3_flags, l2_flags, l1_flags, special_path); - spin_unlock(&vmap_lock); + if (take_lock) + spin_unlock(&vmap_lock); return res; } diff --git a/include/arch/x86/pagetable.h b/include/arch/x86/pagetable.h index 5b6399a6..fdf1688a 100644 --- a/include/arch/x86/pagetable.h +++ b/include/arch/x86/pagetable.h @@ -240,11 +240,11 @@ static inline pte_t *l1_table_entry(pte_t *tab, const void *va) { } static inline pgentry_t pgentry_from_paddr(paddr_t pa, unsigned long flags) { - return (pgentry_t) ((pa & ~(PADDR_MASK & PAGE_MASK)) | (flags & _PAGE_ALL_FLAGS)); + return (pgentry_t)((pa & ~(PADDR_MASK & PAGE_MASK)) | (flags & _PAGE_ALL_FLAGS)); } static inline paddr_t paddr_from_pgentry(pgentry_t pgentry) { - return (paddr_t) (pgentry & ~PADDR_MASK) & PAGE_MASK; + return (paddr_t)(pgentry & ~PADDR_MASK) & PAGE_MASK; } static inline pgentry_t pgentry_from_mfn(mfn_t mfn, unsigned long flags) { @@ -391,7 +391,7 @@ void *__vmap_paging(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, unsigned long l4_flags, #endif unsigned long l3_flags, unsigned long l2_flags, - unsigned long l1_flags, bool special_path); + unsigned long l1_flags, bool special_path, bool paging_lock); #endif /* __ASSEMBLY__ */ diff --git a/include/mm/pmm.h b/include/mm/pmm.h index 454dcdca..330f82d7 100644 --- a/include/mm/pmm.h +++ b/include/mm/pmm.h @@ -142,6 +142,7 @@ static inline bool is_frame_free(const frame_t *frame) { return frame->flags.free; } +void refill_from_paging(void); frame_t *get_backup_frame(void); frame_t *_get_backup_frame(void); diff --git a/mm/pmm.c b/mm/pmm.c index 33c1dabc..a839da6d 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -41,7 +41,9 @@ static frames_array_t early_frames; static list_head_t free_frames[MAX_PAGE_ORDER + 1]; static list_head_t busy_frames[MAX_PAGE_ORDER + 1]; -#define MIN_NUM_4K_FRAMES 2 + 4 + (MAX_PAGE_ORDER - PAGE_ORDER_4K) +/* 1 frame for array frame, 3 for mapping it and then one additional frame for each page + * order we might have to split */ +#define MIN_NUM_4K_FRAMES 1 + 3 + (MAX_PAGE_ORDER - PAGE_ORDER_4K) static size_t frames_count[MAX_PAGE_ORDER + 1]; static spinlock_t lock = SPINLOCK_INIT; @@ -97,7 +99,7 @@ static inline void init_frames_array(frames_array_t *array) { list_add(&array->list, &frames); } -static frames_array_t *new_frames_array(void) { +static frames_array_t *new_frames_array(bool paging_lock) { frames_array_t *array; frame_t *frame; @@ -111,7 +113,8 @@ static frames_array_t *new_frames_array(void) { array = __vmap_paging( &cr3, _ptr(_ul(mfn_to_virt_map(frame->mfn)) & PAGE_ORDER_TO_MASK(PAGE_ORDER_4K)), - frame->mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, L1_PROT, true); + frame->mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, L1_PROT, true, + paging_lock); if (!array) goto error; } @@ -160,7 +163,18 @@ static inline frames_array_t *get_frames_array(void) { return array; } - return new_frames_array(); + return new_frames_array(false); +} + +static inline size_t get_frames_array_space(void) { + size_t space = 0; + frames_array_t *array; + + list_for_each_entry (array, &frames, list) { + space += array->meta.free_count; + } + + return space; } static inline bool put_frames_array(frames_array_t *array) { @@ -195,7 +209,7 @@ static inline frame_t *take_frame(frame_t *frame, frames_array_t *array) { array->meta.free_count--; if (--total_free_frames <= MIN_FREE_FRAMES_THRESHOLD) - new_frames_array(); + new_frames_array(false); return frame; } @@ -693,3 +707,13 @@ void map_frames_array(void) { BUG_ON(!vmap_kern_4k(va, mfn, L1_PROT)); } } + +void refill_from_paging(void) { + // make sure we have enough space to refill the 4K frames + // so we need space for one frame per + if (get_frames_array_space() < MIN_NUM_4K_FRAMES) { + new_frames_array(false); + } + + try_create_4k_frames(); +} \ No newline at end of file From 808a2c299b4c5d5ba9deec437d6e3c98c6110376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20R=C3=BCegge?= Date: Tue, 3 Dec 2024 09:55:53 +0100 Subject: [PATCH 4/8] better pmm refill implementation --- arch/x86/pagetables.c | 74 ++++++++-------------- include/arch/x86/pagetable.h | 9 +-- include/mm/pmm.h | 3 +- mm/pmm.c | 116 ++++++++++++++++++++++------------- 4 files changed, 102 insertions(+), 100 deletions(-) diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c index 6c99ac42..b1ea54c7 100644 --- a/arch/x86/pagetables.c +++ b/arch/x86/pagetables.c @@ -196,17 +196,11 @@ static inline void clean_pagetable(void *tab) { set_pgentry(e, MFN_INVALID, PT_NO_FLAGS); } -static mfn_t get_cr3_mfn(cr3_t *cr3_entry, bool special_path) { +static mfn_t get_cr3_mfn(cr3_t *cr3_entry) { void *cr3_mapped = NULL; if (mfn_invalid(cr3_entry->mfn)) { - frame_t *frame; - if (special_path) { - frame = _get_backup_frame(); - } - else { - frame = get_backup_frame(); - } + frame_t *frame = get_free_frame_norefill(); BUG_ON(!frame); frame->flags.pagetable = 1; @@ -238,8 +232,7 @@ static inline void pgentry_fixup_flags(pgentry_t *entry, unsigned long flags) { } } -static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flags, - bool special_path) { +static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flags) { pgentry_t *tab, *entry; mfn_t mfn; @@ -250,13 +243,7 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag mfn = mfn_from_pgentry(*entry); if (mfn_invalid(mfn)) { - frame_t *frame; - if (special_path) { - frame = _get_backup_frame(); - } - else { - frame = get_backup_frame(); - } + frame_t *frame = get_free_frame_norefill(); BUG_ON(!frame); frame->flags.pagetable = 1; @@ -277,12 +264,12 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag * MAP_FAILED when failed to map a NULL (0x0) virtual address and otherwise * it returns the same virtual address passed as argument. */ -void *__vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, +static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, #if defined(__x86_64__) - unsigned long l4_flags, + unsigned long l4_flags, #endif - unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags, - bool special_path) { + unsigned long l3_flags, unsigned long l2_flags, + unsigned long l1_flags) { mfn_t l1t_mfn, l2t_mfn, l3t_mfn; pgentry_t *tab, *entry; @@ -290,8 +277,7 @@ void *__vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, return va ? NULL : MAP_FAILED; #if defined(__x86_64__) - l3t_mfn = get_pgentry_mfn(get_cr3_mfn(cr3_ptr, special_path), l4_table_index(va), - l4_flags, special_path); + l3t_mfn = get_pgentry_mfn(get_cr3_mfn(cr3_ptr), l4_table_index(va), l4_flags); #else l3t_mfn = get_cr3_mfn(cr3_ptr); #endif @@ -304,7 +290,7 @@ void *__vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, goto done; } - l2t_mfn = get_pgentry_mfn(l3t_mfn, l3_table_index(va), l3_flags, special_path); + l2t_mfn = get_pgentry_mfn(l3t_mfn, l3_table_index(va), l3_flags); if (order == PAGE_ORDER_2M) { tab = tmp_map_mfn(l2t_mfn); @@ -314,7 +300,7 @@ void *__vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, goto done; } - l1t_mfn = get_pgentry_mfn(l2t_mfn, l2_table_index(va), l2_flags, special_path); + l1t_mfn = get_pgentry_mfn(l2t_mfn, l2_table_index(va), l2_flags); tab = tmp_map_mfn(l1t_mfn); entry = &tab[l1_table_index(va)]; @@ -325,28 +311,6 @@ void *__vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, refill_from_paging(); return va; } -static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, -#if defined(__x86_64__) - unsigned long l4_flags, -#endif - unsigned long l3_flags, unsigned long l2_flags, - unsigned long l1_flags) { - return __vmap(cr3_ptr, va, mfn, order, l4_flags, l3_flags, l2_flags, l1_flags, false); -} -void *__vmap_paging(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, -#if defined(__x86_64__) - unsigned long l4_flags, -#endif - unsigned long l3_flags, unsigned long l2_flags, - unsigned long l1_flags, bool special_path, bool take_lock) { - if (take_lock) - spin_lock(&vmap_lock); - void *res = __vmap(cr3_ptr, va, mfn, order, l4_flags, l3_flags, l2_flags, l1_flags, - special_path); - if (take_lock) - spin_unlock(&vmap_lock); - return res; -} static inline void *__vmap_1g(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned long l4_flags, unsigned long l3_flags) { @@ -457,6 +421,22 @@ void *vmap_4k(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned long l1_flags, return va; } +void *vmap_frame_refill(void *va, mfn_t mfn, bool paging_lock) { + unsigned long _va = _ul(va) & PAGE_ORDER_TO_MASK(PAGE_ORDER_4K); + + dprintk("%s: va: 0x%p mfn: 0x%lx\n", __func__, va, mfn); + + ASSERT(vmap_lock); + + if (paging_lock) + spin_lock(&vmap_lock); + va = _vmap_4k(&cr3, _ptr(_va), mfn, L1_PROT, false); + if (paging_lock) + spin_unlock(&vmap_lock); + + return va; +} + static inline void init_tmp_mapping(void) { pte_t *tab = get_l1_table(_tmp_mapping); _tmp_mapping_entry = (pgentry_t *) l1_table_entry(tab, _tmp_mapping); diff --git a/include/arch/x86/pagetable.h b/include/arch/x86/pagetable.h index fdf1688a..34e53ba9 100644 --- a/include/arch/x86/pagetable.h +++ b/include/arch/x86/pagetable.h @@ -360,6 +360,8 @@ static inline void *vmap_kern_4k(void *va, mfn_t mfn, unsigned long l1_flags) { return vmap_4k(&cr3, va, mfn, l1_flags, false); } +void *vmap_frame_refill(void *va, mfn_t mfn, bool paging_lock); + static inline void *vmap_user(void *va, mfn_t mfn, unsigned int order, #if defined(__x86_64__) unsigned long l4_flags, @@ -386,13 +388,6 @@ static inline void setup_tlb_global(void) { write_cr4(opt_tlb_global ? (cr4 | X86_CR4_PGE) : (cr4 & ~X86_CR4_PGE)); } -void *__vmap_paging(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order, -#if defined(__x86_64__) - unsigned long l4_flags, -#endif - unsigned long l3_flags, unsigned long l2_flags, - unsigned long l1_flags, bool special_path, bool paging_lock); - #endif /* __ASSEMBLY__ */ #endif /* KTF_PAGETABLE_H */ diff --git a/include/mm/pmm.h b/include/mm/pmm.h index 330f82d7..baea26cf 100644 --- a/include/mm/pmm.h +++ b/include/mm/pmm.h @@ -143,8 +143,7 @@ static inline bool is_frame_free(const frame_t *frame) { } void refill_from_paging(void); -frame_t *get_backup_frame(void); -frame_t *_get_backup_frame(void); +frame_t *get_free_frame_norefill(void); #endif /* __ASSEMBLY__ */ diff --git a/mm/pmm.c b/mm/pmm.c index a839da6d..81e11b4e 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -41,13 +41,23 @@ static frames_array_t early_frames; static list_head_t free_frames[MAX_PAGE_ORDER + 1]; static list_head_t busy_frames[MAX_PAGE_ORDER + 1]; -/* 1 frame for array frame, 3 for mapping it and then one additional frame for each page - * order we might have to split */ -#define MIN_NUM_4K_FRAMES 1 + 3 + (MAX_PAGE_ORDER - PAGE_ORDER_4K) +/* 1 frame for array frame, 3 for mapping it, times to for paging/pmm collisions */ +#define MIN_NUM_4K_FRAMES 2 * (1 + 3) static size_t frames_count[MAX_PAGE_ORDER + 1]; +static bool refilling; static spinlock_t lock = SPINLOCK_INIT; +static void spin_lock_norefill() { + spin_lock(&lock); + + while (refilling) { + spin_unlock(&lock); + /* TODO: some backoff would be nice here... */ + spin_lock(&lock); + } +} + void display_frames_count(void) { printk("Avail memory frames: (total size: %lu MB)\n", total_phys_memory / MB(1)); @@ -97,9 +107,10 @@ static inline void init_frames_array(frames_array_t *array) { for (unsigned i = 0; i < ARRAY_SIZE(array->frames); i++) init_frame(&array->frames[i]); list_add(&array->list, &frames); + total_free_frames += array->meta.free_count; } -static frames_array_t *new_frames_array(bool paging_lock) { +static frames_array_t *_new_frames_array(bool paging_lock) { frames_array_t *array; frame_t *frame; @@ -110,11 +121,13 @@ static frames_array_t *new_frames_array(bool paging_lock) { if (!boot_flags.virt) array = (frames_array_t *) mfn_to_virt_kern(frame->mfn); else { - array = __vmap_paging( - &cr3, - _ptr(_ul(mfn_to_virt_map(frame->mfn)) & PAGE_ORDER_TO_MASK(PAGE_ORDER_4K)), - frame->mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, L1_PROT, true, - paging_lock); + /* switch to special refilling mode to avoid deadlock with paging */ + refilling = true; + /* only paging will be allowed to take the lock while refilling */ + spin_unlock(&lock); + array = vmap_frame_refill(mfn_to_virt_map(frame->mfn), frame->mfn, paging_lock); + spin_lock(&lock); + refilling = false; if (!array) goto error; } @@ -123,13 +136,17 @@ static frames_array_t *new_frames_array(bool paging_lock) { init_frames_array(array); - total_free_frames += array->meta.free_count; return array; error: panic("PMM: Unable to allocate new page for frame array"); UNREACHABLE(); } +static frames_array_t *new_frames_array(void) { + dprintk("%s: new_frames_array\n", __func__); + return _new_frames_array(true); +} + static void del_frames_array(frames_array_t *array) { ASSERT(array); @@ -163,18 +180,7 @@ static inline frames_array_t *get_frames_array(void) { return array; } - return new_frames_array(false); -} - -static inline size_t get_frames_array_space(void) { - size_t space = 0; - frames_array_t *array; - - list_for_each_entry (array, &frames, list) { - space += array->meta.free_count; - } - - return space; + return new_frames_array(); } static inline bool put_frames_array(frames_array_t *array) { @@ -209,7 +215,7 @@ static inline frame_t *take_frame(frame_t *frame, frames_array_t *array) { array->meta.free_count--; if (--total_free_frames <= MIN_FREE_FRAMES_THRESHOLD) - new_frames_array(false); + new_frames_array(); return frame; } @@ -443,7 +449,7 @@ static frame_t *_find_mfn_frame(list_head_t *list, mfn_t mfn, unsigned int order frame_t *find_free_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *frame; - spin_lock(&lock); + spin_lock_norefill(); frame = _find_mfn_frame(free_frames, mfn, order); spin_unlock(&lock); @@ -453,7 +459,7 @@ frame_t *find_free_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *find_busy_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *frame; - spin_lock(&lock); + spin_lock_norefill(); frame = _find_mfn_frame(busy_frames, mfn, order); spin_unlock(&lock); @@ -463,7 +469,7 @@ frame_t *find_busy_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *find_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *frame; - spin_lock(&lock); + spin_lock_norefill(); frame = _find_mfn_frame(busy_frames, mfn, order); if (!frame) frame = _find_mfn_frame(free_frames, mfn, order); @@ -488,7 +494,7 @@ static frame_t *_find_paddr_frame(list_head_t *list, paddr_t paddr) { frame_t *find_free_paddr_frame(paddr_t paddr) { frame_t *frame; - spin_lock(&lock); + spin_lock_norefill(); frame = _find_paddr_frame(free_frames, paddr); spin_unlock(&lock); @@ -498,7 +504,7 @@ frame_t *find_free_paddr_frame(paddr_t paddr) { frame_t *find_busy_paddr_frame(paddr_t paddr) { frame_t *frame; - spin_lock(&lock); + spin_lock_norefill(); frame = _find_paddr_frame(busy_frames, paddr); spin_unlock(&lock); @@ -508,7 +514,7 @@ frame_t *find_busy_paddr_frame(paddr_t paddr) { frame_t *find_paddr_frame(paddr_t paddr) { frame_t *frame; - spin_lock(&lock); + spin_lock_norefill(); frame = _find_paddr_frame(busy_frames, paddr); if (!frame) frame = _find_paddr_frame(free_frames, paddr); @@ -590,6 +596,22 @@ static void merge_frames(frame_t *first) { merge_frames(first); } +static inline bool enough_array_frames(void) { + frames_array_t *array; + size_t count = 0; + + list_for_each_entry (array, &frames, list) { + count += array->meta.free_count; + /* TODO is this number correct ? */ + if (count >= (MIN_FREE_FRAMES_THRESHOLD + MIN_NUM_4K_FRAMES + + (MAX_PAGE_ORDER - PAGE_ORDER_4K))) { + return true; + } + } + + return false; +} + static inline bool enough_4k_frames(void) { frame_t *frame; int count = 0; @@ -616,7 +638,7 @@ static void try_create_4k_frames(void) { * This function does not split larger frames. */ frame_t *get_free_frames_cond(free_frames_cond_t cb) { - spin_lock(&lock); + spin_lock_norefill(); try_create_4k_frames(); for_each_order (order) { frame_t *frame; @@ -640,7 +662,7 @@ frame_t *get_free_frames(unsigned int order) { if (order > MAX_PAGE_ORDER) return NULL; - spin_lock(&lock); + spin_lock_norefill(); if (order == PAGE_ORDER_4K) try_create_4k_frames(); @@ -665,7 +687,7 @@ void put_free_frames(mfn_t mfn, unsigned int order) { ASSERT(order <= MAX_PAGE_ORDER); - spin_lock(&lock); + spin_lock_norefill(); frame = _find_mfn_frame(busy_frames, mfn, order); if (!frame) { warning("PMM: unable to find frame: %lx, order: %u among busy frames", mfn, @@ -680,20 +702,17 @@ void put_free_frames(mfn_t mfn, unsigned int order) { spin_unlock(&lock); } -frame_t *get_backup_frame(void) { - // return get_free_frame(); +frame_t *get_free_frame_norefill(void) { frame_t *frame; spin_lock(&lock); frame = reserve_frame(get_first_frame(free_frames, PAGE_ORDER_4K)); spin_unlock(&lock); - return frame; -} + /* we ran out of reserved frames. increase MIN_NUM_4K_FRAMES */ + BUG_ON(!frame); -frame_t *_get_backup_frame(void) { - // return get_free_frame(); - return reserve_frame(get_first_frame(free_frames, PAGE_ORDER_4K)); + return frame; } void map_frames_array(void) { @@ -709,11 +728,20 @@ void map_frames_array(void) { } void refill_from_paging(void) { - // make sure we have enough space to refill the 4K frames - // so we need space for one frame per - if (get_frames_array_space() < MIN_NUM_4K_FRAMES) { - new_frames_array(false); + + printk("%s: start\n", __func__); + spin_lock(&lock); + + /* if a refill is already active then they are responsible */ + if (!refilling) { + /* make sure we have enough space to refill the 4K frames */ + while (!enough_array_frames()) { + _new_frames_array(false); + } + + /* refill the 4K frames */ + try_create_4k_frames(); } - try_create_4k_frames(); + spin_unlock(&lock); } \ No newline at end of file From f003584169395a422f2d576d964da53535dccf5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20R=C3=BCegge?= Date: Tue, 3 Dec 2024 10:22:32 +0100 Subject: [PATCH 5/8] only block relevant functions during refill --- arch/x86/pagetables.c | 8 ++++++-- mm/pmm.c | 28 +++++++++++++++++----------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c index b1ea54c7..07dbc882 100644 --- a/arch/x86/pagetables.c +++ b/arch/x86/pagetables.c @@ -426,13 +426,17 @@ void *vmap_frame_refill(void *va, mfn_t mfn, bool paging_lock) { dprintk("%s: va: 0x%p mfn: 0x%lx\n", __func__, va, mfn); - ASSERT(vmap_lock); - if (paging_lock) spin_lock(&vmap_lock); + else + ASSERT(vmap_lock); + va = _vmap_4k(&cr3, _ptr(_va), mfn, L1_PROT, false); + if (paging_lock) spin_unlock(&vmap_lock); + else + ASSERT(vmap_lock); return va; } diff --git a/mm/pmm.c b/mm/pmm.c index 81e11b4e..5941262c 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -41,13 +41,15 @@ static frames_array_t early_frames; static list_head_t free_frames[MAX_PAGE_ORDER + 1]; static list_head_t busy_frames[MAX_PAGE_ORDER + 1]; -/* 1 frame for array frame, 3 for mapping it, times to for paging/pmm collisions */ +/* 1 frame for array frame, 3 for mapping it, times two for paging/pmm collisions */ #define MIN_NUM_4K_FRAMES 2 * (1 + 3) static size_t frames_count[MAX_PAGE_ORDER + 1]; static bool refilling; static spinlock_t lock = SPINLOCK_INIT; +static void try_create_4k_frames(void); + static void spin_lock_norefill() { spin_lock(&lock); @@ -110,7 +112,7 @@ static inline void init_frames_array(frames_array_t *array) { total_free_frames += array->meta.free_count; } -static frames_array_t *_new_frames_array(bool paging_lock) { +static frames_array_t *_new_frames_array(bool from_paging) { frames_array_t *array; frame_t *frame; @@ -122,11 +124,13 @@ static frames_array_t *_new_frames_array(bool paging_lock) { array = (frames_array_t *) mfn_to_virt_kern(frame->mfn); else { /* switch to special refilling mode to avoid deadlock with paging */ + ASSERT(!refilling); refilling = true; /* only paging will be allowed to take the lock while refilling */ spin_unlock(&lock); - array = vmap_frame_refill(mfn_to_virt_map(frame->mfn), frame->mfn, paging_lock); + array = vmap_frame_refill(mfn_to_virt_map(frame->mfn), frame->mfn, from_paging); spin_lock(&lock); + ASSERT(refilling); refilling = false; if (!array) goto error; @@ -136,6 +140,9 @@ static frames_array_t *_new_frames_array(bool paging_lock) { init_frames_array(array); + /* since paging will back off from refilling we are responsible to do it here */ + try_create_4k_frames(); + return array; error: panic("PMM: Unable to allocate new page for frame array"); @@ -449,7 +456,7 @@ static frame_t *_find_mfn_frame(list_head_t *list, mfn_t mfn, unsigned int order frame_t *find_free_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *frame; - spin_lock_norefill(); + spin_lock(&lock); frame = _find_mfn_frame(free_frames, mfn, order); spin_unlock(&lock); @@ -459,7 +466,7 @@ frame_t *find_free_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *find_busy_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *frame; - spin_lock_norefill(); + spin_lock(&lock); frame = _find_mfn_frame(busy_frames, mfn, order); spin_unlock(&lock); @@ -469,7 +476,7 @@ frame_t *find_busy_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *find_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *frame; - spin_lock_norefill(); + spin_lock(&lock); frame = _find_mfn_frame(busy_frames, mfn, order); if (!frame) frame = _find_mfn_frame(free_frames, mfn, order); @@ -494,7 +501,7 @@ static frame_t *_find_paddr_frame(list_head_t *list, paddr_t paddr) { frame_t *find_free_paddr_frame(paddr_t paddr) { frame_t *frame; - spin_lock_norefill(); + spin_lock(&lock); frame = _find_paddr_frame(free_frames, paddr); spin_unlock(&lock); @@ -504,7 +511,7 @@ frame_t *find_free_paddr_frame(paddr_t paddr) { frame_t *find_busy_paddr_frame(paddr_t paddr) { frame_t *frame; - spin_lock_norefill(); + spin_lock(&lock); frame = _find_paddr_frame(busy_frames, paddr); spin_unlock(&lock); @@ -514,7 +521,7 @@ frame_t *find_busy_paddr_frame(paddr_t paddr) { frame_t *find_paddr_frame(paddr_t paddr) { frame_t *frame; - spin_lock_norefill(); + spin_lock(&lock); frame = _find_paddr_frame(busy_frames, paddr); if (!frame) frame = _find_paddr_frame(free_frames, paddr); @@ -687,7 +694,7 @@ void put_free_frames(mfn_t mfn, unsigned int order) { ASSERT(order <= MAX_PAGE_ORDER); - spin_lock_norefill(); + spin_lock(&lock); frame = _find_mfn_frame(busy_frames, mfn, order); if (!frame) { warning("PMM: unable to find frame: %lx, order: %u among busy frames", mfn, @@ -729,7 +736,6 @@ void map_frames_array(void) { void refill_from_paging(void) { - printk("%s: start\n", __func__); spin_lock(&lock); /* if a refill is already active then they are responsible */ From 2285c6b3ad8bce427989d80edef7e45629c7de12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20R=C3=BCegge?= Date: Tue, 3 Dec 2024 10:41:52 +0100 Subject: [PATCH 6/8] simplify and remove debug code --- mm/pmm.c | 31 ++++++++----------------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/mm/pmm.c b/mm/pmm.c index 5941262c..c4dc613f 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -41,8 +41,11 @@ static frames_array_t early_frames; static list_head_t free_frames[MAX_PAGE_ORDER + 1]; static list_head_t busy_frames[MAX_PAGE_ORDER + 1]; -/* 1 frame for array frame, 3 for mapping it, times two for paging/pmm collisions */ -#define MIN_NUM_4K_FRAMES 2 * (1 + 3) +/* 3 for paging and 4 for refilling (1 array frame, 3 for mapping it) */ +#define MIN_NUM_4K_FRAMES 3 + (1 + 3) +/* enough frames to refill the 4K frames in worst case without running our of space */ +#define MIN_NOREFILL_FREE_FRAMES_THRESHOLD \ + (MIN_FREE_FRAMES_THRESHOLD + MIN_NUM_4K_FRAMES + (MAX_PAGE_ORDER - PAGE_ORDER_4K)) static size_t frames_count[MAX_PAGE_ORDER + 1]; static bool refilling; @@ -150,7 +153,6 @@ static frames_array_t *_new_frames_array(bool from_paging) { } static frames_array_t *new_frames_array(void) { - dprintk("%s: new_frames_array\n", __func__); return _new_frames_array(true); } @@ -603,22 +605,6 @@ static void merge_frames(frame_t *first) { merge_frames(first); } -static inline bool enough_array_frames(void) { - frames_array_t *array; - size_t count = 0; - - list_for_each_entry (array, &frames, list) { - count += array->meta.free_count; - /* TODO is this number correct ? */ - if (count >= (MIN_FREE_FRAMES_THRESHOLD + MIN_NUM_4K_FRAMES + - (MAX_PAGE_ORDER - PAGE_ORDER_4K))) { - return true; - } - } - - return false; -} - static inline bool enough_4k_frames(void) { frame_t *frame; int count = 0; @@ -735,13 +721,12 @@ void map_frames_array(void) { } void refill_from_paging(void) { - spin_lock(&lock); /* if a refill is already active then they are responsible */ if (!refilling) { - /* make sure we have enough space to refill the 4K frames */ - while (!enough_array_frames()) { + /* ensure enough space to refill 4K frames without frame array allocation */ + while (total_free_frames < MIN_NOREFILL_FREE_FRAMES_THRESHOLD) { _new_frames_array(false); } @@ -750,4 +735,4 @@ void refill_from_paging(void) { } spin_unlock(&lock); -} \ No newline at end of file +} From d01825bde3cacf2cc759497e6d5964082b95ac3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20R=C3=BCegge?= Date: Wed, 4 Dec 2024 10:39:05 +0100 Subject: [PATCH 7/8] fix header --- include/mm/pmm.h | 5 ++--- mm/pmm.c | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/include/mm/pmm.h b/include/mm/pmm.h index baea26cf..f1c020a0 100644 --- a/include/mm/pmm.h +++ b/include/mm/pmm.h @@ -73,6 +73,7 @@ extern void init_pmm(void); extern frame_t *get_free_frames_cond(free_frames_cond_t cb); extern frame_t *get_free_frames(unsigned int order); +extern frame_t *get_free_frame_norefill(void); extern void put_free_frames(mfn_t mfn, unsigned int order); extern void reclaim_frame(mfn_t mfn, unsigned int order); @@ -84,6 +85,7 @@ extern frame_t *find_busy_paddr_frame(paddr_t paddr); extern frame_t *find_paddr_frame(paddr_t paddr); extern void map_frames_array(void); +extern void refill_from_paging(void); /* Static definitions */ @@ -142,9 +144,6 @@ static inline bool is_frame_free(const frame_t *frame) { return frame->flags.free; } -void refill_from_paging(void); -frame_t *get_free_frame_norefill(void); - #endif /* __ASSEMBLY__ */ #endif /* KTF_PMM_H */ diff --git a/mm/pmm.c b/mm/pmm.c index c4dc613f..e22b8d30 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -129,7 +129,7 @@ static frames_array_t *_new_frames_array(bool from_paging) { /* switch to special refilling mode to avoid deadlock with paging */ ASSERT(!refilling); refilling = true; - /* only paging will be allowed to take the lock while refilling */ + /* only paging will be allowed to take the lock for writing while refilling */ spin_unlock(&lock); array = vmap_frame_refill(mfn_to_virt_map(frame->mfn), frame->mfn, from_paging); spin_lock(&lock); From 60e488b9d7ba31532f9b02836f11cab673338298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20R=C3=BCegge?= Date: Thu, 5 Dec 2024 08:26:06 +0100 Subject: [PATCH 8/8] pmm: add priority access for paging --- mm/pmm.c | 96 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 52 insertions(+), 44 deletions(-) diff --git a/mm/pmm.c b/mm/pmm.c index e22b8d30..f02834f8 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -48,19 +48,21 @@ static list_head_t busy_frames[MAX_PAGE_ORDER + 1]; (MIN_FREE_FRAMES_THRESHOLD + MIN_NUM_4K_FRAMES + (MAX_PAGE_ORDER - PAGE_ORDER_4K)) static size_t frames_count[MAX_PAGE_ORDER + 1]; -static bool refilling; static spinlock_t lock = SPINLOCK_INIT; +static bool refilling; +static spinlock_t priority_lock = SPINLOCK_INIT; + static void try_create_4k_frames(void); -static void spin_lock_norefill() { +static void spin_lock_standard() { spin_lock(&lock); + spin_lock(&priority_lock); +} - while (refilling) { - spin_unlock(&lock); - /* TODO: some backoff would be nice here... */ - spin_lock(&lock); - } +static void spin_unlock_standard() { + spin_unlock(&priority_lock); + spin_unlock(&lock); } void display_frames_count(void) { @@ -129,12 +131,16 @@ static frames_array_t *_new_frames_array(bool from_paging) { /* switch to special refilling mode to avoid deadlock with paging */ ASSERT(!refilling); refilling = true; + /* only paging will be allowed to take the lock for writing while refilling */ - spin_unlock(&lock); + spin_unlock(&priority_lock); array = vmap_frame_refill(mfn_to_virt_map(frame->mfn), frame->mfn, from_paging); - spin_lock(&lock); + spin_lock(&priority_lock); + + /* switch back to normal mode */ ASSERT(refilling); refilling = false; + if (!array) goto error; } @@ -458,9 +464,9 @@ static frame_t *_find_mfn_frame(list_head_t *list, mfn_t mfn, unsigned int order frame_t *find_free_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *frame; - spin_lock(&lock); + spin_lock_standard(); frame = _find_mfn_frame(free_frames, mfn, order); - spin_unlock(&lock); + spin_unlock_standard(); return frame; } @@ -468,9 +474,9 @@ frame_t *find_free_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *find_busy_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *frame; - spin_lock(&lock); + spin_lock_standard(); frame = _find_mfn_frame(busy_frames, mfn, order); - spin_unlock(&lock); + spin_unlock_standard(); return frame; } @@ -478,11 +484,11 @@ frame_t *find_busy_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *find_mfn_frame(mfn_t mfn, unsigned int order) { frame_t *frame; - spin_lock(&lock); + spin_lock_standard(); frame = _find_mfn_frame(busy_frames, mfn, order); if (!frame) frame = _find_mfn_frame(free_frames, mfn, order); - spin_unlock(&lock); + spin_unlock_standard(); return frame; } @@ -503,9 +509,9 @@ static frame_t *_find_paddr_frame(list_head_t *list, paddr_t paddr) { frame_t *find_free_paddr_frame(paddr_t paddr) { frame_t *frame; - spin_lock(&lock); + spin_lock_standard(); frame = _find_paddr_frame(free_frames, paddr); - spin_unlock(&lock); + spin_unlock_standard(); return frame; } @@ -513,9 +519,9 @@ frame_t *find_free_paddr_frame(paddr_t paddr) { frame_t *find_busy_paddr_frame(paddr_t paddr) { frame_t *frame; - spin_lock(&lock); + spin_lock_standard(); frame = _find_paddr_frame(busy_frames, paddr); - spin_unlock(&lock); + spin_unlock_standard(); return frame; } @@ -523,11 +529,11 @@ frame_t *find_busy_paddr_frame(paddr_t paddr) { frame_t *find_paddr_frame(paddr_t paddr) { frame_t *frame; - spin_lock(&lock); + spin_lock_standard(); frame = _find_paddr_frame(busy_frames, paddr); if (!frame) frame = _find_paddr_frame(free_frames, paddr); - spin_unlock(&lock); + spin_unlock_standard(); return frame; } @@ -631,7 +637,7 @@ static void try_create_4k_frames(void) { * This function does not split larger frames. */ frame_t *get_free_frames_cond(free_frames_cond_t cb) { - spin_lock_norefill(); + spin_lock_standard(); try_create_4k_frames(); for_each_order (order) { frame_t *frame; @@ -639,12 +645,12 @@ frame_t *get_free_frames_cond(free_frames_cond_t cb) { list_for_each_entry (frame, &free_frames[order], list) { if (cb(frame)) { reserve_frame(frame); - spin_unlock(&lock); + spin_unlock_standard(); return frame; } } } - spin_unlock(&lock); + spin_unlock_standard(); return NULL; } @@ -655,7 +661,7 @@ frame_t *get_free_frames(unsigned int order) { if (order > MAX_PAGE_ORDER) return NULL; - spin_lock_norefill(); + spin_lock_standard(); if (order == PAGE_ORDER_4K) try_create_4k_frames(); @@ -663,14 +669,14 @@ frame_t *get_free_frames(unsigned int order) { BUG_ON(order == PAGE_ORDER_4K); frame = find_larger_frame(free_frames, order); if (!frame) { - spin_unlock(&lock); + spin_unlock_standard(); return NULL; } split_frame(frame); } frame = reserve_frame(get_first_frame(free_frames, order)); - spin_unlock(&lock); + spin_unlock_standard(); return frame; } @@ -680,7 +686,7 @@ void put_free_frames(mfn_t mfn, unsigned int order) { ASSERT(order <= MAX_PAGE_ORDER); - spin_lock(&lock); + spin_lock_standard(); frame = _find_mfn_frame(busy_frames, mfn, order); if (!frame) { warning("PMM: unable to find frame: %lx, order: %u among busy frames", mfn, @@ -692,20 +698,7 @@ void put_free_frames(mfn_t mfn, unsigned int order) { merge_frames(frame); unlock: - spin_unlock(&lock); -} - -frame_t *get_free_frame_norefill(void) { - frame_t *frame; - - spin_lock(&lock); - frame = reserve_frame(get_first_frame(free_frames, PAGE_ORDER_4K)); - spin_unlock(&lock); - - /* we ran out of reserved frames. increase MIN_NUM_4K_FRAMES */ - BUG_ON(!frame); - - return frame; + spin_unlock_standard(); } void map_frames_array(void) { @@ -720,8 +713,23 @@ void map_frames_array(void) { } } +/* functions for paging to avoid deadlocks */ + +frame_t *get_free_frame_norefill(void) { + frame_t *frame; + + spin_lock(&priority_lock); + frame = reserve_frame(get_first_frame(free_frames, PAGE_ORDER_4K)); + spin_unlock(&priority_lock); + + /* we ran out of reserved frames. increase MIN_NUM_4K_FRAMES */ + BUG_ON(!frame); + + return frame; +} + void refill_from_paging(void) { - spin_lock(&lock); + spin_lock(&priority_lock); /* if a refill is already active then they are responsible */ if (!refilling) { @@ -734,5 +742,5 @@ void refill_from_paging(void) { try_create_4k_frames(); } - spin_unlock(&lock); + spin_unlock(&priority_lock); }