From 646208fbb008da5b57b82b2e6fe0b7fdec435f66 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Fri, 17 Nov 2023 10:29:21 +0100 Subject: [PATCH 1/4] arch,pt: add PT_LEVELS macro defining page table levels Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/pagetables.c | 13 ++----------- include/arch/x86/page.h | 6 ++++++ 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c index ed2af3bf..23c0e754 100644 --- a/arch/x86/pagetables.c +++ b/arch/x86/pagetables.c @@ -121,11 +121,6 @@ static void dump_pagetable(mfn_t table, int level) { } void dump_pagetables(cr3_t *cr3_ptr) { -#if defined(__x86_64__) - int level = 4; -#else - int level = 3; -#endif ASSERT(cr3_ptr); if (mfn_invalid(cr3_ptr->mfn)) { warning("CR3: 0x%lx is invalid", cr3.paddr); @@ -134,18 +129,14 @@ void dump_pagetables(cr3_t *cr3_ptr) { printk("Page Tables: CR3 paddr: 0x%lx\n", cr3.paddr); spin_lock(&vmap_lock); - dump_pagetable(cr3_ptr->mfn, level); + dump_pagetable(cr3_ptr->mfn, PT_LEVELS); spin_unlock(&vmap_lock); } static void dump_pagetable_va(cr3_t *cr3_ptr, void *va) { paddr_t tab_paddr; pgentry_t *tab; -#if defined(__x86_64__) - int level = 4; -#else - int level = 3; -#endif + int level = PT_LEVELS; ASSERT(cr3_ptr); if (mfn_invalid(cr3_ptr->mfn)) { diff --git a/include/arch/x86/page.h b/include/arch/x86/page.h index edb7c668..7b3b8f30 100644 --- a/include/arch/x86/page.h +++ b/include/arch/x86/page.h @@ -158,6 +158,12 @@ typedef unsigned long mfn_t; #define VA_BITS 32 #endif +#ifdef __i386__ +#define PT_LEVELS 3 /* assumes PAE */ +#else +#define PT_LEVELS (la57_enabled() ? 5 : 4) +#endif + #define _paddr(addr) ((paddr_t) _ul(addr)) #define PADDR_INVALID (~0x0UL) From a2e11ae25ddcf21eca15d9f149d0880afa74e94e Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Tue, 31 Oct 2023 14:46:04 +0100 Subject: [PATCH 2/4] arch,pt: add page tables map/unmap functions The map_pagetables() function maps all page tables specified in its second parameter into the address space specified by its first parameter. If second parameter is NULL, the current address space page tables are mapped. Only page table frames are being mapped. After mapping a helper function family: get_pte(), get_pde(), ... can be used without causing exceptions. The unmap_pagetables() function unmaps all page tables specified in its second parameter from address space indicated by its first parameter. If second parameter is NULL, the current address space page tables are being unmapped. Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/pagetables.c | 92 ++++++++++++++++++++++++++++++++++++ include/arch/x86/pagetable.h | 11 +++++ tests/unittests.c | 7 +++ 3 files changed, 110 insertions(+) diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c index 23c0e754..e2f3b388 100644 --- a/arch/x86/pagetables.c +++ b/arch/x86/pagetables.c @@ -542,6 +542,98 @@ static inline void init_cr3(cr3_t *cr3_ptr) { cr3_ptr->mfn = MFN_INVALID; } +static inline bool is_huge_page_level(int level) { + if (level == 2) + return true; +#if defined(__i386__) + return false; +#else + return level == 3; +#endif +} + +static void map_pagetable(cr3_t *cr3_ptr, mfn_t table, int level) { + void *va = mfn_to_virt_kern(table); + pte_t *pt; + + pt = _vmap(cr3_ptr, va, table, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, L1_PROT); + BUG_ON(!pt); + + for (int i = 0; i < level_to_entries(level) && level > 1; i++) { + if (mfn_invalid(pt[i].mfn)) + continue; + + /* Do not map 2MB/1GB large pages */ + if (!is_huge_page_level(level) || !is_pgentry_huge(pt[i].entry)) + map_pagetable(cr3_ptr, pt[i].mfn, level - 1); + } +} + +void map_pagetables(cr3_t *to_cr3, cr3_t *from_cr3) { + ASSERT(to_cr3); + if (mfn_invalid(to_cr3->mfn)) { + warning("Target CR3: 0x%lx is invalid", to_cr3->paddr); + return; + } + + if (!from_cr3) + from_cr3 = to_cr3; + else if (mfn_invalid(from_cr3->mfn)) { + warning("Source CR3: 0x%lx is invalid", from_cr3->paddr); + return; + } + + dprintk("Mapping all page tables of CR3: 0x%lx to CR3: 0x%lx\n", from_cr3->paddr, + to_cr3->paddr); + + spin_lock(&vmap_lock); + /* Assume PML4 is not mapped */ + map_pagetable(to_cr3, from_cr3->mfn, PT_LEVELS); + spin_unlock(&vmap_lock); +} + +static void unmap_pagetable(cr3_t *cr3_ptr, mfn_t table, int level) { + mfn_t tmp_entry_mfn = virt_to_mfn(_tmp_mapping_entry); + pte_t *pt = mfn_to_virt_kern(table); + + for (int i = 0; i < level_to_entries(level) && level > 1; i++) { + if (mfn_invalid(pt[i].mfn)) + continue; + + /* Do not touch the tmp_mapping entry! */ + if (pt[i].mfn == tmp_entry_mfn) + continue; + + /* Do not touch 2MB/1GB large pages */ + if (!is_huge_page_level(level) || !is_pgentry_huge(pt[i].entry)) + unmap_pagetable(cr3_ptr, pt[i].mfn, level - 1); + } + _vunmap(cr3_ptr, pt, NULL, NULL); +} + +void unmap_pagetables(cr3_t *from_cr3, cr3_t *of_cr3) { + ASSERT(from_cr3); + if (mfn_invalid(from_cr3->mfn)) { + warning("Target CR3: 0x%lx is invalid", from_cr3->paddr); + return; + } + + if (!of_cr3) + of_cr3 = from_cr3; + else if (mfn_invalid(of_cr3->mfn)) { + warning("Source CR3: 0x%lx is invalid", of_cr3->paddr); + return; + } + + dprintk("Unmapping all page tables of CR3: 0x%lx from CR3: 0x%lx\n", of_cr3->paddr, + from_cr3->paddr); + + spin_lock(&vmap_lock); + /* Assume PML4 is mapped */ + unmap_pagetable(from_cr3, of_cr3->mfn, PT_LEVELS); + spin_unlock(&vmap_lock); +} + void init_pagetables(void) { init_cr3(&cr3); init_cr3(&user_cr3); diff --git a/include/arch/x86/pagetable.h b/include/arch/x86/pagetable.h index 405d7973..71cd15c3 100644 --- a/include/arch/x86/pagetable.h +++ b/include/arch/x86/pagetable.h @@ -284,6 +284,14 @@ static inline void set_pgentry(pgentry_t *e, mfn_t mfn, unsigned long flags) { barrier(); } +static inline bool is_pgentry_huge(pgentry_t e) { + return !!(e & _PAGE_PSE); +} + +static inline bool is_pgentry_present(pgentry_t e) { + return !!(e & _PAGE_PRESENT); +} + /* External declarations */ extern pte_t l1_pt_entries1[L1_PT_ENTRIES]; @@ -307,6 +315,9 @@ extern int get_user_va_mfn_order(void *va, mfn_t *mfn, unsigned int *order); extern frame_t *find_kern_va_frame(const void *va); extern frame_t *find_user_va_frame(const void *va); +extern void map_pagetables(cr3_t *to_cr3, cr3_t *from_cr3); +extern void unmap_pagetables(cr3_t *from_cr3, cr3_t *of_cr3); + #endif /* __ASSEMBLY__ */ #endif /* KTF_PAGETABLE_H */ diff --git a/tests/unittests.c b/tests/unittests.c index 28e9b3cf..d5ff8d2d 100644 --- a/tests/unittests.c +++ b/tests/unittests.c @@ -190,6 +190,13 @@ int unit_tests(void *_unused) { cpu_freq_expect("Prototyp Amazing Foo One @ 1GHz", 1000000000); cpu_freq_expect("Prototyp Amazing Foo Two @ 1.00GHz", 1000000000); + map_pagetables(&cr3, NULL); + map_pagetables(&cr3, &user_cr3); + pte_t *pte = get_pte(unit_tests); + printk("PTE: 0x%lx\n", pte->entry); + unmap_pagetables(&cr3, NULL); + unmap_pagetables(&cr3, &user_cr3); + task_t *task1, *task2, *task_user1, *task_user1_se, *task_user1_int80, *task_user2, *task_user3, *task_user4; From 960f538a830340b4c555f0ca6e3dbc716259d8c9 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Tue, 31 Oct 2023 14:46:38 +0100 Subject: [PATCH 3/4] arch, pt: add va specific page tables map/unmap functions The map_pagetables_va() function maps page tables pertinent to the virtual address specified by its second parameter into the address space indicated by its first parameter. It is a fine-grained, lightweight version of the map_pagetables(). The unmap_pagetables_va() function unmaps page tables pertinent to the virtual address specified by its second parameter into the address space indicated by its first parameter. It is a fine-grained, lightweight version of the unmap_pagetables(). The unmap_pagetables_va() performs the unmapping regardless of the mapping status of any of the page tables levels. Signed-off-by: Pawel Wieczorkiewicz --- arch/x86/pagetables.c | 146 +++++++++++++++++++++++++++++++++++ include/arch/x86/pagetable.h | 2 + tests/unittests.c | 5 ++ 3 files changed, 153 insertions(+) diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c index e2f3b388..755b0b82 100644 --- a/arch/x86/pagetables.c +++ b/arch/x86/pagetables.c @@ -634,6 +634,152 @@ void unmap_pagetables(cr3_t *from_cr3, cr3_t *of_cr3) { spin_unlock(&vmap_lock); } +int map_pagetables_va(cr3_t *cr3_ptr, void *va) { + pgentry_t *tab; + int err = -EINVAL; + + ASSERT(cr3_ptr); + if (mfn_invalid(cr3_ptr->mfn)) { + warning("CR3: 0x%lx is invalid", cr3_ptr->paddr); + return err; + } + + if (!is_canon_va(va)) { + warning("Virtual address 0x%p is not canonical", va); + return err; + } + + err = -EFAULT; + spin_lock(&vmap_lock); + tab = _vmap(cr3_ptr, mfn_to_virt_kern(cr3_ptr->mfn), cr3_ptr->mfn, PAGE_ORDER_4K, + L4_PROT, L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto unlock; + +#if defined(__x86_64__) + pml4_t *l4e = l4_table_entry((pml4_t *) tab, va); + if (mfn_invalid(l4e->mfn)) { + err = -ENOENT; + goto unlock; + } + + tab = _vmap(cr3_ptr, mfn_to_virt_kern(l4e->mfn), l4e->mfn, PAGE_ORDER_4K, L4_PROT, + L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto unlock; +#endif + + pdpe_t *l3e = l3_table_entry((pdpe_t *) tab, va); + if (mfn_invalid(l3e->mfn)) { + err = -ENOENT; + goto unlock; + } + + if (l3e->PS) + goto done; + + tab = _vmap(cr3_ptr, mfn_to_virt_kern(l3e->mfn), l3e->mfn, PAGE_ORDER_4K, L4_PROT, + L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto unlock; + + pde_t *l2e = l2_table_entry((pde_t *) tab, va); + if (mfn_invalid(l2e->mfn)) { + err = -ENOENT; + goto unlock; + } + + if (l2e->PS) + goto done; + + tab = _vmap(cr3_ptr, mfn_to_virt_kern(l2e->mfn), l2e->mfn, PAGE_ORDER_4K, L4_PROT, + L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto unlock; + +done: + err = 0; +unlock: + spin_unlock(&vmap_lock); + return err; +} + +int unmap_pagetables_va(cr3_t *cr3_ptr, void *va) { + mfn_t tmp_entry_mfn = virt_to_mfn(_tmp_mapping_entry); + pgentry_t *tab, *tables[PT_LEVELS] = {NULL}; + int level = 0; + int err = -EINVAL; + + ASSERT(cr3_ptr); + if (mfn_invalid(cr3_ptr->mfn)) { + warning("CR3: 0x%lx is invalid", cr3_ptr->paddr); + return err; + } + + if (!is_canon_va(va)) { + warning("Virtual address 0x%p is not canonical", va); + return err; + } + + err = -EFAULT; + spin_lock(&vmap_lock); + tab = _vmap(cr3_ptr, mfn_to_virt_kern(cr3_ptr->mfn), cr3_ptr->mfn, PAGE_ORDER_4K, + L4_PROT, L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto cleanup; + tables[level++] = tab; + +#if defined(__x86_64__) + pml4_t *l4e = l4_table_entry((pml4_t *) tab, va); + if (mfn_invalid(l4e->mfn)) { + err = -ENOENT; + goto cleanup; + } + + tab = _vmap(cr3_ptr, mfn_to_virt_kern(l4e->mfn), l4e->mfn, PAGE_ORDER_4K, L4_PROT, + L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto cleanup; + tables[level++] = tab; +#endif + + pdpe_t *l3e = l3_table_entry((pdpe_t *) tab, va); + if (mfn_invalid(l3e->mfn)) { + err = -ENOENT; + goto cleanup; + } + + if (l3e->PS) + goto done; + + tab = _vmap(cr3_ptr, mfn_to_virt_kern(l3e->mfn), l3e->mfn, PAGE_ORDER_4K, L4_PROT, + L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto cleanup; + tables[level++] = tab; + + pde_t *l2e = l2_table_entry((pde_t *) tab, va); + if (mfn_invalid(l2e->mfn)) { + err = -ENOENT; + goto cleanup; + } + + if (l2e->PS) + goto done; + + /* Do not touch the tmp_mapping entry! */ + if (l2e->mfn != tmp_entry_mfn) + tables[level] = mfn_to_virt_kern(l2e->mfn); + +done: + err = 0; +cleanup: + for (unsigned i = 0; i < ARRAY_SIZE(tables) && tables[i]; i++) + _vunmap(cr3_ptr, tables[i], NULL, NULL); + spin_unlock(&vmap_lock); + return err; +} + void init_pagetables(void) { init_cr3(&cr3); init_cr3(&user_cr3); diff --git a/include/arch/x86/pagetable.h b/include/arch/x86/pagetable.h index 71cd15c3..eceae80b 100644 --- a/include/arch/x86/pagetable.h +++ b/include/arch/x86/pagetable.h @@ -317,6 +317,8 @@ extern frame_t *find_user_va_frame(const void *va); extern void map_pagetables(cr3_t *to_cr3, cr3_t *from_cr3); extern void unmap_pagetables(cr3_t *from_cr3, cr3_t *of_cr3); +extern int map_pagetables_va(cr3_t *cr3_ptr, void *va); +extern int unmap_pagetables_va(cr3_t *cr3_ptr, void *va); #endif /* __ASSEMBLY__ */ diff --git a/tests/unittests.c b/tests/unittests.c index d5ff8d2d..f0b321fb 100644 --- a/tests/unittests.c +++ b/tests/unittests.c @@ -197,6 +197,11 @@ int unit_tests(void *_unused) { unmap_pagetables(&cr3, NULL); unmap_pagetables(&cr3, &user_cr3); + map_pagetables_va(&cr3, unit_tests); + pte_t *pte2 = get_pte(unit_tests); + printk("PTE: 0x%lx\n", pte2->entry); + unmap_pagetables_va(&cr3, unit_tests); + task_t *task1, *task2, *task_user1, *task_user1_se, *task_user1_int80, *task_user2, *task_user3, *task_user4; From 76077d2d57ce894758739f81b40f0b524dea4a21 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Tue, 31 Oct 2023 14:41:06 +0100 Subject: [PATCH 4/4] mm,pmm: remove map_used_memory() completely Finally, this ugly hack goes away. Signed-off-by: Pawel Wieczorkiewicz --- include/mm/pmm.h | 3 +-- mm/pmm.c | 13 ------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/include/mm/pmm.h b/include/mm/pmm.h index 79f193d9..246c225c 100644 --- a/include/mm/pmm.h +++ b/include/mm/pmm.h @@ -31,7 +31,7 @@ #include struct frame_flags { - uint16_t : 11, mapped : 1, uncacheable : 1, free : 1, pagetable : 1; + uint16_t : 12, uncacheable : 1, free : 1, pagetable : 1; }; typedef struct frame_flags frame_flags_t; @@ -83,7 +83,6 @@ extern frame_t *find_free_paddr_frame(paddr_t paddr); extern frame_t *find_busy_paddr_frame(paddr_t paddr); extern frame_t *find_paddr_frame(paddr_t paddr); -extern void map_used_memory(void); extern void map_frames_array(void); /* Static definitions */ diff --git a/mm/pmm.c b/mm/pmm.c index 6158614a..6622f805 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -638,19 +638,6 @@ void put_free_frames(mfn_t mfn, unsigned int order) { spin_unlock(&lock); } -void map_used_memory(void) { - frame_t *frame; - - for_each_order (order) { - list_for_each_entry (frame, &busy_frames[order], list) { - if (!frame->flags.mapped) { - kmap(frame->mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT); - frame->flags.mapped = true; - } - } - } -} - void map_frames_array(void) { frames_array_t *array;