diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c index ed2af3bf..755b0b82 100644 --- a/arch/x86/pagetables.c +++ b/arch/x86/pagetables.c @@ -121,11 +121,6 @@ static void dump_pagetable(mfn_t table, int level) { } void dump_pagetables(cr3_t *cr3_ptr) { -#if defined(__x86_64__) - int level = 4; -#else - int level = 3; -#endif ASSERT(cr3_ptr); if (mfn_invalid(cr3_ptr->mfn)) { warning("CR3: 0x%lx is invalid", cr3.paddr); @@ -134,18 +129,14 @@ void dump_pagetables(cr3_t *cr3_ptr) { printk("Page Tables: CR3 paddr: 0x%lx\n", cr3.paddr); spin_lock(&vmap_lock); - dump_pagetable(cr3_ptr->mfn, level); + dump_pagetable(cr3_ptr->mfn, PT_LEVELS); spin_unlock(&vmap_lock); } static void dump_pagetable_va(cr3_t *cr3_ptr, void *va) { paddr_t tab_paddr; pgentry_t *tab; -#if defined(__x86_64__) - int level = 4; -#else - int level = 3; -#endif + int level = PT_LEVELS; ASSERT(cr3_ptr); if (mfn_invalid(cr3_ptr->mfn)) { @@ -551,6 +542,244 @@ static inline void init_cr3(cr3_t *cr3_ptr) { cr3_ptr->mfn = MFN_INVALID; } +static inline bool is_huge_page_level(int level) { + if (level == 2) + return true; +#if defined(__i386__) + return false; +#else + return level == 3; +#endif +} + +static void map_pagetable(cr3_t *cr3_ptr, mfn_t table, int level) { + void *va = mfn_to_virt_kern(table); + pte_t *pt; + + pt = _vmap(cr3_ptr, va, table, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, L1_PROT); + BUG_ON(!pt); + + for (int i = 0; i < level_to_entries(level) && level > 1; i++) { + if (mfn_invalid(pt[i].mfn)) + continue; + + /* Do not map 2MB/1GB large pages */ + if (!is_huge_page_level(level) || !is_pgentry_huge(pt[i].entry)) + map_pagetable(cr3_ptr, pt[i].mfn, level - 1); + } +} + +void map_pagetables(cr3_t *to_cr3, cr3_t *from_cr3) { + ASSERT(to_cr3); + if (mfn_invalid(to_cr3->mfn)) { + warning("Target CR3: 0x%lx is invalid", to_cr3->paddr); + return; + } + + if (!from_cr3) + from_cr3 = to_cr3; + else if (mfn_invalid(from_cr3->mfn)) { + warning("Source CR3: 0x%lx is invalid", from_cr3->paddr); + return; + } + + dprintk("Mapping all page tables of CR3: 0x%lx to CR3: 0x%lx\n", from_cr3->paddr, + to_cr3->paddr); + + spin_lock(&vmap_lock); + /* Assume PML4 is not mapped */ + map_pagetable(to_cr3, from_cr3->mfn, PT_LEVELS); + spin_unlock(&vmap_lock); +} + +static void unmap_pagetable(cr3_t *cr3_ptr, mfn_t table, int level) { + mfn_t tmp_entry_mfn = virt_to_mfn(_tmp_mapping_entry); + pte_t *pt = mfn_to_virt_kern(table); + + for (int i = 0; i < level_to_entries(level) && level > 1; i++) { + if (mfn_invalid(pt[i].mfn)) + continue; + + /* Do not touch the tmp_mapping entry! */ + if (pt[i].mfn == tmp_entry_mfn) + continue; + + /* Do not touch 2MB/1GB large pages */ + if (!is_huge_page_level(level) || !is_pgentry_huge(pt[i].entry)) + unmap_pagetable(cr3_ptr, pt[i].mfn, level - 1); + } + _vunmap(cr3_ptr, pt, NULL, NULL); +} + +void unmap_pagetables(cr3_t *from_cr3, cr3_t *of_cr3) { + ASSERT(from_cr3); + if (mfn_invalid(from_cr3->mfn)) { + warning("Target CR3: 0x%lx is invalid", from_cr3->paddr); + return; + } + + if (!of_cr3) + of_cr3 = from_cr3; + else if (mfn_invalid(of_cr3->mfn)) { + warning("Source CR3: 0x%lx is invalid", of_cr3->paddr); + return; + } + + dprintk("Unmapping all page tables of CR3: 0x%lx from CR3: 0x%lx\n", of_cr3->paddr, + from_cr3->paddr); + + spin_lock(&vmap_lock); + /* Assume PML4 is mapped */ + unmap_pagetable(from_cr3, of_cr3->mfn, PT_LEVELS); + spin_unlock(&vmap_lock); +} + +int map_pagetables_va(cr3_t *cr3_ptr, void *va) { + pgentry_t *tab; + int err = -EINVAL; + + ASSERT(cr3_ptr); + if (mfn_invalid(cr3_ptr->mfn)) { + warning("CR3: 0x%lx is invalid", cr3_ptr->paddr); + return err; + } + + if (!is_canon_va(va)) { + warning("Virtual address 0x%p is not canonical", va); + return err; + } + + err = -EFAULT; + spin_lock(&vmap_lock); + tab = _vmap(cr3_ptr, mfn_to_virt_kern(cr3_ptr->mfn), cr3_ptr->mfn, PAGE_ORDER_4K, + L4_PROT, L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto unlock; + +#if defined(__x86_64__) + pml4_t *l4e = l4_table_entry((pml4_t *) tab, va); + if (mfn_invalid(l4e->mfn)) { + err = -ENOENT; + goto unlock; + } + + tab = _vmap(cr3_ptr, mfn_to_virt_kern(l4e->mfn), l4e->mfn, PAGE_ORDER_4K, L4_PROT, + L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto unlock; +#endif + + pdpe_t *l3e = l3_table_entry((pdpe_t *) tab, va); + if (mfn_invalid(l3e->mfn)) { + err = -ENOENT; + goto unlock; + } + + if (l3e->PS) + goto done; + + tab = _vmap(cr3_ptr, mfn_to_virt_kern(l3e->mfn), l3e->mfn, PAGE_ORDER_4K, L4_PROT, + L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto unlock; + + pde_t *l2e = l2_table_entry((pde_t *) tab, va); + if (mfn_invalid(l2e->mfn)) { + err = -ENOENT; + goto unlock; + } + + if (l2e->PS) + goto done; + + tab = _vmap(cr3_ptr, mfn_to_virt_kern(l2e->mfn), l2e->mfn, PAGE_ORDER_4K, L4_PROT, + L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto unlock; + +done: + err = 0; +unlock: + spin_unlock(&vmap_lock); + return err; +} + +int unmap_pagetables_va(cr3_t *cr3_ptr, void *va) { + mfn_t tmp_entry_mfn = virt_to_mfn(_tmp_mapping_entry); + pgentry_t *tab, *tables[PT_LEVELS] = {NULL}; + int level = 0; + int err = -EINVAL; + + ASSERT(cr3_ptr); + if (mfn_invalid(cr3_ptr->mfn)) { + warning("CR3: 0x%lx is invalid", cr3_ptr->paddr); + return err; + } + + if (!is_canon_va(va)) { + warning("Virtual address 0x%p is not canonical", va); + return err; + } + + err = -EFAULT; + spin_lock(&vmap_lock); + tab = _vmap(cr3_ptr, mfn_to_virt_kern(cr3_ptr->mfn), cr3_ptr->mfn, PAGE_ORDER_4K, + L4_PROT, L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto cleanup; + tables[level++] = tab; + +#if defined(__x86_64__) + pml4_t *l4e = l4_table_entry((pml4_t *) tab, va); + if (mfn_invalid(l4e->mfn)) { + err = -ENOENT; + goto cleanup; + } + + tab = _vmap(cr3_ptr, mfn_to_virt_kern(l4e->mfn), l4e->mfn, PAGE_ORDER_4K, L4_PROT, + L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto cleanup; + tables[level++] = tab; +#endif + + pdpe_t *l3e = l3_table_entry((pdpe_t *) tab, va); + if (mfn_invalid(l3e->mfn)) { + err = -ENOENT; + goto cleanup; + } + + if (l3e->PS) + goto done; + + tab = _vmap(cr3_ptr, mfn_to_virt_kern(l3e->mfn), l3e->mfn, PAGE_ORDER_4K, L4_PROT, + L3_PROT, L2_PROT, L1_PROT); + if (!tab) + goto cleanup; + tables[level++] = tab; + + pde_t *l2e = l2_table_entry((pde_t *) tab, va); + if (mfn_invalid(l2e->mfn)) { + err = -ENOENT; + goto cleanup; + } + + if (l2e->PS) + goto done; + + /* Do not touch the tmp_mapping entry! */ + if (l2e->mfn != tmp_entry_mfn) + tables[level] = mfn_to_virt_kern(l2e->mfn); + +done: + err = 0; +cleanup: + for (unsigned i = 0; i < ARRAY_SIZE(tables) && tables[i]; i++) + _vunmap(cr3_ptr, tables[i], NULL, NULL); + spin_unlock(&vmap_lock); + return err; +} + void init_pagetables(void) { init_cr3(&cr3); init_cr3(&user_cr3); diff --git a/include/arch/x86/page.h b/include/arch/x86/page.h index edb7c668..7b3b8f30 100644 --- a/include/arch/x86/page.h +++ b/include/arch/x86/page.h @@ -158,6 +158,12 @@ typedef unsigned long mfn_t; #define VA_BITS 32 #endif +#ifdef __i386__ +#define PT_LEVELS 3 /* assumes PAE */ +#else +#define PT_LEVELS (la57_enabled() ? 5 : 4) +#endif + #define _paddr(addr) ((paddr_t) _ul(addr)) #define PADDR_INVALID (~0x0UL) diff --git a/include/arch/x86/pagetable.h b/include/arch/x86/pagetable.h index 405d7973..eceae80b 100644 --- a/include/arch/x86/pagetable.h +++ b/include/arch/x86/pagetable.h @@ -284,6 +284,14 @@ static inline void set_pgentry(pgentry_t *e, mfn_t mfn, unsigned long flags) { barrier(); } +static inline bool is_pgentry_huge(pgentry_t e) { + return !!(e & _PAGE_PSE); +} + +static inline bool is_pgentry_present(pgentry_t e) { + return !!(e & _PAGE_PRESENT); +} + /* External declarations */ extern pte_t l1_pt_entries1[L1_PT_ENTRIES]; @@ -307,6 +315,11 @@ extern int get_user_va_mfn_order(void *va, mfn_t *mfn, unsigned int *order); extern frame_t *find_kern_va_frame(const void *va); extern frame_t *find_user_va_frame(const void *va); +extern void map_pagetables(cr3_t *to_cr3, cr3_t *from_cr3); +extern void unmap_pagetables(cr3_t *from_cr3, cr3_t *of_cr3); +extern int map_pagetables_va(cr3_t *cr3_ptr, void *va); +extern int unmap_pagetables_va(cr3_t *cr3_ptr, void *va); + #endif /* __ASSEMBLY__ */ #endif /* KTF_PAGETABLE_H */ diff --git a/include/mm/pmm.h b/include/mm/pmm.h index 79f193d9..246c225c 100644 --- a/include/mm/pmm.h +++ b/include/mm/pmm.h @@ -31,7 +31,7 @@ #include struct frame_flags { - uint16_t : 11, mapped : 1, uncacheable : 1, free : 1, pagetable : 1; + uint16_t : 12, uncacheable : 1, free : 1, pagetable : 1; }; typedef struct frame_flags frame_flags_t; @@ -83,7 +83,6 @@ extern frame_t *find_free_paddr_frame(paddr_t paddr); extern frame_t *find_busy_paddr_frame(paddr_t paddr); extern frame_t *find_paddr_frame(paddr_t paddr); -extern void map_used_memory(void); extern void map_frames_array(void); /* Static definitions */ diff --git a/mm/pmm.c b/mm/pmm.c index 6158614a..6622f805 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -638,19 +638,6 @@ void put_free_frames(mfn_t mfn, unsigned int order) { spin_unlock(&lock); } -void map_used_memory(void) { - frame_t *frame; - - for_each_order (order) { - list_for_each_entry (frame, &busy_frames[order], list) { - if (!frame->flags.mapped) { - kmap(frame->mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT); - frame->flags.mapped = true; - } - } - } -} - void map_frames_array(void) { frames_array_t *array; diff --git a/tests/unittests.c b/tests/unittests.c index 28e9b3cf..f0b321fb 100644 --- a/tests/unittests.c +++ b/tests/unittests.c @@ -190,6 +190,18 @@ int unit_tests(void *_unused) { cpu_freq_expect("Prototyp Amazing Foo One @ 1GHz", 1000000000); cpu_freq_expect("Prototyp Amazing Foo Two @ 1.00GHz", 1000000000); + map_pagetables(&cr3, NULL); + map_pagetables(&cr3, &user_cr3); + pte_t *pte = get_pte(unit_tests); + printk("PTE: 0x%lx\n", pte->entry); + unmap_pagetables(&cr3, NULL); + unmap_pagetables(&cr3, &user_cr3); + + map_pagetables_va(&cr3, unit_tests); + pte_t *pte2 = get_pte(unit_tests); + printk("PTE: 0x%lx\n", pte2->entry); + unmap_pagetables_va(&cr3, unit_tests); + task_t *task1, *task2, *task_user1, *task_user1_se, *task_user1_int80, *task_user2, *task_user3, *task_user4;