General improvements and cleanup to page tables handling

arch/x86/pagetables.c

@@ -24,12 +24,16 @@
  */
 #include <console.h>
 #include <ktf.h>
+#include <multiboot2.h>
 #include <page.h>
 #include <pagetable.h>
 #include <setup.h>
 #include <spinlock.h>
 #include <string.h>
 
+static uint8_t _tmp_mapping[PAGE_SIZE] __aligned(PAGE_SIZE);
+static pgentry_t *_tmp_mapping_entry;
+
 cr3_t __aligned(PAGE_SIZE) cr3;
 cr3_t user_cr3;
 
@@ -100,17 +104,11 @@ void dump_pagetables(cr3_t cr3) {
     dump_page_table(paddr_to_virt_kern(cr3.paddr), 4);
 }
 
-static void *init_map_mfn(mfn_t mfn) {
-    static uint8_t _tmp[PAGE_SIZE] __aligned(PAGE_SIZE);
-    pgentry_t *e;
-
+static inline void *tmp_map_mfn(mfn_t mfn) {
     BUG_ON(mfn_invalid(mfn));
-
-    e = (pgentry_t *) l1_table_entry(get_l1_table(_tmp), _tmp);
-    BUG_ON(!e);
-    set_pgentry(e, mfn, L1_PROT);
-
-    return _tmp;
+    set_pgentry(_tmp_mapping_entry, mfn, L1_PROT);
+    invlpg(_tmp_mapping);
+    return _tmp_mapping;
 }
 
 static mfn_t get_cr3_mfn(cr3_t *cr3_entry) {
@@ -121,7 +119,7 @@ static mfn_t get_cr3_mfn(cr3_t *cr3_entry) {
         BUG_ON(!frame);
 
         cr3_entry->mfn = frame->mfn;
-        cr3_mapped = init_map_mfn(cr3_entry->mfn);
+        cr3_mapped = tmp_map_mfn(cr3_entry->mfn);
         memset(cr3_mapped, 0, PAGE_SIZE);
     }
 
@@ -154,17 +152,17 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag
 
     BUG_ON(mfn_invalid(tab_mfn));
 
-    tab = init_map_mfn(tab_mfn);
+    tab = tmp_map_mfn(tab_mfn);
     entry = &tab[index];
 
     mfn = mfn_from_pgentry(*entry);
     if (mfn_invalid(mfn)) {
         frame_t *frame = get_free_frame();
         BUG_ON(!frame);
 
-        set_pgentry(entry, frame->mfn, flags);
-        mfn = mfn_from_pgentry(*entry);
-        tab = init_map_mfn(mfn);
+        mfn = frame->mfn;
+        set_pgentry(entry, mfn, flags);
+        tab = tmp_map_mfn(mfn);
         memset(tab, 0, PAGE_SIZE);
     }
     else {
@@ -199,26 +197,29 @@ static void *_vmap(cr3_t *cr3_ptr, void *va, mfn_t mfn, unsigned int order,
 #endif
 
     if (order == PAGE_ORDER_1G) {
-        tab = init_map_mfn(l3t_mfn);
+        tab = tmp_map_mfn(l3t_mfn);
         entry = &tab[l3_table_index(va)];
         set_pgentry(entry, mfn, l3_flags | _PAGE_PSE);
+        invlpg(va);
         goto done;
     }
 
     l2t_mfn = get_pgentry_mfn(l3t_mfn, l3_table_index(va), l3_flags);
 
     if (order == PAGE_ORDER_2M) {
-        tab = init_map_mfn(l2t_mfn);
+        tab = tmp_map_mfn(l2t_mfn);
         entry = &tab[l2_table_index(va)];
         set_pgentry(entry, mfn, l2_flags | _PAGE_PSE);
+        invlpg(va);
         goto done;
     }
 
     l1t_mfn = get_pgentry_mfn(l2t_mfn, l2_table_index(va), l2_flags);
 
-    tab = init_map_mfn(l1t_mfn);
+    tab = tmp_map_mfn(l1t_mfn);
     entry = &tab[l1_table_index(va)];
     set_pgentry(entry, mfn, l1_flags);
+    invlpg(va);
 
 done:
     spin_unlock(&lock);
@@ -246,7 +247,28 @@ void *vmap_user(void *va, mfn_t mfn, unsigned int order,
                  l1_flags);
 }
 
+static inline void init_tmp_mapping(void) {
+    pte_t *tab = get_l1_table(_tmp_mapping);
+    _tmp_mapping_entry = (pgentry_t *) l1_table_entry(tab, _tmp_mapping);
+    BUG_ON(!_tmp_mapping_entry);
+}
+
+static void map_tmp_mapping_entry(void) {
+    pml4_t *l3e = l4_table_entry(mfn_to_virt(cr3.mfn), _tmp_mapping);
+    pdpe_t *l2e = l3_table_entry(mfn_to_virt(l3e->mfn), _tmp_mapping);
+    pde_t *l1e = l2_table_entry(mfn_to_virt(l2e->mfn), _tmp_mapping);
+    pte_t *entry = l1_table_entry(mfn_to_virt(l1e->mfn), _tmp_mapping);
+
+    /* Map _tmp_mapping_entry PTE of new page tables */
+    kmap_4k(l1e->mfn, L1_PROT);
+
+    /* Point _tmp_mapping_entry at new page tables location */
+    _tmp_mapping_entry = paddr_to_virt_kern(_paddr(entry));
+}
+
 void init_pagetables(void) {
+    init_tmp_mapping();
+
     for_each_memory_range (r) {
         switch (r->base) {
         case VIRT_IDENT_BASE:
@@ -270,5 +292,9 @@ void init_pagetables(void) {
         }
     }
 
-    map_used_memory();
+    map_frames_array();
+    map_multiboot_areas();
+    map_tmp_mapping_entry();
+
+    write_cr3(cr3.paddr);
 }

common/setup.c

@@ -222,17 +222,15 @@ void __noreturn __text_init kernel_start(uint32_t multiboot_magic, unsigned long
 
     /* Setup final pagetables */
     init_pagetables();
-
-    map_multiboot_areas();
-    map_bios_area();
-
-    write_cr3(cr3.paddr);
     boot_flags.virt = true;
 
     WRITE_SP(get_free_pages_top(PAGE_ORDER_2M, GFP_KERNEL_MAP));
+
     if (opt_debug)
         dump_pagetables(cr3);
 
+    map_bios_area();
+
     if (setup_framebuffer())
         display_banner();
     else

include/arch/x86/pagetable.h

@@ -272,7 +272,6 @@ static inline pdpe_t *get_pdpe(const void *va) {
 static inline void set_pgentry(pgentry_t *e, mfn_t mfn, unsigned long flags) {
     *e = pgentry_from_mfn(mfn, flags);
     barrier();
-    flush_tlb();
 }
 
 /* External declarations */

include/lib.h

@@ -324,6 +324,10 @@ static inline void str(unsigned int *selector) {
     asm volatile("str %0" : "=m"(*selector));
 }
 
+static inline void invlpg(void *addr) {
+    asm volatile("invlpg (%0)" ::"r"(addr) : "memory");
+}
+
 static inline void flush_tlb(void) {
     write_cr3(read_cr3());
 }

include/mm/pmm.h

@@ -77,6 +77,7 @@ extern void put_free_frames(mfn_t mfn, unsigned int order);
 extern void reclaim_frame(mfn_t mfn, unsigned int order);
 
 extern void map_used_memory(void);
+extern void map_frames_array(void);
 
 /* Static definitions */
 

mm/pmm.c

@@ -570,3 +570,9 @@ void map_used_memory(void) {
         }
     }
 }
+
+void map_frames_array(void) {
+    frames_array_t *array;
+
+    list_for_each_entry (array, &frames, list) { kmap_4k(virt_to_mfn(array), L1_PROT); }
+}