From 320f68b83372d65acdf4948fdb74b679077f2c73 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Mon, 24 Aug 2020 15:26:06 +0200 Subject: [PATCH] zap identity mappings after init (#64) * pmm: export display_frames_count() Signed-off-by: Pawel Wieczorkiewicz * smp: use dedicated variable for CR3 Do not keep final, global cr3 variable in .data.init section. When init sections are being reclaimed, this breaks pagetables interface. Instead, for SMP AP booting, use dedicated ap_cr3 variable, which can be reclaimed after SMP initialization. Signed-off-by: Pawel Wieczorkiewicz * smp: minor cleanup * Rename smp_init() to init_smp() to follow naming convention * Put boot_cpu() and init_smp() into the .text.init section Signed-off-by: Pawel Wieczorkiewicz * pmm: zap boot mappings after initialization * PMM: - Add reclaim_frame() non-static wrapper over add_frame(); it adds a frame being reclaimed to the list of free frames. * Compiler: - Add IS_INIT_SECTION() macro; it decides, if section is an init section, based on its name. * Setup: - Modify zap_boot_mappings() to iterate over all defined memory regions and unmap plus reclaim all frames belonging to identity mapped init sections. Additionally, zero-out non-text sections. Signed-off-by: Pawel Wieczorkiewicz Co-authored-by: Bjoern Doebel --- arch/x86/pagetables.c | 2 +- common/kernel.c | 1 + common/setup.c | 27 +++++++++++++-------------- include/compiler.h | 4 ++++ include/mm/pmm.h | 2 ++ include/setup.h | 2 ++ include/smp/smp.h | 2 +- mm/pmm.c | 17 ++++++++++++----- smp/bootstrap.S | 2 +- smp/smp.c | 7 +++++-- 10 files changed, 42 insertions(+), 24 deletions(-) diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c index 91e5e897..babf7eb3 100644 --- a/arch/x86/pagetables.c +++ b/arch/x86/pagetables.c @@ -30,7 +30,7 @@ #include #include -cr3_t __data_init cr3; +cr3_t cr3; static inline const char *dump_pte_flags(char *buf, size_t size, pte_t pte) { /* clang-format off */ diff --git a/common/kernel.c b/common/kernel.c index bb9bc4be..97e5c2c4 100644 --- a/common/kernel.c +++ b/common/kernel.c @@ -50,6 +50,7 @@ void kernel_main(void) { if (kernel_cmdline) printk("Command line: %s\n", kernel_cmdline); + zap_boot_mappings(); display_memory_map(); display_multiboot_mmap(); diff --git a/common/setup.c b/common/setup.c index 38be845a..29a1b895 100644 --- a/common/setup.c +++ b/common/setup.c @@ -66,17 +66,18 @@ static __always_inline void zero_bss(void) { memset(_ptr(__start_bss_user), 0x0, _ptr(__end_bss_user) - _ptr(__start_bss_user)); } -static __always_inline void zap_boot_mappings(void) { -#if defined(__x86_64__) - memset(paddr_to_virt_kern(virt_to_paddr(l4_pt_entries)), 0, - L4_PT_ENTRIES * sizeof(pgentry_t)); -#endif - memset(paddr_to_virt_kern(virt_to_paddr(l3_pt_entries)), 0, - L3_PT_ENTRIES * sizeof(pgentry_t)); - memset(paddr_to_virt_kern(virt_to_paddr(l2_pt_entries)), 0, - L2_PT_ENTRIES * sizeof(pgentry_t)); - memset(paddr_to_virt_kern(virt_to_paddr(l1_pt_entries)), 0, - L1_PT_ENTRIES * sizeof(pgentry_t)); +void zap_boot_mappings(void) { + for_each_memory_range (r) { + if (r->base == VIRT_IDENT_BASE && IS_INIT_SECTION(r->name)) { + if (strcmp(r->name, ".text.init")) + memset(r->start, 0, r->end - r->start); + + for (mfn_t mfn = virt_to_mfn(r->start); mfn < virt_to_mfn(r->end); mfn++) { + vunmap(mfn_to_virt(mfn), PAGE_ORDER_4K); + reclaim_frame(mfn, PAGE_ORDER_4K); + } + } + } } void __noreturn __text_init kernel_start(uint32_t multiboot_magic, @@ -113,15 +114,13 @@ void __noreturn __text_init kernel_start(uint32_t multiboot_magic, init_traps(0); - zap_boot_mappings(); - init_slab(); init_apic(APIC_MODE_XAPIC); init_tasks(); - smp_init(); + init_smp(); /* Jump from .text.init section to .text */ asm volatile("push %0; ret" ::"r"(&kernel_main)); diff --git a/include/compiler.h b/include/compiler.h index 82a285fe..e035da0f 100644 --- a/include/compiler.h +++ b/include/compiler.h @@ -69,6 +69,10 @@ #define __data_init __section(".data.init") #define __bss_init __section(".bss.init") +#define IS_INIT_SECTION(name) \ + (!strcmp(name, ".text.init") || !strcmp(name, ".data.init") || \ + !strcmp(name, ".bss.init")) + #define __user_text __section(".text.user") #define __user_data __section(".data.user") #define __user_bss __section(".bss.user") diff --git a/include/mm/pmm.h b/include/mm/pmm.h index 66662007..cc52684a 100644 --- a/include/mm/pmm.h +++ b/include/mm/pmm.h @@ -79,6 +79,7 @@ typedef struct frame frame_t; /* External definitions */ extern void display_memory_map(void); +extern void display_frames_count(void); extern addr_range_t get_memory_range(paddr_t pa); extern paddr_t get_memory_range_start(paddr_t pa); @@ -90,6 +91,7 @@ extern void init_pmm(void); extern mfn_t get_free_frames(unsigned int order); extern void put_frame(mfn_t mfn, unsigned int order); +extern void reclaim_frame(mfn_t mfn, unsigned int order); extern void map_used_memory(void); diff --git a/include/setup.h b/include/setup.h index 9182efc0..1994f0c0 100644 --- a/include/setup.h +++ b/include/setup.h @@ -47,6 +47,8 @@ static inline void get_com_ports(void) { com_ports[1] = 0x2f8; } +extern void zap_boot_mappings(void); + #endif /* __ASSEMBLY__ */ #endif /* KTF_SETUP_H */ diff --git a/include/smp/smp.h b/include/smp/smp.h index 31c526cd..402e6962 100644 --- a/include/smp/smp.h +++ b/include/smp/smp.h @@ -33,7 +33,7 @@ /* External declarations */ -extern void smp_init(void); +extern void init_smp(void); extern unsigned get_nr_cpus(void); /* Static declarations */ diff --git a/mm/pmm.c b/mm/pmm.c index 7e247ff8..dfff3b42 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -33,6 +33,8 @@ #include #include +size_t total_phys_memory; + static list_head_t free_frames[MAX_PAGE_ORDER + 1]; static list_head_t busy_frames[MAX_PAGE_ORDER + 1]; @@ -114,8 +116,8 @@ paddr_t get_memory_range_end(paddr_t pa) { return _paddr(r.end); } -static void display_frames_count(size_t size) { - printk("Avail memory frames: (total size: %lu MB)\n", size / MB(1)); +void display_frames_count(void) { + printk("Avail memory frames: (total size: %lu MB)\n", total_phys_memory / MB(1)); for_each_order (order) { size_t count = frames_count[order]; @@ -149,6 +151,12 @@ static void add_frame(paddr_t *pa, unsigned int order, bool initial) { frames_count[order]++; } +void reclaim_frame(mfn_t mfn, unsigned int order) { + paddr_t pa = mfn_to_paddr(mfn); + + add_frame(&pa, order, false); +} + static size_t process_memory_range(unsigned index) { paddr_t start, end, cur; addr_range_t range; @@ -200,7 +208,6 @@ bool paddr_invalid(paddr_t pa) { } void init_pmm(void) { - size_t total_size = 0; unsigned num; printk("Initialize Physical Memory Manager\n"); @@ -215,9 +222,9 @@ void init_pmm(void) { /* Skip low memory range */ for (unsigned int i = 1; i < num; i++) - total_size += process_memory_range(i); + total_phys_memory += process_memory_range(i); - display_frames_count(total_size); + display_frames_count(); if (opt_debug) { frame_t *frame; diff --git a/smp/bootstrap.S b/smp/bootstrap.S index a17b1a8f..cee5332e 100644 --- a/smp/bootstrap.S +++ b/smp/bootstrap.S @@ -55,7 +55,7 @@ GLOBAL(ap_start) or $(X86_CR4_PAE | X86_CR4_PSE), %eax mov %eax, %cr4 - mov (cr3), %eax + mov (ap_cr3), %eax mov %eax, %cr3 /* Enable long mode */ diff --git a/smp/smp.c b/smp/smp.c index 83886006..a8de7fd3 100644 --- a/smp/smp.c +++ b/smp/smp.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,7 @@ static unsigned nr_cpus; static unsigned ap_cpuid; static bool ap_callin; +cr3_t __data_init ap_cr3; void __noreturn ap_startup(void) { write_sp(get_free_pages_top(PAGE_ORDER_2M, GFP_KERNEL)); @@ -60,7 +62,7 @@ void __noreturn ap_startup(void) { UNREACHABLE(); } -static void boot_cpu(unsigned int cpu) { +static __text_init void boot_cpu(unsigned int cpu) { percpu_t *percpu = get_percpu_page(cpu); uint64_t icr; @@ -91,7 +93,7 @@ static void boot_cpu(unsigned int cpu) { dprintk("AP: %u Done \n", cpu); } -void smp_init(void) { +void __text_init init_smp(void) { unsigned mp_nr_cpus = mptables_init(); unsigned acpi_nr_cpus = acpi_get_nr_cpus(); @@ -102,6 +104,7 @@ void smp_init(void) { } printk("Initializing SMP support (CPUs: %u)\n", nr_cpus); + ap_cr3 = cr3; for (unsigned int i = 0; i < nr_cpus; i++) boot_cpu(i);