diff --git a/CODEOWNERS b/CODEOWNERS index f7e509aea919..f2cac1d390bf 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -542,6 +542,7 @@ /subsys/dfu/ @nvlsianpu /subsys/tracing/ @nashif @wentongwu /subsys/debug/asan_hacks.c @vanwinkeljan @aescolar @daor-oti +/subsys/demand_paging/ @andrewboie /subsys/disk/disk_access_spi_sdhc.c @JunYangNXP /subsys/disk/disk_access_sdhc.h @JunYangNXP /subsys/disk/disk_access_usdhc.c @JunYangNXP diff --git a/arch/Kconfig b/arch/Kconfig index 81918352ea6e..34b45f4951aa 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -61,6 +61,7 @@ config X86 select ARCH_HAS_GDBSTUB if !X86_64 select ARCH_HAS_TIMING_FUNCTIONS select ARCH_HAS_THREAD_LOCAL_STORAGE + select ARCH_HAS_DEMAND_PAGING help x86 architecture @@ -517,6 +518,40 @@ config CPU_HAS_MMU This hidden option is selected when the CPU has a Memory Management Unit (MMU). +config ARCH_HAS_DEMAND_PAGING + bool + help + This hidden configuration should be selected by the architecture if + demand paging is supported. + +config ARCH_HAS_RESERVED_PAGE_FRAMES + bool + help + This hidden configuration should be selected by the architecture if + certain RAM page frames need to be marked as reserved and never used for + memory mappings. The architecture will need to implement + arch_reserved_pages_update(). + +config ARCH_MAPS_ALL_RAM + bool + help + This hidden option is selected by the architecture to inform the kernel + that all RAM is mapped at boot, and not just the bounds of the Zephyr image. + If RAM starts at 0x0, the first page must remain un-mapped to catch NULL + pointer dereferences. With this enabled, the kernel will not assume that + virtual memory addresses past the kernel image are available for mappings, + but instead takes into account an entire RAM mapping instead. + + This is typically set by architectures which need direct access to all memory. + It is the architecture's responsibility to mark reserved memory regions + as such in arch_reserved_pages_update(). + + Although the kernel will not disturb this RAM mapping by re-mapping the associated + virtual addresses elsewhere, this is limited to only management of the + virtual address space. The kernel's page frame ontology will not consider + this mapping at all; non-kernel pages will be considered free (unless marked + as reserved) and Z_PAGE_FRAME_MAPPED will not be set. + menuconfig MMU bool "Enable MMU features" depends on CPU_HAS_MMU @@ -533,21 +568,10 @@ config MMU_PAGE_SIZE support multiple page sizes, put the smallest one here. config KERNEL_VM_BASE - hex "Base virtual address to link the kernel" + hex "Virtual address space base address" default $(dt_chosen_reg_addr_hex,$(DT_CHOSEN_Z_SRAM)) help - Define the base virtual memory address for the core kernel. - - The kernel expects a mappings for all physical RAM regions starting at - this virtual address, with any unused space up to the size denoted by - KERNEL_VM_SIZE available for memory mappings. This base address denotes - the start of the RAM mapping and may not be the base address of the - kernel itself, but the offset of the kernel here will be the same as the - offset from the beginning of physical memory where it was loaded. - - If there are multiple physical RAM regions which are discontinuous in - the physical memory map, they should all be mapped in a continuous - virtual region, with bounds defined by KERNEL_RAM_SIZE. + Define the base of the kernel's address space. By default, this is the same as the DT_CHOSEN_Z_SRAM physical base SRAM address from DTS, in which case RAM will be identity-mapped. Some @@ -555,7 +579,7 @@ config KERNEL_VM_BASE just one RAM region and doing this makes linking much simpler, as at least when the kernel boots all virtual RAM addresses are the same as their physical address (demand paging at runtime may later modify - this for some subset of non-pinned pages). + this for non-pinned page frames). Otherwise, if RAM isn't identity-mapped: 1. It is the architecture's responsibility to transition the @@ -568,34 +592,72 @@ config KERNEL_VM_BASE double-linking of paging structures to make the instruction pointer transition simpler). -config KERNEL_RAM_SIZE - hex "Total size of RAM mappings in bytes" - default $(dt_chosen_reg_size_hex,$(DT_CHOSEN_Z_SRAM)) + Zephyr does not implement a split address space and if multiple + page tables are in use, they all have the same virtual-to-physical + mappings (with potentially different permissions). + +config KERNEL_VM_OFFSET + hex "Kernel offset within address space" + default 0 help - Indicates to the kernel the total size of RAM that is mapped. The - kernel expects that all physical RAM has a memory mapping in the virtual - address space, and that these RAM mappings are all within the virtual - region [KERNEL_VM_BASE..KERNEL_VM_BASE + KERNEL_RAM_SIZE). + Offset that the kernel image begins within its address space, + if this is not the same offset from the beginning of RAM. + + Some care may need to be taken in selecting this value. In certain + build-time cases, or when a physical address cannot be looked up + in page tables, the equation: + + virt = phys + ((KERNEL_VM_BASE + KERNEL_VM_OFFSET) - + SRAM_BASE_ADDRESS) + + Will be used to convert between physical and virtual addresses for + memory that is mapped at boot. + + This uncommon and is only necessary if the beginning of VM and + physical memory have dissimilar alignment. config KERNEL_VM_SIZE hex "Size of kernel address space in bytes" - default 0xC0000000 + default 0x800000 help Size of the kernel's address space. Constraining this helps control how much total memory can be used for page tables. - The difference between KERNEL_RAM_SIZE and KERNEL_VM_SIZE indicates the + The difference between KERNEL_VM_BASE and KERNEL_VM_SIZE indicates the size of the virtual region for runtime memory mappings. This is needed for mapping driver MMIO regions, as well as special RAM mapping use-cases such as VSDO pages, memory mapped thread stacks, and anonymous memory - mappings. + mappings. The kernel itself will be mapped in here as well at boot. - The system currently assumes all RAM can be mapped in the virtual address - space. Systems with very large amounts of memory (such as 512M or more) + Systems with very large amounts of memory (such as 512M or more) will want to use a 64-bit build of Zephyr, there are no plans to implement a notion of "high" memory in Zephyr to work around physical - RAM which can't have a boot-time mapping due to a too-small address space. + RAM size larger than the defined bounds of the virtual address space. + +config DEMAND_PAGING + bool "Enable demand paging [EXPERIMENTAL]" + depends on ARCH_HAS_DEMAND_PAGING + help + Enable demand paging. Requires architecture support in how the kernel + is linked and the implementation of an eviction algorithm and a + backing store for evicted pages. + +if DEMAND_PAGING +config DEMAND_PAGING_ALLOW_IRQ + bool "Allow interrupts during page-ins/outs" + help + Allow interrupts to be serviced while pages are being evicted or + retrieved from the backing store. This is much better for system + latency, but any code running in interrupt context that page faults + will cause a kernel panic. Such code must work with exclusively pinned + code and data pages. + + The scheduler is still disabled during this operation. + If this option is disabled, the page fault servicing logic + runs with interrupts disabled for the entire operation. However, + ISRs may also page fault. +endif # DEMAND_PAGING endif # MMU menuconfig MPU diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 2b4c5eb1cdbd..068361e58dec 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -95,6 +95,7 @@ endchoice config ACPI bool "ACPI (Advanced Configuration and Power Interface) support" + select ARCH_MAPS_ALL_RAM help Allow retrieval of platform configuration at runtime. @@ -189,29 +190,6 @@ config X86_MMU and creates a set of page tables at boot time that is runtime- mutable. -config X86_MMU_PAGE_POOL_PAGES - int "Number of pages to reserve for building page tables" - default 0 - depends on X86_MMU - help - Define the number of pages in the pool used to allocate page table - data structures at runtime. - - Pages might need to be drawn from the pool during memory mapping - operations, unless the address space has been completely pre-allocated. - - Pages will need to drawn from the pool to initialize memory domains. - This does not include the default memory domain if KPTI=n. - - The specific value used here depends on the size of physical RAM, - how much additional virtual memory will be mapped at runtime, and - how many memory domains need to be initialized. - - The current suite of Zephyr test cases may initialize at most two - additional memory domains besides the default domain. - - Unused pages in this pool cannot be used for other purposes. - config X86_COMMON_PAGE_TABLE bool "Use a single page table for all threads" default n @@ -224,6 +202,18 @@ config X86_COMMON_PAGE_TABLE page tables in place. This is much slower, but uses much less RAM for page tables. +config X86_MAX_ADDITIONAL_MEM_DOMAINS + int "Maximum number of memory domains" + default 3 + depends on X86_MMU && USERSPACE && !X86_COMMON_PAGE_TABLE + help + The initial page tables at boot are pre-allocated, and used for the + default memory domain. Instantiation of additional memory domains + if common page tables are in use requires a pool of free pinned + memory pages for constructing page tables. + + Zephyr test cases assume 3 additional domains can be instantiated. + config X86_NO_MELTDOWN bool help diff --git a/arch/x86/core/fatal.c b/arch/x86/core/fatal.c index fd37522381a6..b8778c6122a8 100644 --- a/arch/x86/core/fatal.c +++ b/arch/x86/core/fatal.c @@ -10,6 +10,7 @@ #include #include #include +#include LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); #if defined(CONFIG_BOARD_QEMU_X86) || defined(CONFIG_BOARD_QEMU_X86_64) @@ -359,6 +360,44 @@ static const struct z_exc_handle exceptions[] = { void z_x86_page_fault_handler(z_arch_esf_t *esf) { +#ifdef CONFIG_DEMAND_PAGING + if ((esf->errorCode & PF_P) == 0) { + /* Page was non-present at time exception happened. + * Get faulting virtual address from CR2 register + */ + void *virt = z_x86_cr2_get(); + bool was_valid_access; + +#ifdef CONFIG_X86_KPTI + /* Protection ring is lowest 2 bits in interrupted CS */ + bool was_user = ((esf->cs & 0x3) != 0U); + + /* Need to check if the interrupted context was a user thread + * that hit a non-present page that was flipped due to KPTI in + * the thread's page tables, in which case this is an access + * violation and we should treat this as an error. + * + * We're probably not locked, but if there is a race, we will + * be fine, the kernel page fault code will later detect that + * the page is present in the kernel's page tables and the + * instruction will just be re-tried, producing another fault. + */ + if (was_user && + !z_x86_kpti_is_access_ok(virt, get_ptables(esf))) { + was_valid_access = false; + } else +#else + { + was_valid_access = z_page_fault(virt); + } +#endif /* CONFIG_X86_KPTI */ + if (was_valid_access) { + /* Page fault handled, re-try */ + return; + } + } +#endif /* CONFIG_DEMAND_PAGING */ + #if !defined(CONFIG_X86_64) && defined(CONFIG_DEBUG_COREDUMP) z_x86_exception_vector = IV_PAGE_FAULT; #endif diff --git a/arch/x86/core/x86_mmu.c b/arch/x86/core/x86_mmu.c index ce9073c2e39b..e99d4dec3f0a 100644 --- a/arch/x86/core/x86_mmu.c +++ b/arch/x86/core/x86_mmu.c @@ -17,7 +17,9 @@ #include #include #include +#include #include +#include LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); @@ -194,24 +196,25 @@ static const struct paging_level paging_levels[] = { #define NUM_PD_ENTRIES 1024U #define NUM_PT_ENTRIES 1024U #endif /* !CONFIG_X86_64 && !CONFIG_X86_PAE */ + /* Memory range covered by an instance of various table types */ -#define PT_AREA ((uintptr_t)(CONFIG_MMU_PAGE_SIZE * NUM_PT_ENTRIES)) +#define PT_AREA ((uintptr_t)(CONFIG_MMU_PAGE_SIZE * NUM_PT_ENTRIES)) #define PD_AREA (PT_AREA * NUM_PD_ENTRIES) #ifdef CONFIG_X86_64 #define PDPT_AREA (PD_AREA * NUM_PDPT_ENTRIES) #endif -#define VM_ADDR DT_REG_ADDR(DT_CHOSEN(zephyr_sram)) -#define VM_SIZE DT_REG_SIZE(DT_CHOSEN(zephyr_sram)) +#define VM_ADDR CONFIG_KERNEL_VM_BASE +#define VM_SIZE CONFIG_KERNEL_VM_SIZE /* Define a range [PT_START, PT_END) which is the memory range - * covered by all the page tables needed for system RAM + * covered by all the page tables needed for the address space */ #define PT_START ((uintptr_t)ROUND_DOWN(VM_ADDR, PT_AREA)) #define PT_END ((uintptr_t)ROUND_UP(VM_ADDR + VM_SIZE, PT_AREA)) -/* Number of page tables needed to cover system RAM. Depends on the specific - * bounds of system RAM, but roughly 1 page table per 2MB of RAM +/* Number of page tables needed to cover address space. Depends on the specific + * bounds, but roughly 1 page table per 2MB of RAM */ #define NUM_PT ((PT_END - PT_START) / PT_AREA) @@ -221,8 +224,8 @@ static const struct paging_level paging_levels[] = { */ #define PD_START ((uintptr_t)ROUND_DOWN(VM_ADDR, PD_AREA)) #define PD_END ((uintptr_t)ROUND_UP(VM_ADDR + VM_SIZE, PD_AREA)) -/* Number of page directories needed to cover system RAM. Depends on the - * specific bounds of system RAM, but roughly 1 page directory per 1GB of RAM +/* Number of page directories needed to cover the address space. Depends on the + * specific bounds, but roughly 1 page directory per 1GB of RAM */ #define NUM_PD ((PD_END - PD_START) / PD_AREA) #else @@ -232,13 +235,11 @@ static const struct paging_level paging_levels[] = { #ifdef CONFIG_X86_64 /* Same semantics as above, but for the page directory pointer tables needed - * to cover system RAM. On 32-bit there is just one 4-entry PDPT. + * to cover the address space. On 32-bit there is just one 4-entry PDPT. */ #define PDPT_START ((uintptr_t)ROUND_DOWN(VM_ADDR, PDPT_AREA)) #define PDPT_END ((uintptr_t)ROUND_UP(VM_ADDR + VM_SIZE, PDPT_AREA)) -/* Number of PDPTs needed to cover system RAM. Depends on the - * specific bounds of system RAM, but roughly 1 PDPT per 512GB of RAM - */ +/* Number of PDPTs needed to cover the address space. 1 PDPT per 512GB of VM */ #define NUM_PDPT ((PDPT_END - PDPT_START) / PDPT_AREA) /* All pages needed for page tables, using computed values plus one more for @@ -699,47 +700,6 @@ void z_x86_dump_mmu_flags(pentry_t *ptables, void *virt) } #endif /* CONFIG_EXCEPTION_DEBUG */ -/* - * Pool of free memory pages for creating new page tables, as needed. - * - * XXX: This is very crude, once obtained, pages may not be returned. Tuning - * the optimal value of CONFIG_X86_MMU_PAGE_POOL_PAGES is not intuitive, - * Better to have a kernel managed page pool of unused RAM that can be used for - * this, sbrk(), and other anonymous mappings. See #29526 - */ -static uint8_t __noinit - page_pool[CONFIG_MMU_PAGE_SIZE * CONFIG_X86_MMU_PAGE_POOL_PAGES] - __aligned(CONFIG_MMU_PAGE_SIZE); - -static uint8_t *page_pos = page_pool + sizeof(page_pool); - -/* Return a zeroed and suitably aligned memory page for page table data - * from the global page pool - */ -static void *page_pool_get(void) -{ - void *ret; - - if (page_pos == page_pool) { - ret = NULL; - } else { - page_pos -= CONFIG_MMU_PAGE_SIZE; - ret = page_pos; - } - - if (ret != NULL) { - memset(ret, 0, CONFIG_MMU_PAGE_SIZE); - } - - return ret; -} - -/* Debugging function to show how many pages are free in the pool */ -static inline unsigned int pages_free(void) -{ - return (page_pos - page_pool) / CONFIG_MMU_PAGE_SIZE; -} - /* Reset permissions on a PTE to original state when the mapping was made */ static inline pentry_t reset_pte(pentry_t old_val) { @@ -848,12 +808,6 @@ static inline bool atomic_pte_cas(pentry_t *target, pentry_t old_value, */ #define OPTION_RESET BIT(2) -/* Indicates that allocations from the page pool are allowed to instantiate - * new paging structures. Only necessary when establishing new mappings - * and the entire address space isn't pre-allocated. - */ -#define OPTION_ALLOC BIT(3) - /** * Atomically update bits in a page table entry * @@ -922,10 +876,6 @@ static inline pentry_t pte_atomic_update(pentry_t *pte, pentry_t update_val, * modified by another CPU, using atomic operations to update the requested * bits and return the previous PTE value. * - * This function is NOT atomic with respect to allocating intermediate - * paging structures, and this must be called with x86_mmu_lock held if - * OPTION_ALLOC is used. - * * Common mask values: * MASK_ALL - Update all PTE bits. Exitsing state totally discarded. * MASK_PERM - Only update permission bits. All other bits and physical @@ -937,17 +887,13 @@ static inline pentry_t pte_atomic_update(pentry_t *pte, pentry_t update_val, * @param [out] old_val_ptr Filled in with previous PTE value. May be NULL. * @param mask What bits to update in the PTE (ignored if OPTION_RESET) * @param options Control options, described above - * @retval 0 Success - * @retval -ENOMEM allocation required and no free pages (only if OPTION_ALLOC) */ -static int page_map_set(pentry_t *ptables, void *virt, pentry_t entry_val, - pentry_t *old_val_ptr, pentry_t mask, uint32_t options) +static void page_map_set(pentry_t *ptables, void *virt, pentry_t entry_val, + pentry_t *old_val_ptr, pentry_t mask, uint32_t options) { pentry_t *table = ptables; bool flush = (options & OPTION_FLUSH) != 0U; - assert_virt_addr_aligned(virt); - for (int level = 0; level < NUM_LEVELS; level++) { int index; pentry_t *entryp; @@ -965,42 +911,20 @@ static int page_map_set(pentry_t *ptables, void *virt, pentry_t entry_val, break; } - /* This is a non-leaf entry */ - if ((*entryp & MMU_P) == 0U) { - /* Not present. Never done a mapping here yet, need - * some RAM for linked tables - */ - void *new_table; - - __ASSERT((options & OPTION_ALLOC) != 0, - "missing page table and allocations disabled"); - - new_table = page_pool_get(); - - if (new_table == NULL) { - return -ENOMEM; - } - - *entryp = ((pentry_t)z_x86_phys_addr(new_table) | - INT_FLAGS); - table = new_table; - } else { - /* We fail an assertion here due to no support for - * splitting existing bigpage mappings. - * If the PS bit is not supported at some level (like - * in a PML4 entry) it is always reserved and must be 0 - */ - __ASSERT((*entryp & MMU_PS) == 0U, - "large page encountered"); - table = next_table(*entryp, level); - } + /* We fail an assertion here due to no support for + * splitting existing bigpage mappings. + * If the PS bit is not supported at some level (like + * in a PML4 entry) it is always reserved and must be 0 + */ + __ASSERT((*entryp & MMU_PS) == 0U, "large page encountered"); + table = next_table(*entryp, level); + __ASSERT(table != NULL, + "missing page table level %d when trying to map %p", + level + 1, virt); } - if (flush) { tlb_flush_page(virt); } - - return 0; } /** @@ -1009,8 +933,6 @@ static int page_map_set(pentry_t *ptables, void *virt, pentry_t entry_val, * See documentation for page_map_set() for additional notes about masks and * supported options. * - * Must call this with x86_mmu_lock held if OPTION_ALLOC is used. - * * It is vital to remember that all virtual-to-physical mappings must be * the same with respect to supervisor mode regardless of what thread is * scheduled (and therefore, if multiple sets of page tables exist, which one @@ -1030,14 +952,11 @@ static int page_map_set(pentry_t *ptables, void *virt, pentry_t entry_val, * @param mask What bits to update in each PTE. Un-set bits will never be * modified. Ignored if OPTION_RESET. * @param options Control options, described above - * @retval 0 Success - * @retval -ENOMEM allocation required and no free pages (only if OPTION_ALLOC) */ -static int range_map_ptables(pentry_t *ptables, void *virt, uintptr_t phys, - size_t size, pentry_t entry_flags, pentry_t mask, - uint32_t options) +static void range_map_ptables(pentry_t *ptables, void *virt, uintptr_t phys, + size_t size, pentry_t entry_flags, pentry_t mask, + uint32_t options) { - int ret; bool reset = (options & OPTION_RESET) != 0U; assert_addr_aligned(phys); @@ -1061,14 +980,9 @@ static int range_map_ptables(pentry_t *ptables, void *virt, uintptr_t phys, entry_val = (phys + offset) | entry_flags; } - ret = page_map_set(ptables, dest_virt, entry_val, NULL, mask, - options); - if (ret != 0) { - return ret; - } + page_map_set(ptables, dest_virt, entry_val, NULL, mask, + options); } - - return 0; } /** @@ -1092,14 +1006,10 @@ static int range_map_ptables(pentry_t *ptables, void *virt, uintptr_t phys, * be preserved. Ignored if OPTION_RESET. * @param options Control options. Do not set OPTION_USER here. OPTION_FLUSH * will trigger a TLB shootdown after all tables are updated. - * @retval 0 Success - * @retval -ENOMEM page table allocation required, but no free pages */ -static int range_map(void *virt, uintptr_t phys, size_t size, - pentry_t entry_flags, pentry_t mask, uint32_t options) +static void range_map(void *virt, uintptr_t phys, size_t size, + pentry_t entry_flags, pentry_t mask, uint32_t options) { - int ret = 0; - LOG_DBG("%s: %p -> %p (%zu) flags " PRI_ENTRY " mask " PRI_ENTRY " opt 0x%x", __func__, (void *)phys, virt, size, entry_flags, mask, options); @@ -1130,47 +1040,29 @@ static int range_map(void *virt, uintptr_t phys, size_t size, struct arch_mem_domain *domain = CONTAINER_OF(node, struct arch_mem_domain, node); - ret = range_map_ptables(domain->ptables, virt, phys, size, - entry_flags, mask, - options | OPTION_USER); - if (ret != 0) { - /* NOTE: Currently we do not un-map a partially - * completed mapping. - */ - goto out_unlock; - } + range_map_ptables(domain->ptables, virt, phys, size, + entry_flags, mask, options | OPTION_USER); } #endif /* CONFIG_USERSPACE */ - ret = range_map_ptables(z_x86_kernel_ptables, virt, phys, size, - entry_flags, mask, options); -#if defined(CONFIG_USERSPACE) && !defined(CONFIG_X86_COMMON_PAGE_TABLE) -out_unlock: -#endif /* CONFIG_USERSPACE */ - if (ret == 0 && (options & OPTION_ALLOC) != 0) { - LOG_DBG("page pool pages free: %u / %u", pages_free(), - CONFIG_X86_MMU_PAGE_POOL_PAGES); - } + range_map_ptables(z_x86_kernel_ptables, virt, phys, size, entry_flags, + mask, options); #ifdef CONFIG_SMP if ((options & OPTION_FLUSH) != 0U) { tlb_shootdown(); } #endif /* CONFIG_SMP */ - return ret; } -static inline int range_map_unlocked(void *virt, uintptr_t phys, size_t size, - pentry_t entry_flags, pentry_t mask, - uint32_t options) +static inline void range_map_unlocked(void *virt, uintptr_t phys, size_t size, + pentry_t entry_flags, pentry_t mask, + uint32_t options) { - int ret; k_spinlock_key_t key; key = k_spin_lock(&x86_mmu_lock); - ret = range_map(virt, phys, size, entry_flags, mask, options); + range_map(virt, phys, size, entry_flags, mask, options); k_spin_unlock(&x86_mmu_lock, key); - - return ret; } static pentry_t flags_to_entry(uint32_t flags) @@ -1193,7 +1085,7 @@ static pentry_t flags_to_entry(uint32_t flags) case K_MEM_CACHE_WB: break; default: - return -ENOTSUP; + __ASSERT(false, "bad memory mapping flags 0x%x", flags); } if ((flags & K_MEM_PERM_RW) != 0U) { @@ -1212,10 +1104,10 @@ static pentry_t flags_to_entry(uint32_t flags) } /* map new region virt..virt+size to phys with provided arch-neutral flags */ -int arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) +void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) { - return range_map_unlocked(virt, phys, size, flags_to_entry(flags), - MASK_ALL, OPTION_ALLOC); + range_map_unlocked(virt, phys, size, flags_to_entry(flags), + MASK_ALL, 0); } #if CONFIG_X86_STACK_PROTECTION @@ -1229,8 +1121,8 @@ void z_x86_set_stack_guard(k_thread_stack_t *stack) * Guard page is always the first page of the stack object for both * kernel and thread stacks. */ - (void)range_map_unlocked(stack, 0, CONFIG_MMU_PAGE_SIZE, - MMU_P | ENTRY_XD, MASK_PERM, OPTION_FLUSH); + range_map_unlocked(stack, 0, CONFIG_MMU_PAGE_SIZE, + MMU_P | ENTRY_XD, MASK_PERM, OPTION_FLUSH); } #endif /* CONFIG_X86_STACK_PROTECTION */ @@ -1332,14 +1224,14 @@ int arch_buffer_validate(void *addr, size_t size, int write) static inline void reset_region(uintptr_t start, size_t size) { - (void)range_map_unlocked((void *)start, 0, size, 0, 0, - OPTION_FLUSH | OPTION_RESET); + range_map_unlocked((void *)start, 0, size, 0, 0, + OPTION_FLUSH | OPTION_RESET); } static inline void apply_region(uintptr_t start, size_t size, pentry_t attr) { - (void)range_map_unlocked((void *)start, 0, size, attr, MASK_PERM, - OPTION_FLUSH); + range_map_unlocked((void *)start, 0, size, attr, MASK_PERM, + OPTION_FLUSH); } /* Cache of the current memory domain applied to the common page tables and @@ -1465,6 +1357,44 @@ void arch_mem_domain_destroy(struct k_mem_domain *domain) #else /* Memory domains each have a set of page tables assigned to them */ +/* + * Pool of free memory pages for copying page tables, as needed. + */ +#define PTABLE_COPY_SIZE (NUM_TABLE_PAGES * CONFIG_MMU_PAGE_SIZE) + +static uint8_t __noinit + page_pool[PTABLE_COPY_SIZE * CONFIG_X86_MAX_ADDITIONAL_MEM_DOMAINS] + __aligned(CONFIG_MMU_PAGE_SIZE); + +static uint8_t *page_pos = page_pool + sizeof(page_pool); + +/* Return a zeroed and suitably aligned memory page for page table data + * from the global page pool + */ +static void *page_pool_get(void) +{ + void *ret; + + if (page_pos == page_pool) { + ret = NULL; + } else { + page_pos -= CONFIG_MMU_PAGE_SIZE; + ret = page_pos; + } + + if (ret != NULL) { + memset(ret, 0, CONFIG_MMU_PAGE_SIZE); + } + + return ret; +} + +/* Debugging function to show how many pages are free in the pool */ +static inline unsigned int pages_free(void) +{ + return (page_pos - page_pool) / CONFIG_MMU_PAGE_SIZE; +} + /** * Duplicate an entire set of page tables * @@ -1635,9 +1565,6 @@ int arch_mem_domain_init(struct k_mem_domain *domain) if (ret == 0) { sys_slist_append(&x86_domain_list, &domain->arch.node); } - - LOG_DBG("page pool pages free: %u / %u\n", pages_free(), - CONFIG_X86_MMU_PAGE_POOL_PAGES); k_spin_unlock(&x86_mmu_lock, key); return ret; @@ -1775,3 +1702,172 @@ void z_x86_current_stack_perms(void) #endif } #endif /* CONFIG_USERSPACE */ + +#ifdef CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES +/* Selected on PC-like targets at the SOC level. + * + * Best is to do some E820 or similar enumeration to specifically identify + * all page frames which are reserved by the hardware or firmware. + * + * For now, just reserve everything in the first megabyte of physical memory. + */ +void arch_reserved_pages_update(void) +{ + for (uintptr_t pos = 0; pos < (1024 * 1024); + pos += CONFIG_MMU_PAGE_SIZE) { + struct z_page_frame *pf = z_phys_to_page_frame(pos); + + pf->flags |= Z_PAGE_FRAME_RESERVED; + } +} +#endif /* CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES */ + +#ifdef CONFIG_DEMAND_PAGING +#define PTE_MASK (paging_levels[PTE_LEVEL].mask) + +void arch_mem_page_out(void *addr, uintptr_t location) +{ + pentry_t mask = PTE_MASK | MMU_P | MMU_A; + + /* Accessed bit set to guarantee the entry is not completely 0 in + * case of location value 0. A totally 0 PTE is un-mapped. + */ + range_map(addr, location, CONFIG_MMU_PAGE_SIZE, MMU_A, mask, + OPTION_FLUSH); +} + +void arch_mem_page_in(void *addr, uintptr_t phys) +{ + pentry_t mask = PTE_MASK | MMU_P | MMU_D | MMU_A; + + range_map(addr, phys, CONFIG_MMU_PAGE_SIZE, MMU_P, mask, + OPTION_FLUSH); +} + +void arch_mem_scratch(uintptr_t phys) +{ + page_map_set(z_x86_page_tables_get(), Z_SCRATCH_PAGE, + phys | MMU_P | MMU_RW | MMU_XD, NULL, MASK_ALL, + OPTION_FLUSH); +} + +uintptr_t arch_page_info_get(void *addr, uintptr_t *phys, bool clear_accessed) +{ + pentry_t all_pte, mask; + uint32_t options; + + /* What to change, if anything, in the page_map_set() calls */ + if (clear_accessed) { + mask = MMU_A; + options = OPTION_FLUSH; + } else { + /* In this configuration page_map_set() just queries the + * page table and makes no changes + */ + mask = 0; + options = 0; + } + + page_map_set(z_x86_kernel_ptables, addr, 0, &all_pte, mask, options); + + /* Un-mapped PTEs are completely zeroed. No need to report anything + * else in this case. + */ + if (all_pte == 0) { + return ARCH_DATA_PAGE_NOT_MAPPED; + } + +#if defined(CONFIG_USERSPACE) && !defined(CONFIG_X86_COMMON_PAGE_TABLE) + /* Don't bother looking at other page tables if non-present as we + * are not required to report accurate accessed/dirty in this case + * and all mappings are otherwise the same. + */ + if ((all_pte & MMU_P) != 0) { + sys_snode_t *node; + + /* IRQs are locked, safe to do this */ + SYS_SLIST_FOR_EACH_NODE(&x86_domain_list, node) { + pentry_t cur_pte; + struct arch_mem_domain *domain = + CONTAINER_OF(node, struct arch_mem_domain, + node); + + page_map_set(domain->ptables, addr, 0, &cur_pte, + mask, options | OPTION_USER); + + /* Logical OR of relevant PTE in all page tables. + * addr/location and present state should be identical + * among them. + */ + all_pte |= cur_pte; + } + } +#endif /* USERSPACE && ~X86_COMMON_PAGE_TABLE */ + + /* NOTE: We are truncating the PTE on PAE systems, whose pentry_t + * are larger than a uintptr_t. + * + * We currently aren't required to report back XD state (bit 63), and + * Zephyr just doesn't support large physical memory on 32-bit + * systems, PAE was only implemented for XD support. + */ + if (phys != NULL) { + *phys = (uintptr_t)get_entry_phys(all_pte, PTE_LEVEL); + } + + /* We don't filter out any other bits in the PTE and the kernel + * ignores them. For the case of ARCH_DATA_PAGE_NOT_MAPPED, + * we use a bit which is never set in a real PTE (the PAT bit) in the + * current system. + * + * The other ARCH_DATA_PAGE_* macros are defined to their corresponding + * bits in the PTE. + */ + return (uintptr_t)all_pte; +} + +enum arch_page_location arch_page_location_get(void *addr, uintptr_t *location) +{ + pentry_t pte; + int level; + + /* TODO: since we only have to query the current set of page tables, + * could optimize this with recursive page table mapping + */ + pentry_get(&level, &pte, z_x86_page_tables_get(), addr); + + if (pte == 0) { + /* Not mapped */ + return ARCH_PAGE_LOCATION_BAD; + } + + __ASSERT(level == PTE_LEVEL, "bigpage found at %p", addr); + *location = (uintptr_t)get_entry_phys(pte, PTE_LEVEL); + + if ((pte & MMU_P) != 0) { + return ARCH_PAGE_LOCATION_PAGED_IN; + } else { + return ARCH_PAGE_LOCATION_PAGED_OUT; + } +} + +#ifdef CONFIG_X86_KPTI +bool z_x86_kpti_is_access_ok(void *addr, pentry_t *ptables) +{ + pentry_t pte; + int level; + + pentry_get(&level, &pte, ptables, addr); + + /* Might as well also check if it's un-mapped, normally we don't + * fetch the PTE from the page tables until we are inside + * z_page_fault() and call arch_page_fault_status_get() + */ + if (level != PTE_LEVEL || pte == 0 || is_flipped_pte(pte)) { + return false; + } + + return true; +} +#endif /* CONFIG_X86_KPTI */ +#endif /* CONFIG_DEMAND_PAGING */ diff --git a/arch/x86/gen_mmu.py b/arch/x86/gen_mmu.py index 6afbd877df13..22e7e738be8b 100755 --- a/arch/x86/gen_mmu.py +++ b/arch/x86/gen_mmu.py @@ -10,12 +10,11 @@ consult the IA Architecture SW Developer Manual, volume 3a, chapter 4. This script produces the initial page tables installed into the CPU -at early boot. These pages will have an identity mapping at -CONFIG_SRAM_BASE_ADDRESS of size CONFIG_SRAM_SIZE. The script takes -the 'zephyr_prebuilt.elf' as input to obtain region sizes, certain -memory addresses, and configuration values. +at early boot. These pages will have an identity mapping of the kernel +image. The script takes the 'zephyr_prebuilt.elf' as input to obtain region +sizes, certain memory addresses, and configuration values. -If CONFIG_SRAM_REGION_PERMISSIONS is not enabled, all RAM will be +If CONFIG_SRAM_REGION_PERMISSIONS is not enabled, the kernel image will be mapped with the Present and Write bits set. The linker scripts shouldn't add page alignment padding between sections. @@ -297,7 +296,17 @@ def levels(self): some kind of leaf page table class (Pt or PtXd)""" raise NotImplementedError() - def map_page(self, virt_addr, phys_addr, flags): + def new_child_table(self, table, virt_addr, depth): + new_table_addr = self.get_new_mmutable_addr() + new_table = self.levels[depth]() + debug("new %s at physical addr 0x%x" + % (self.levels[depth].__name__, new_table_addr)) + self.tables[new_table_addr] = new_table + table.map(virt_addr, new_table_addr, INT_FLAGS) + + return new_table + + def map_page(self, virt_addr, phys_addr, flags, reserve): """Map a virtual address to a physical address in the page tables, with provided access flags""" table = self.toplevel @@ -306,18 +315,29 @@ def map_page(self, virt_addr, phys_addr, flags): for depth in range(1, len(self.levels)): # Create child table if needed if not table.has_entry(virt_addr): - new_table_addr = self.get_new_mmutable_addr() - new_table = self.levels[depth]() - debug("new %s at physical addr 0x%x" - % (self.levels[depth].__name__, new_table_addr)) - self.tables[new_table_addr] = new_table - table.map(virt_addr, new_table_addr, INT_FLAGS) - table = new_table + table = self.new_child_table(table, virt_addr, depth) else: table = self.tables[table.lookup(virt_addr)] # Set up entry in leaf page table - table.map(virt_addr, phys_addr, flags) + if not reserve: + table.map(virt_addr, phys_addr, flags) + + def reserve(self, virt_base, size): + debug("Reserving paging structures 0x%x (%d)" % + (virt_base, size)) + + align_check(virt_base, size) + + # How much memory is covered by leaf page table + scope = 1 << self.levels[-2].addr_shift + + if virt_base % scope != 0: + error("misaligned virtual address space, 0x%x not a multiple of 0x%x" % + (virt_base, scope)) + + for addr in range(virt_base, virt_base + size, scope): + self.map_page(addr, 0, 0, True) def map(self, phys_base, size, flags): """Identity map an address range in the page tables, with provided @@ -332,7 +352,7 @@ def map(self, phys_base, size, flags): # Never map the NULL page continue - self.map_page(addr, addr, flags) + self.map_page(addr, addr, flags, False) def set_region_perms(self, name, flags): """Set access permissions for a named region that is already mapped @@ -436,14 +456,29 @@ def main(): debug("building %s" % pclass.__name__) - ram_base = syms["CONFIG_SRAM_BASE_ADDRESS"] - ram_size = syms["CONFIG_SRAM_SIZE"] * 1024 + vm_base = syms["CONFIG_KERNEL_VM_BASE"] + # Work around #31562 + vm_size = syms["CONFIG_KERNEL_VM_SIZE"] & 0xFFFFFFFF + + if isdef("CONFIG_ARCH_MAPS_ALL_RAM"): + image_base = syms["CONFIG_SRAM_BASE_ADDRESS"] + image_size = syms["CONFIG_SRAM_SIZE"] * 1024 + else: + image_base = syms["z_mapped_start"] + image_size = syms["z_mapped_size"] ptables_phys = syms["z_x86_pagetables_start"] - debug("Base addresses: physical 0x%x size %d" % (ram_base, ram_size)) + debug("Address space: 0x%x - 0x%x size %x" % + (vm_base, vm_base + vm_size, vm_size)) + + debug("Zephyr image: 0x%x - 0x%x size %x" % + (image_base, image_base + image_size, image_size)) is_perm_regions = isdef("CONFIG_SRAM_REGION_PERMISSIONS") + if image_size >= vm_size: + error("VM size is too small (have 0x%x need more than 0x%x)" % (vm_size, image_size)) + if is_perm_regions: # Don't allow execution by default for any pages. We'll adjust this # in later calls to pt.set_region_perms() @@ -452,7 +487,19 @@ def main(): map_flags = FLAG_P pt = pclass(ptables_phys) - pt.map(ram_base, ram_size, map_flags | ENTRY_RW) + # Instantiate all the paging structures for the address space + pt.reserve(vm_base, vm_size) + # Map the zephyr image + pt.map(image_base, image_size, map_flags | ENTRY_RW) + + if isdef("CONFIG_X86_64"): + # 64-bit has a special region in the first 64K to bootstrap other CPUs + # from real mode + locore_base = syms["_locore_start"] + locore_size = syms["_lodata_end"] - locore_base + debug("Base addresses: physical 0x%x size %d" % (locore_base, + locore_size)) + pt.map(locore_base, locore_size, map_flags | ENTRY_RW) if isdef("CONFIG_XIP"): # Additionally identity-map all ROM as read-only diff --git a/arch/x86/include/x86_mmu.h b/arch/x86/include/x86_mmu.h index c4c807538386..02835aa69499 100644 --- a/arch/x86/include/x86_mmu.h +++ b/arch/x86/include/x86_mmu.h @@ -81,9 +81,10 @@ * walking and creating page tables. */ #ifdef CONFIG_MMU -#define Z_X86_VIRT_OFFSET (CONFIG_KERNEL_VM_BASE - CONFIG_SRAM_BASE_ADDRESS) +#define Z_X86_VIRT_OFFSET ((CONFIG_KERNEL_VM_BASE + CONFIG_KERNEL_VM_OFFSET) - \ + CONFIG_SRAM_BASE_ADDRESS) #else -#define Z_X86_VIRT_OFFSET 0 +#define Z_X86_VIRT_OFFSET 0 #endif /* ASM code */ @@ -168,6 +169,13 @@ void z_x86_set_stack_guard(k_thread_stack_t *stack); * IDT, etc) */ extern uint8_t z_shared_kernel_page_start; + +#ifdef CONFIG_DEMAND_PAGING +/* Called from page fault handler. ptables here is the ptage tables for the + * faulting user thread and not the current set of page tables + */ +extern bool z_x86_kpti_is_access_ok(void *virt, pentry_t *ptables) +#endif /* CONFIG_DEMAND_PAGING */ #endif /* CONFIG_X86_KPTI */ #endif /* CONFIG_USERSPACE */ diff --git a/boards/x86/acrn/acrn.dts b/boards/x86/acrn/acrn.dts index b5c3c06cee75..a4979cc71f88 100644 --- a/boards/x86/acrn/acrn.dts +++ b/boards/x86/acrn/acrn.dts @@ -9,6 +9,7 @@ #include #define DT_DRAM_SIZE DT_SIZE_K(8192) +#define DT_DRAM_BASE 0 #include diff --git a/boards/x86/acrn/acrn_defconfig b/boards/x86/acrn/acrn_defconfig index 4910fc409cc2..0933db3a22f2 100644 --- a/boards/x86/acrn/acrn_defconfig +++ b/boards/x86/acrn/acrn_defconfig @@ -13,4 +13,4 @@ CONFIG_UART_CONSOLE=y CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=25000000 CONFIG_BUILD_OUTPUT_BIN=y CONFIG_SHELL_BACKEND_SERIAL_INTERRUPT_DRIVEN=n -CONFIG_X86_MMU_PAGE_POOL_PAGES=29 +CONFIG_KERNEL_VM_SIZE=0x1000000 diff --git a/boards/x86/acrn/acrn_ehl_crb_defconfig b/boards/x86/acrn/acrn_ehl_crb_defconfig index f119d09a89c7..932a27588ea0 100644 --- a/boards/x86/acrn/acrn_ehl_crb_defconfig +++ b/boards/x86/acrn/acrn_ehl_crb_defconfig @@ -13,4 +13,4 @@ CONFIG_UART_CONSOLE=y CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=1900000000 CONFIG_BUILD_OUTPUT_BIN=y CONFIG_SHELL_BACKEND_SERIAL_INTERRUPT_DRIVEN=n -CONFIG_X86_MMU_PAGE_POOL_PAGES=29 +CONFIG_KERNEL_VM_SIZE=0x1000000 diff --git a/boards/x86/ehl_crb/Kconfig.defconfig b/boards/x86/ehl_crb/Kconfig.defconfig index 76a6749000f1..c7ed624ef5b6 100644 --- a/boards/x86/ehl_crb/Kconfig.defconfig +++ b/boards/x86/ehl_crb/Kconfig.defconfig @@ -12,9 +12,6 @@ config BUILD_OUTPUT_STRIPPED config MP_NUM_CPUS default 2 -config X86_MMU_PAGE_POOL_PAGES - default 3072 if X86_MMU - endif # BOARD_EHL_CRB if BOARD_EHL_CRB_SBL @@ -28,9 +25,6 @@ config BUILD_OUTPUT_STRIPPED config MP_NUM_CPUS default 2 -config X86_MMU_PAGE_POOL_PAGES - default 3072 if X86_MMU - config SHELL_BACKEND_SERIAL_INTERRUPT_DRIVEN depends on SHELL_BACKEND_SERIAL default n diff --git a/boards/x86/ehl_crb/ehl_crb_defconfig b/boards/x86/ehl_crb/ehl_crb_defconfig index def86d2703cb..f1a2e6025eef 100644 --- a/boards/x86/ehl_crb/ehl_crb_defconfig +++ b/boards/x86/ehl_crb/ehl_crb_defconfig @@ -10,4 +10,4 @@ CONFIG_UART_NS16550=y CONFIG_UART_CONSOLE=y CONFIG_X2APIC=y CONFIG_SMP=y -CONFIG_X86_MMU_PAGE_POOL_PAGES=3092 +CONFIG_KERNEL_VM_SIZE=0x80800000 diff --git a/boards/x86/ehl_crb/ehl_crb_sbl_defconfig b/boards/x86/ehl_crb/ehl_crb_sbl_defconfig index 9dcb22c4014d..35fc7182f399 100644 --- a/boards/x86/ehl_crb/ehl_crb_sbl_defconfig +++ b/boards/x86/ehl_crb/ehl_crb_sbl_defconfig @@ -10,4 +10,4 @@ CONFIG_UART_NS16550=y CONFIG_UART_CONSOLE=y CONFIG_X2APIC=y CONFIG_SMP=y -CONFIG_X86_MMU_PAGE_POOL_PAGES=3092 +CONFIG_KERNEL_VM_SIZE=0x80800000 diff --git a/boards/x86/minnowboard/Kconfig.defconfig b/boards/x86/minnowboard/Kconfig.defconfig index 06e54af69418..158d8f2a7fd8 100644 --- a/boards/x86/minnowboard/Kconfig.defconfig +++ b/boards/x86/minnowboard/Kconfig.defconfig @@ -8,7 +8,4 @@ config BOARD config BUILD_OUTPUT_STRIPPED default y -config X86_MMU_PAGE_POOL_PAGES - default 3086 if X86_MMU - endif # BOARD_MINNOWBOARD diff --git a/boards/x86/minnowboard/minnowboard_defconfig b/boards/x86/minnowboard/minnowboard_defconfig index bd4a34645e77..8f2fc2f93131 100644 --- a/boards/x86/minnowboard/minnowboard_defconfig +++ b/boards/x86/minnowboard/minnowboard_defconfig @@ -10,3 +10,4 @@ CONFIG_SERIAL=y CONFIG_UART_NS16550=y CONFIG_UART_CONSOLE=y CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=25000000 +CONFIG_KERNEL_VM_SIZE=0x80800000 diff --git a/boards/x86/qemu_x86/qemu_x86.dts b/boards/x86/qemu_x86/qemu_x86.dts index 03cfdab115b5..25d6849bc431 100644 --- a/boards/x86/qemu_x86/qemu_x86.dts +++ b/boards/x86/qemu_x86/qemu_x86.dts @@ -4,6 +4,9 @@ #include +#ifndef DT_DRAM_BASE +#define DT_DRAM_BASE 0 +#endif #ifndef DT_DRAM_SIZE #define DT_DRAM_SIZE DT_SIZE_K(4096) #endif diff --git a/boards/x86/qemu_x86/qemu_x86_64_defconfig b/boards/x86/qemu_x86/qemu_x86_64_defconfig index 85dda2cb7a19..8f1cbe0d91fa 100644 --- a/boards/x86/qemu_x86/qemu_x86_64_defconfig +++ b/boards/x86/qemu_x86/qemu_x86_64_defconfig @@ -17,4 +17,3 @@ CONFIG_MP_NUM_CPUS=2 CONFIG_X86_MMU=y CONFIG_X86_VERY_EARLY_CONSOLE=y CONFIG_QEMU_ICOUNT=n -CONFIG_X86_MMU_PAGE_POOL_PAGES=23 diff --git a/boards/x86/qemu_x86/qemu_x86_64_nokpti_defconfig b/boards/x86/qemu_x86/qemu_x86_64_nokpti_defconfig index 0f161f810791..082f0427f7b0 100644 --- a/boards/x86/qemu_x86/qemu_x86_64_nokpti_defconfig +++ b/boards/x86/qemu_x86/qemu_x86_64_nokpti_defconfig @@ -18,4 +18,3 @@ CONFIG_X86_MMU=y CONFIG_X86_VERY_EARLY_CONSOLE=y CONFIG_QEMU_ICOUNT=n CONFIG_X86_KPTI=n -CONFIG_X86_MMU_PAGE_POOL_PAGES=16 diff --git a/boards/x86/qemu_x86/qemu_x86_coverage_defconfig b/boards/x86/qemu_x86/qemu_x86_coverage_defconfig index 7ee9871a78e2..24929976a1e1 100644 --- a/boards/x86/qemu_x86/qemu_x86_coverage_defconfig +++ b/boards/x86/qemu_x86/qemu_x86_coverage_defconfig @@ -12,7 +12,6 @@ CONFIG_UART_CONSOLE=y CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=25000000 CONFIG_TEST_RANDOM_GENERATOR=y CONFIG_X86_MMU=y -CONFIG_X86_MMU_PAGE_POOL_PAGES=17 CONFIG_DEBUG_INFO=y CONFIG_SCHED_SCALABLE=y CONFIG_WAITQ_SCALABLE=y diff --git a/boards/x86/qemu_x86/qemu_x86_defconfig b/boards/x86/qemu_x86/qemu_x86_defconfig index e4fb41517122..9db929558b7d 100644 --- a/boards/x86/qemu_x86/qemu_x86_defconfig +++ b/boards/x86/qemu_x86/qemu_x86_defconfig @@ -12,7 +12,6 @@ CONFIG_UART_CONSOLE=y CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=25000000 CONFIG_TEST_RANDOM_GENERATOR=y CONFIG_X86_MMU=y -CONFIG_X86_MMU_PAGE_POOL_PAGES=17 CONFIG_DEBUG_INFO=y CONFIG_SCHED_SCALABLE=y CONFIG_WAITQ_SCALABLE=y diff --git a/boards/x86/qemu_x86/qemu_x86_nokpti_defconfig b/boards/x86/qemu_x86/qemu_x86_nokpti_defconfig index 31557f2c3eb3..6443b51b8f19 100644 --- a/boards/x86/qemu_x86/qemu_x86_nokpti_defconfig +++ b/boards/x86/qemu_x86/qemu_x86_nokpti_defconfig @@ -12,7 +12,6 @@ CONFIG_UART_CONSOLE=y CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=25000000 CONFIG_TEST_RANDOM_GENERATOR=y CONFIG_X86_MMU=y -CONFIG_X86_MMU_PAGE_POOL_PAGES=12 CONFIG_DEBUG_INFO=y CONFIG_SCHED_SCALABLE=y CONFIG_WAITQ_SCALABLE=y diff --git a/boards/x86/qemu_x86/qemu_x86_nopae_defconfig b/boards/x86/qemu_x86/qemu_x86_nopae_defconfig index 2724a0dc825f..60dc38b6ea1d 100644 --- a/boards/x86/qemu_x86/qemu_x86_nopae_defconfig +++ b/boards/x86/qemu_x86/qemu_x86_nopae_defconfig @@ -12,7 +12,6 @@ CONFIG_UART_CONSOLE=y CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=25000000 CONFIG_TEST_RANDOM_GENERATOR=y CONFIG_X86_MMU=y -CONFIG_X86_MMU_PAGE_POOL_PAGES=10 CONFIG_DEBUG_INFO=y CONFIG_SCHED_SCALABLE=y CONFIG_WAITQ_SCALABLE=y diff --git a/boards/x86/qemu_x86/qemu_x86_tiny.dts b/boards/x86/qemu_x86/qemu_x86_tiny.dts index a32afaa5b862..9ce52b60d88f 100644 --- a/boards/x86/qemu_x86/qemu_x86_tiny.dts +++ b/boards/x86/qemu_x86/qemu_x86_tiny.dts @@ -4,5 +4,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -#define DT_DRAM_SIZE DT_SIZE_K(2048) +#define DT_DRAM_BASE 0x100000 +#define DT_DRAM_SIZE DT_SIZE_K(256) #include "qemu_x86.dts" diff --git a/boards/x86/qemu_x86/qemu_x86_tiny_defconfig b/boards/x86/qemu_x86/qemu_x86_tiny_defconfig index bd2d264b25ff..f9a2f477a9df 100644 --- a/boards/x86/qemu_x86/qemu_x86_tiny_defconfig +++ b/boards/x86/qemu_x86/qemu_x86_tiny_defconfig @@ -22,4 +22,8 @@ CONFIG_X86_PAE=n CONFIG_X86_COMMON_PAGE_TABLE=y CONFIG_X86_KPTI=n CONFIG_KERNEL_VM_SIZE=0x400000 -CONFIG_X86_MMU_PAGE_POOL_PAGES=0 +CONFIG_KERNEL_VM_BASE=0x0 +CONFIG_KERNEL_VM_OFFSET=0x100000 +CONFIG_X86_KERNEL_OFFSET=0 +CONFIG_DEMAND_PAGING=y +CONFIG_BACKING_STORE_RAM=y diff --git a/boards/x86/up_squared/Kconfig.defconfig b/boards/x86/up_squared/Kconfig.defconfig index 09e127b79623..83be6ec1f8d5 100644 --- a/boards/x86/up_squared/Kconfig.defconfig +++ b/boards/x86/up_squared/Kconfig.defconfig @@ -11,9 +11,6 @@ config BUILD_OUTPUT_STRIPPED config MP_NUM_CPUS default 2 -config X86_MMU_PAGE_POOL_PAGES - default 3092 if X86_MMU - endif # BOARD_UP_SQUARED @@ -25,7 +22,4 @@ config BOARD config BUILD_OUTPUT_STRIPPED default y -config X86_MMU_PAGE_POOL_PAGES - default 3086 if X86_MMU - endif # BOARD_UP_SQUARED_32 diff --git a/boards/x86/up_squared/up_squared_32_defconfig b/boards/x86/up_squared/up_squared_32_defconfig index 2094fb73ea62..8d39814de640 100644 --- a/boards/x86/up_squared/up_squared_32_defconfig +++ b/boards/x86/up_squared/up_squared_32_defconfig @@ -10,3 +10,4 @@ CONFIG_UART_NS16550=y CONFIG_UART_CONSOLE=y CONFIG_I2C=y CONFIG_X2APIC=y +CONFIG_KERNEL_VM_SIZE=0x80800000 diff --git a/boards/x86/up_squared/up_squared_defconfig b/boards/x86/up_squared/up_squared_defconfig index 5e8005a539c6..ac80434b3d62 100644 --- a/boards/x86/up_squared/up_squared_defconfig +++ b/boards/x86/up_squared/up_squared_defconfig @@ -11,3 +11,4 @@ CONFIG_UART_CONSOLE=y CONFIG_X2APIC=y CONFIG_SMP=y CONFIG_MP_NUM_CPUS=2 +CONFIG_KERNEL_VM_SIZE=0x80800000 diff --git a/dts/x86/ia32.dtsi b/dts/x86/ia32.dtsi index 1b1cd087c400..bda175c76c7b 100644 --- a/dts/x86/ia32.dtsi +++ b/dts/x86/ia32.dtsi @@ -29,7 +29,7 @@ dram0: memory@0 { device_type = "memory"; - reg = <0x0 DT_DRAM_SIZE>; + reg = ; }; soc { diff --git a/include/arch/arm/aarch64/scripts/linker.ld b/include/arch/arm/aarch64/scripts/linker.ld index fd8d887664e3..2b123e5fb30a 100644 --- a/include/arch/arm/aarch64/scripts/linker.ld +++ b/include/arch/arm/aarch64/scripts/linker.ld @@ -104,7 +104,9 @@ SECTIONS SECTION_PROLOGUE(_TEXT_SECTION_NAME,,) { _image_text_start = .; - +#ifndef CONFIG_XIP + z_mapped_start = .; +#endif _vector_start = .; KEEP(*(.exc_vector_table)) KEEP(*(".exc_vector_table.*")) @@ -210,6 +212,9 @@ SECTIONS */ . = ALIGN(_region_min_align); _image_ram_start = .; +#ifdef CONFIG_XIP + z_mapped_start = .; +#endif /* Located in generated directory. This file is populated by the * zephyr_linker_sources() Cmake function. @@ -285,6 +290,8 @@ SECTIONS _image_ram_end = .; _end = .; /* end of image */ + z_mapped_end = .; + z_mapped_size = z_mapped_end - z_mapped_start; __kernel_ram_end = RAM_ADDR + RAM_SIZE; __kernel_ram_size = __kernel_ram_end - __kernel_ram_start; diff --git a/include/arch/x86/ia32/linker.ld b/include/arch/x86/ia32/linker.ld index 234a616883fe..75be8137fba6 100644 --- a/include/arch/x86/ia32/linker.ld +++ b/include/arch/x86/ia32/linker.ld @@ -51,14 +51,20 @@ #define RAMABLE_REGION RAM #endif +#ifdef CONFIG_MMU + #define MMU_PAGE_ALIGN . = ALIGN(CONFIG_MMU_PAGE_SIZE); +#else + #define MMU_PAGE_ALIGN +#endif + /* Used to align areas with separate memory permission characteristics * so that the page permissions can be set in the MMU. Without this, * the kernel is just one blob with the same RWX permissions on all RAM */ #ifdef CONFIG_SRAM_REGION_PERMISSIONS - #define MMU_PAGE_ALIGN . = ALIGN(CONFIG_MMU_PAGE_SIZE); + #define MMU_PAGE_ALIGN_PERM MMU_PAGE_ALIGN #else - #define MMU_PAGE_ALIGN + #define MMU_PAGE_ALIGN_PERM #endif ENTRY(CONFIG_KERNEL_ENTRY) @@ -90,6 +96,9 @@ SECTIONS SECTION_PROLOGUE(_TEXT_SECTION_NAME,,) { _image_text_start = .; +#ifndef CONFIG_XIP + z_mapped_start = .; +#endif /* Located in generated directory. This file is populated by calling * zephyr_linker_sources(ROM_START ...). This typically contains the vector @@ -112,7 +121,7 @@ SECTIONS #include - MMU_PAGE_ALIGN + MMU_PAGE_ALIGN_PERM } GROUP_LINK_IN(ROMABLE_REGION) _image_text_end = .; @@ -156,7 +165,7 @@ SECTIONS #include - MMU_PAGE_ALIGN + MMU_PAGE_ALIGN_PERM /* ROM ends here, position counter will now be in RAM areas */ #ifdef CONFIG_XIP _image_rom_end = .; @@ -177,6 +186,10 @@ SECTIONS /* RAMABLE_REGION */ GROUP_START(RAMABLE_REGION) +#ifdef CONFIG_XIP + MMU_PAGE_ALIGN + z_mapped_start = .; +#endif /* Located in generated directory. This file is populated by the * zephyr_linker_sources() Cmake function. */ @@ -184,8 +197,8 @@ SECTIONS #ifdef CONFIG_USERSPACE /* APP SHARED MEMORY REGION */ -#define SMEM_PARTITION_ALIGN(size) MMU_PAGE_ALIGN -#define APP_SHARED_ALIGN MMU_PAGE_ALIGN +#define SMEM_PARTITION_ALIGN(size) MMU_PAGE_ALIGN_PERM +#define APP_SHARED_ALIGN MMU_PAGE_ALIGN_PERM #include @@ -198,7 +211,7 @@ SECTIONS SECTION_PROLOGUE(_BSS_SECTION_NAME, (NOLOAD),) { - MMU_PAGE_ALIGN + MMU_PAGE_ALIGN_PERM #if !defined(CONFIG_USERSPACE) _image_ram_start = .; #endif @@ -227,7 +240,7 @@ SECTIONS #include - MMU_PAGE_ALIGN + MMU_PAGE_ALIGN_PERM SECTION_DATA_PROLOGUE(_DATA_SECTION_NAME,,) { @@ -261,7 +274,7 @@ SECTIONS #include #ifdef CONFIG_X86_KPTI - MMU_PAGE_ALIGN + MMU_PAGE_ALIGN_PERM z_shared_kernel_page_start = .; /* Special page containing supervisor data that is still mapped in * user mode page tables. IDT, GDT, TSSes, trampoline stack, and @@ -302,7 +315,7 @@ SECTIONS #ifdef CONFIG_X86_KPTI z_trampoline_stack_start = .; - MMU_PAGE_ALIGN + MMU_PAGE_ALIGN_PERM z_trampoline_stack_end = .; z_shared_kernel_page_end = .; @@ -334,6 +347,8 @@ SECTIONS _image_ram_end = .; _image_ram_all = (KERNEL_BASE_ADDR + KERNEL_RAM_SIZE) - _image_ram_start; + z_mapped_end = .; + z_mapped_size = z_mapped_end - z_mapped_start; _end = .; /* end of image */ GROUP_END(RAMABLE_REGION) diff --git a/include/arch/x86/intel64/linker.ld b/include/arch/x86/intel64/linker.ld index 70e78db41f6b..ef091a4de648 100644 --- a/include/arch/x86/intel64/linker.ld +++ b/include/arch/x86/intel64/linker.ld @@ -9,14 +9,16 @@ #define ROMABLE_REGION RAM #define RAMABLE_REGION RAM +#define MMU_PAGE_ALIGN . = ALIGN(CONFIG_MMU_PAGE_SIZE); + /* Used to align areas with separate memory permission characteristics * so that the page permissions can be set in the MMU. Without this, * the kernel is just one blob with the same RWX permissions on all RAM */ #ifdef CONFIG_SRAM_REGION_PERMISSIONS - #define MMU_PAGE_ALIGN . = ALIGN(CONFIG_MMU_PAGE_SIZE); + #define MMU_PAGE_ALIGN_PERM MMU_PAGE_ALIGN #else - #define MMU_PAGE_ALIGN + #define MMU_PAGE_ALIGN_PERM #endif ENTRY(CONFIG_KERNEL_ENTRY) @@ -34,12 +36,12 @@ SECTIONS _locore_start = .; *(.locore) *(.locore.*) - MMU_PAGE_ALIGN + MMU_PAGE_ALIGN_PERM _locore_end = .; _lorodata_start = .; *(.lorodata) - MMU_PAGE_ALIGN + MMU_PAGE_ALIGN_PERM _lodata_start = .; *(.lodata) @@ -54,7 +56,7 @@ SECTIONS * On x86-64 the IDT is in rodata and doesn't need to be in the * trampoline page. */ - MMU_PAGE_ALIGN + MMU_PAGE_ALIGN_PERM z_shared_kernel_page_start = .; #endif /* CONFIG_X86_KPTI */ @@ -63,14 +65,14 @@ SECTIONS #ifdef CONFIG_X86_KPTI *(.trampolines) - MMU_PAGE_ALIGN + MMU_PAGE_ALIGN_PERM z_shared_kernel_page_end = .; ASSERT(z_shared_kernel_page_end - z_shared_kernel_page_start == 4096, "shared kernel area is not one memory page"); #endif /* CONFIG_X86_KPTI */ - MMU_PAGE_ALIGN + . = ALIGN(CONFIG_MMU_PAGE_SIZE); _lodata_end = .; } > LOCORE @@ -87,12 +89,13 @@ SECTIONS { _image_rom_start = .; _image_text_start = .; + z_mapped_start = .; *(.text) *(.text.*) #include - MMU_PAGE_ALIGN + MMU_PAGE_ALIGN_PERM } GROUP_LINK_IN(ROMABLE_REGION) _image_text_end = .; @@ -122,15 +125,15 @@ SECTIONS #include - MMU_PAGE_ALIGN + MMU_PAGE_ALIGN_PERM _image_rodata_end = .; _image_rodata_size = _image_rodata_end - _image_rodata_start; _image_rom_end = .; #ifdef CONFIG_USERSPACE /* APP SHARED MEMORY REGION */ -#define SMEM_PARTITION_ALIGN(size) MMU_PAGE_ALIGN -#define APP_SHARED_ALIGN MMU_PAGE_ALIGN +#define SMEM_PARTITION_ALIGN(size) MMU_PAGE_ALIGN_PERM +#define APP_SHARED_ALIGN MMU_PAGE_ALIGN_PERM #include @@ -147,7 +150,7 @@ SECTIONS SECTION_PROLOGUE(_BSS_SECTION_NAME, (NOLOAD), ALIGN(16)) { - MMU_PAGE_ALIGN + MMU_PAGE_ALIGN_PERM #ifndef CONFIG_USERSPACE _image_ram_start = .; #endif @@ -179,14 +182,17 @@ SECTIONS /* Must be last in RAM */ #include - + MMU_PAGE_ALIGN _image_ram_end = .; + z_mapped_end = .; _end = .; /* All unused memory also owned by the kernel for heaps */ __kernel_ram_end = KERNEL_BASE_ADDR + KERNEL_RAM_SIZE; __kernel_ram_size = __kernel_ram_end - __kernel_ram_start; + z_mapped_size = z_mapped_end - z_mapped_start; + #include /DISCARD/ : diff --git a/include/arch/x86/mmustructs.h b/include/arch/x86/mmustructs.h index 36334e394ed2..511d6f61e349 100644 --- a/include/arch/x86/mmustructs.h +++ b/include/arch/x86/mmustructs.h @@ -24,6 +24,14 @@ #define Z_X86_MMU_XD 0 #endif +/* For these we'll just use the same bits in the PTE */ +#define ARCH_DATA_PAGE_DIRTY ((uintptr_t)BIT(6)) +#define ARCH_DATA_PAGE_LOADED ((uintptr_t)BIT(0)) +#define ARCH_DATA_PAGE_ACCESSED ((uintptr_t)BIT(5)) + +/* Use an PAT bit for this one since it's never set in a mapped PTE */ +#define ARCH_DATA_PAGE_NOT_MAPPED ((uintptr_t)BIT(7)) + /* Always true with 32-bit page tables, don't enable * CONFIG_EXECUTE_XOR_WRITE and expect it to work for you */ diff --git a/include/linker/kobject.ld b/include/linker/kobject.ld index d5499efc6e4f..c02c10719b6e 100644 --- a/include/linker/kobject.ld +++ b/include/linker/kobject.ld @@ -5,6 +5,7 @@ */ #ifdef CONFIG_USERSPACE + z_kobject_data_begin = .; /* Constraints: * * - changes to the size of this section between build phases diff --git a/include/linker/linker-defs.h b/include/linker/linker-defs.h index 91dc1abb07a6..383fef15e848 100644 --- a/include/linker/linker-defs.h +++ b/include/linker/linker-defs.h @@ -182,6 +182,13 @@ extern char __data_ram_start[]; extern char __data_ram_end[]; #endif /* CONFIG_XIP */ +#ifdef CONFIG_MMU +/* Virtual addresses of page-aligned kernel image mapped into RAM at boot */ +extern char z_mapped_start[]; +extern char z_mapped_end[]; +extern char z_mapped_size[]; +#endif /* CONFIG_MMU */ + /* Includes text and rodata */ extern char _image_rom_start[]; extern char _image_rom_end[]; @@ -296,6 +303,7 @@ extern char z_priv_stacks_ram_start[]; extern char z_priv_stacks_ram_end[]; extern char z_user_stacks_start[]; extern char z_user_stacks_end[]; +extern char z_kobject_data_begin[]; #endif /* CONFIG_USERSPACE */ #ifdef CONFIG_THREAD_LOCAL_STORAGE diff --git a/include/sys/mem_manage.h b/include/sys/mem_manage.h index faa903b0f85b..197c3cd2ead9 100644 --- a/include/sys/mem_manage.h +++ b/include/sys/mem_manage.h @@ -14,13 +14,13 @@ */ /** No caching. Most drivers want this. */ -#define K_MEM_CACHE_NONE 0 +#define K_MEM_CACHE_NONE 2 /** Write-through caching. Used by certain drivers. */ #define K_MEM_CACHE_WT 1 /** Full write-back caching. Any RAM mapped wants this. */ -#define K_MEM_CACHE_WB 2 +#define K_MEM_CACHE_WB 0 /** Reserved bits for cache modes in k_map() flags argument */ #define K_MEM_CACHE_MASK (BIT(3) - 1) @@ -51,9 +51,10 @@ extern "C" { /** * Map a physical memory region into the kernel's virtual address space * - * Given a physical address and a size, return a linear address - * representing the base of where the physical region is mapped in - * the virtual address space for the Zephyr kernel. + * This function is intended for mapping memory-mapped I/O regions into + * the virtual address space. Given a physical address and a size, return a + * linear address representing the base of where the physical region is mapped + * in the virtual address space for the Zephyr kernel. * * This function alters the active page tables in the area reserved * for the kernel. This function will choose the virtual address @@ -70,12 +71,18 @@ extern "C" { * with user access and code execution forbidden. This policy is changed * by passing K_MEM_CACHE_* and K_MEM_PERM_* macros into the 'flags' parameter. * - * If there is insufficient virtual address space for the mapping, or - * bad flags are passed in, or if additional memory is needed to update - * page tables that is not available, this will generate a kernel panic. + * If there is insufficient virtual address space for the mapping this will + * generate a kernel panic. * * This API is only available if CONFIG_MMU is enabled. * + * It is highly discouraged to use this function to map system RAM page + * frames. It may conflict with anonymous memory mappings and demand paging + * and produce undefined behavior. Do not use this for RAM unless you know + * exactly what you are doing. If you need a chunk of memory, use k_mem_map(). + * If you need a contiguous buffer of physical memory, statically declare it + * and pin it at build time, it will be mapped when the system boots. + * * This API is part of infrastructure still under development and may * change. * @@ -87,6 +94,100 @@ extern "C" { void z_phys_map(uint8_t **virt_ptr, uintptr_t phys, size_t size, uint32_t flags); +/* + * k_mem_map() control flags + */ + +/** + * @def K_MEM_MAP_UNINIT + * + * @brief The mapped region is not guaranteed to be zeroed. + * + * This may improve performance. The associated page frames may contain + * indeterminate data, zeroes, or even sensitive information. + * + * This may not be used with K_MEM_PERM_USER as there are no circumstances + * where this is safe. + */ +#define K_MEM_MAP_UNINIT BIT(16) + +/** + * @def K_MEM_MAP_LOCK + * + * Region will be pinned in memory and never paged + * + * Such memory is guaranteed to never produce a page fault due to page-outs + * or copy-on-write once the mapping call has returned. Physical page frames + * will be pre-fetched as necessary and pinned. + */ +#define K_MEM_MAP_LOCK BIT(17) + +/** + * @def K_MEM_MAP_GUARD + * + * A un-mapped virtual guard page will be placed in memory immediately preceding + * the mapped region. This page will still be noted as being used by the + * virtual memory manager. The total size of the allocation will be the + * requested size plus the size of this guard page. The returned address + * pointer will not include the guard page immediately below it. The typical + * use-case is downward-growing thread stacks. + * + * Zephyr treats page faults on this guard page as a fatal K_ERR_STACK_CHK_FAIL + * if it determines it immediately precedes a stack buffer, this is + * implemented in the architecture layer. + */ +#define K_MEM_MAP_GUARD BIT(18) + +/** + * Return the amount of free memory available + * + * The returned value will reflect how many free RAM page frames are available. + * If demand paging is enabled, it may still be possible to allocate more. + * + * The information reported by this function may go stale immediately if + * concurrent memory mappings or page-ins take place. + * + * @return Free physical RAM, in bytes + */ +size_t k_mem_free_get(void); + +/** + * Map anonymous memory into Zephyr's address space + * + * This function effectively increases the data space available to Zephyr. + * The kernel will choose a base virtual address and return it to the caller. + * The memory will have access permissions for all contexts set per the + * provided flags argument. + * + * If user thread access control needs to be managed in any way, do not enable + * K_MEM_PERM_USER flags here; instead manage the region's permissions + * with memory domain APIs after the mapping has been established. Setting + * K_MEM_PERM_USER here will allow all user threads to access this memory + * which is usually undesirable. + * + * Unless K_MEM_MAP_UNINIT is used, the returned memory will be zeroed. + * + * The mapped region is not guaranteed to be physically contiguous in memory. + * Physically contiguous buffers should be allocated statically and pinned + * at build time. + * + * Pages mapped in this way have write-back cache settings. + * + * The returned virtual memory pointer will be page-aligned. The size + * parameter, and any base address for re-mapping purposes must be page- + * aligned. + * + * Many K_MEM_MAP_* flags have been implemented to alter the behavior of this + * function, with details in the documentation for these flags. + * + * @param size Size of the memory mapping. This must be page-aligned. + * @param flags K_MEM_PERM_*, K_MEM_MAP_* control flags. + * @return The mapped memory location, or NULL if insufficient virtual address + * space, insufficient physical memory to establish the mapping, + * or insufficient memory for paging structures. + */ +void *k_mem_map(size_t size, uint32_t flags); + /** * Given an arbitrary region, provide a aligned region that covers it * @@ -103,6 +204,75 @@ void z_phys_map(uint8_t **virt_ptr, uintptr_t phys, size_t size, size_t k_mem_region_align(uintptr_t *aligned_addr, size_t *aligned_size, uintptr_t addr, size_t size, size_t align); +#ifdef CONFIG_DEMAND_PAGING +/** + * Evict a page-aligned virtual memory region to the backing store + * + * Useful if it is known that a memory region will not be used for some time. + * All the data pages within the specified region will be evicted to the + * backing store if they weren't already, with their associated page frames + * marked as available for mappings or page-ins. + * + * None of the associated page frames mapped to the provided region should + * be pinned. + * + * Note that there are no guarantees how long these pages will be evicted, + * they could take page faults immediately. + * + * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be + * called by ISRs as the backing store may be in-use. + * + * @param addr Base page-aligned virtual address + * @param size Page-aligned data region size + * @retval 0 Success + * @retval -ENOMEM Insufficient space in backing store to satisfy request. + * The region may be partially paged out. + */ +int k_mem_page_out(void *addr, size_t size); + +/** + * Load a virtual data region into memory + * + * After the function completes, all the page frames associated with this + * function will be paged in. However, they are not guaranteed to stay there. + * This is useful if the region is known to be used soon. + * + * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be + * called by ISRs as the backing store may be in-use. + * + * @param addr Base page-aligned virtual address + * @param size Page-aligned data region size + */ +void k_mem_page_in(void *addr, size_t size); + +/** + * Pin an aligned virtual data region, paging in as necessary + * + * After the function completes, all the page frames associated with this + * region will be resident in memory and pinned such that they stay that way. + * This is a stronger version of z_mem_page_in(). + * + * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be + * called by ISRs as the backing store may be in-use. + * + * @param addr Base page-aligned virtual address + * @param size Page-aligned data region size + */ +void k_mem_pin(void *addr, size_t size); + +/** + * Un-pin an aligned virtual data region + * + * After the function completes, all the page frames associated with this + * region will be no longer marked as pinned. This does not evict the region, + * follow this with z_mem_page_out() if you need that. + * + * @param addr Base page-aligned virtual address + * @param size Page-aligned data region size + */ +void k_mem_unpin(void *addr, size_t size); +#endif /* CONFIG_DEMAND_PAGING */ + #ifdef __cplusplus } #endif diff --git a/kernel/include/kernel_arch_interface.h b/kernel/include/kernel_arch_interface.h index c784852e809a..907cd6e934e6 100644 --- a/kernel/include/kernel_arch_interface.h +++ b/kernel/include/kernel_arch_interface.h @@ -241,6 +241,13 @@ static inline bool arch_is_in_isr(void); * to this API are assumed to be serialized, and indeed all usage will * originate from kernel/mm.c which handles virtual memory management. * + * Architectures are expected to pre-allocate page tables for the entire + * address space, as defined by CONFIG_KERNEL_VM_BASE and + * CONFIG_KERNEL_VM_SIZE. This operation should never require any kind of + * allocation for paging structures. + * + * Validation of arguments should be done via assertions. + * * This API is part of infrastructure still under development and may * change. * @@ -248,12 +255,8 @@ static inline bool arch_is_in_isr(void); * @param addr Page-aligned Source physical address to map * @param size Page-aligned size of the mapped memory region in bytes * @param flags Caching, access and control flags, see K_MAP_* macros - * @retval 0 Success - * @retval -ENOTSUP Unsupported cache mode with no suitable fallback, or - * unsupported flags - * @retval -ENOMEM Memory for additional paging structures unavailable */ -int arch_mem_map(void *dest, uintptr_t addr, size_t size, uint32_t flags); +void arch_mem_map(void *dest, uintptr_t addr, size_t size, uint32_t flags); /** * Remove mappings for a provided virtual address range @@ -282,6 +285,199 @@ int arch_mem_map(void *dest, uintptr_t addr, size_t size, uint32_t flags); * @param size Page-aligned region size */ void arch_mem_unmap(void *addr, size_t size); + +#ifdef CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES +/** + * Update page frame database with reserved pages + * + * Some page frames within system RAM may not be available for use. A good + * example of this is reserved regions in the first megabyte on PC-like systems. + * + * Implementations of this function should mark all relavent entries in + * z_page_frames with K_PAGE_FRAME_RESERVED. This function is called at + * early system initialization with mm_lock held. + */ +void arch_reserved_pages_update(void); +#endif /* ARCH_HAS_RESERVED_PAGE_FRAMES */ + +#ifdef CONFIG_DEMAND_PAGING +/** + * Update all page tables for a paged-out data page + * + * This function: + * - Sets the data page virtual address to trigger a fault if accessed that + * can be distinguished from access violations or un-mapped pages. + * - Saves the provided location value so that it can retrieved for that + * data page in the page fault handler. + * - The location value semantics are undefined here but the value will be + * always be page-aligned. It could be 0. + * + * If multiple page tables are in use, this must update all page tables. + * This function is called with interrupts locked. + * + * Calling this function on data pages which are already paged out is + * undefined behavior. + * + * This API is part of infrastructure still under development and may change. + */ +void arch_mem_page_out(void *addr, uintptr_t location); + +/** + * Update all page tables for a paged-in data page + * + * This function: + * - Maps the specified virtual data page address to the provided physical + * page frame address, such that future memory accesses will function as + * expected. Access and caching attributes are undisturbed. + * - Clears any accounting for "accessed" and "dirty" states. + * + * If multiple page tables are in use, this must update all page tables. + * This function is called with interrupts locked. + * + * Calling this function on data pages which are already paged in is + * undefined behavior. + * + * This API is part of infrastructure still under development and may change. + */ +void arch_mem_page_in(void *addr, uintptr_t phys); + +/** + * Update current page tables for a temporary mapping + * + * Map a physical page frame address to a special virtual address + * Z_SCRATCH_PAGE, with read/write access to supervisor mode, such that + * when this function returns, the calling context can read/write the page + * frame's contents from the Z_SCRATCH_PAGE address. + * + * This mapping only needs to be done on the current set of page tables, + * as it is only used for a short period of time exclusively by the caller. + * This function is called with interrupts locked. + * + * This API is part of infrastructure still under development and may change. + */ +void arch_mem_scratch(uintptr_t phys); + +enum arch_page_location { + ARCH_PAGE_LOCATION_PAGED_OUT, + ARCH_PAGE_LOCATION_PAGED_IN, + ARCH_PAGE_LOCATION_BAD +}; + +/** + * Fetch location information about a page at a particular address + * + * The function only needs to query the current set of page tables as + * the information it reports must be common to all of them if multiple + * page tables are in use. If multiple page tables are active it is unnecessary + * to iterate over all of them. This may allow certain types of optimizations + * (such as reverse page table mapping on x86). + * + * This function is called with interrupts locked, so that the reported + * information can't become stale while decisions are being made based on it. + * + * Unless otherwise specified, virtual data pages have the same mappings + * across all page tables. Calling this function on data pages that are + * exceptions to this rule (such as the scratch page) is undefined behavior. + * Just check the currently installed page tables and return the information + * in that. + * + * @param addr Virtual data page address that took the page fault + * @param [out] location In the case of ARCH_PAGE_FAULT_PAGED_OUT, the backing + * store location value used to retrieve the data page. In the case of + * ARCH_PAGE_FAULT_PAGED_IN, the physical address the page is mapped to. + * @retval ARCH_PAGE_FAULT_PAGED_OUT The page was evicted to the backing store. + * @retval ARCH_PAGE_FAULT_PAGED_IN The data page is resident in memory. + * @retval ARCH_PAGE_FAULT_BAD The page is un-mapped or otherwise has had + * invalid access + */ +enum arch_page_location arch_page_location_get(void *addr, uintptr_t *location); + +/** + * @def ARCH_DATA_PAGE_ACCESSED + * + * Bit indicating the data page was accessed since the value was last cleared. + * + * Used by marking eviction algorithms. Safe to set this if uncertain. + * + * This bit is undefined if ARCH_DATA_PAGE_LOADED is not set. + */ + + /** + * @def ARCH_DATA_PAGE_DIRTY + * + * Bit indicating the data page, if evicted, will need to be paged out. + * + * Set if the data page was modified since it was last paged out, or if + * it has never been paged out before. Safe to set this if uncertain. + * + * This bit is undefined if ARCH_DATA_PAGE_LOADED is not set. + */ + + /** + * @def ARCH_DATA_PAGE_LOADED + * + * Bit indicating that the data page is loaded into a physical page frame. + * + * If un-set, the data page is paged out or not mapped. + */ + +/** + * @def ARCH_DATA_PAGE_NOT_MAPPED + * + * If ARCH_DATA_PAGE_LOADED is un-set, this will indicate that the page + * is not mapped at all. This bit is undefined if ARCH_DATA_PAGE_LOADED is set. + */ + +/** + * Retrieve page characteristics from the page table(s) + * + * The architecture is responsible for maintaining "accessed" and "dirty" + * states of data pages to support marking eviction algorithms. This can + * either be directly supported by hardware or emulated by modifying + * protection policy to generate faults on reads or writes. In all cases + * the architecture must maintain this information in some way. + * + * For the provided virtual address, report the logical OR of the accessed + * and dirty states for the relevant entries in all active page tables in + * the system if the page is mapped and not paged out. + * + * If clear_accessed is true, the ARCH_DATA_PAGE_ACCESSED flag will be reset. + * This function will report its prior state. If multiple page tables are in + * use, this function clears accessed state in all of them. + * + * This function is called with interrupts locked, so that the reported + * information can't become stale while decisions are being made based on it. + * + * The return value may have other bits set which the caller must ignore. + * + * Clearing accessed state for data pages that are not ARCH_DATA_PAGE_LOADED + * is undefined behavior. + * + * ARCH_DATA_PAGE_DIRTY and ARCH_DATA_PAGE_ACCESSED bits in the return value + * are only significant if ARCH_DATA_PAGE_LOADED is set, otherwise ignore + * them. + * + * ARCH_DATA_PAGE_NOT_MAPPED bit in the return value is only significant + * if ARCH_DATA_PAGE_LOADED is un-set, otherwise ignore it. + * + * Unless otherwise specified, virtual data pages have the same mappings + * across all page tables. Calling this function on data pages that are + * exceptions to this rule (such as the scratch page) is undefined behavior. + * + * This API is part of infrastructure still under development and may change. + * + * @param addr Virtual address to look up in page tables + * @param [out] location If non-NULL, updated with either physical page frame + * address or backing store location depending on + * ARCH_DATA_PAGE_LOADED state. This is not touched if + * ARCH_DATA_PAGE_NOT_MAPPED. + * @param clear_accessed Whether to clear ARCH_DATA_PAGE_ACCESSED state + * @retval Value with ARCH_DATA_PAGE_* bits set reflecting the data page + * configuration + */ +uintptr_t arch_page_info_get(void *addr, uintptr_t *location, + bool clear_accessed); +#endif /* CONFIG_DEMAND_PAGING */ #endif /* CONFIG_MMU */ /** @} */ diff --git a/kernel/include/kernel_internal.h b/kernel/include/kernel_internal.h index a05925d7f3cb..702d5c512cd2 100644 --- a/kernel/include/kernel_internal.h +++ b/kernel/include/kernel_internal.h @@ -192,6 +192,14 @@ void z_thread_mark_switched_out(void); #endif /* CONFIG_INSTRUMENT_THREAD_SWITCHING */ +/* Init hook for page frame management, invoked immediately upon entry of + * main thread, before POST_KERNEL tasks + */ +void z_mem_manage_init(void); + +/* Workaround for build-time page table mapping of the kernel */ +void z_kernel_map_fixup(void); + #ifdef __cplusplus } #endif diff --git a/kernel/include/mmu.h b/kernel/include/mmu.h new file mode 100644 index 000000000000..50981e3ecf3d --- /dev/null +++ b/kernel/include/mmu.h @@ -0,0 +1,440 @@ +/* + * Copyright (c) 2020 Intel Corporation. + * + * SPDX-License-Identifier: Apache-2.0 + */ +#ifndef KERNEL_INCLUDE_MMU_H +#define KERNEL_INCLUDE_MMU_H + +#ifdef CONFIG_MMU + +#include +#include +#include +#include +#include +#include + +/* + * At present, page frame management is only done for main system RAM, + * and we generate paging structures based on CONFIG_SRAM_BASE_ADDRESS + * and CONFIG_SRAM_SIZE. + * + * If we have other RAM regions (DCCM, etc) these typically have special + * properties and shouldn't be used generically for demand paging or + * anonymous mappings. We don't currently maintain an ontology of these in the + * core kernel. + */ +#define Z_PHYS_RAM_START ((uintptr_t)CONFIG_SRAM_BASE_ADDRESS) +#define Z_PHYS_RAM_SIZE ((size_t)KB(CONFIG_SRAM_SIZE)) +#define Z_PHYS_RAM_END (Z_PHYS_RAM_START + Z_PHYS_RAM_SIZE) +#define Z_NUM_PAGE_FRAMES (Z_PHYS_RAM_SIZE / CONFIG_MMU_PAGE_SIZE) + +/** End virtual address of virtual address space */ +#define Z_VIRT_RAM_START ((uint8_t *)CONFIG_KERNEL_VM_BASE) +#define Z_VIRT_RAM_SIZE ((size_t)CONFIG_KERNEL_VM_SIZE) +#define Z_VIRT_RAM_END (Z_VIRT_RAM_START + Z_VIRT_RAM_SIZE) + +/* Boot-time virtual location of the kernel image. */ +#define Z_KERNEL_VIRT_START ((uint8_t *)(&z_mapped_start)) +#define Z_KERNEL_VIRT_END ((uint8_t *)(&z_mapped_end)) +#define Z_KERNEL_VIRT_SIZE ((size_t)(&z_mapped_size)) + +#define Z_VM_OFFSET ((CONFIG_KERNEL_VM_BASE + CONFIG_KERNEL_VM_OFFSET) - \ + CONFIG_SRAM_BASE_ADDRESS) + +/* Only applies to boot RAM mappings within the Zephyr image that have never + * been remapped or paged out. Never use this unless you know exactly what you + * are doing. + */ +#define Z_BOOT_VIRT_TO_PHYS(virt) ((uintptr_t)(((uint8_t *)virt) + Z_VM_OFFSET)) +#define Z_BOOT_PHYS_TO_VIRT(phys) ((uint8_t *)(((uintptr_t)phys) - Z_VM_OFFSET)) + +#ifdef CONFIG_ARCH_MAPS_ALL_RAM +#define Z_FREE_VM_START Z_BOOT_PHYS_TO_VIRT(Z_PHYS_RAM_END) +#else +#define Z_FREE_VM_START Z_KERNEL_VIRT_END +#endif + +/* + * Macros and data structures for physical page frame accounting, + * APIs for use by eviction and backing store algorithms. This code + * is otherwise not application-facing. + */ + +/* + * z_page_frame flags bits + */ + +/** This page contains critical kernel data and will never be swapped */ +#define Z_PAGE_FRAME_PINNED BIT(0) + +/** This physical page is reserved by hardware; we will never use it */ +#define Z_PAGE_FRAME_RESERVED BIT(1) + +/** + * This physical page is mapped to some virtual memory address + * + * Currently, we just support one mapping per page frame. If a page frame + * is mapped to multiple virtual pages then it must be pinned. + */ +#define Z_PAGE_FRAME_MAPPED BIT(2) + +/** + * This page frame is currently involved in a page-in/out operation + */ +#define Z_PAGE_FRAME_BUSY BIT(3) + +/** + * This page frame has a clean copy in the backing store + */ +#define Z_PAGE_FRAME_BACKED BIT(4) + +/** + * Data structure for physical page frames + * + * An array of these is instantiated, one element per physical RAM page. + * Hence it's necessary to constrain its size as much as possible. + */ +struct z_page_frame { + union { + /* If mapped, virtual address this page is mapped to */ + void *addr; + + /* If unmapped and available, free pages list membership. */ + sys_snode_t node; + }; + + /* Z_PAGE_FRAME_* flags */ + uint8_t flags; + + /* TODO: Backing store and eviction algorithms may both need to + * introduce custom members for accounting purposes. Come up with + * a layer of abstraction for this. They may also want additional + * flags bits which shouldn't clobber each other. At all costs + * the total size of struct z_page_frame must be minimized. + */ +} __packed; + +static inline bool z_page_frame_is_pinned(struct z_page_frame *pf) +{ + return (pf->flags & Z_PAGE_FRAME_PINNED) != 0; +} + +static inline bool z_page_frame_is_reserved(struct z_page_frame *pf) +{ + return (pf->flags & Z_PAGE_FRAME_RESERVED) != 0; +} + +static inline bool z_page_frame_is_mapped(struct z_page_frame *pf) +{ + return (pf->flags & Z_PAGE_FRAME_MAPPED) != 0; +} + +static inline bool z_page_frame_is_busy(struct z_page_frame *pf) +{ + return (pf->flags & Z_PAGE_FRAME_BUSY) != 0; +} + +static inline bool z_page_frame_is_backed(struct z_page_frame *pf) +{ + return (pf->flags & Z_PAGE_FRAME_BACKED) != 0; +} + +static inline bool z_page_frame_is_evictable(struct z_page_frame *pf) +{ + return (!z_page_frame_is_reserved(pf) && z_page_frame_is_mapped(pf) && + !z_page_frame_is_pinned(pf) && !z_page_frame_is_busy(pf)); +} + +/* If true, page is not being used for anything, is not reserved, is a member + * of some free pages list, isn't busy, and may be mapped in memory + */ +static inline bool z_page_frame_is_available(struct z_page_frame *page) +{ + return page->flags == 0; +} + +static inline void z_assert_phys_aligned(uintptr_t phys) +{ + __ASSERT(phys % CONFIG_MMU_PAGE_SIZE == 0, + "physical address 0x%lx is not page-aligned", phys); + (void)phys; +} + +extern struct z_page_frame z_page_frames[Z_NUM_PAGE_FRAMES]; + +static inline uintptr_t z_page_frame_to_phys(struct z_page_frame *pf) +{ + return (uintptr_t)((pf - z_page_frames) * CONFIG_MMU_PAGE_SIZE) + + Z_PHYS_RAM_START; +} + +/* Presumes there is but one mapping in the virtual address space */ +static inline void *z_page_frame_to_virt(struct z_page_frame *pf) +{ + return pf->addr; +} + +static inline bool z_is_page_frame(uintptr_t phys) +{ + z_assert_phys_aligned(phys); + return (phys >= Z_PHYS_RAM_START) && (phys < Z_PHYS_RAM_END); +} + +static inline struct z_page_frame *z_phys_to_page_frame(uintptr_t phys) +{ + __ASSERT(z_is_page_frame(phys), + "0x%lx not an SRAM physical address", phys); + + return &z_page_frames[(phys - Z_PHYS_RAM_START) / + CONFIG_MMU_PAGE_SIZE]; +} + +static inline void z_mem_assert_virtual_region(uint8_t *addr, size_t size) +{ + __ASSERT((uintptr_t)addr % CONFIG_MMU_PAGE_SIZE == 0, + "unaligned addr %p", addr); + __ASSERT(size % CONFIG_MMU_PAGE_SIZE == 0, + "unaligned size %zu", size); + __ASSERT(addr + size > addr, + "region %p size %zu zero or wraps around", addr, size); + __ASSERT(addr >= Z_VIRT_RAM_START && addr + size < Z_VIRT_RAM_END, + "invalid virtual address region %p (%zu)", addr, size); +} + +/* Debug function, pretty-print page frame information for all frames + * concisely to printk. + */ +void z_page_frames_dump(void); + +/* Number of free page frames. This information may go stale immediately */ +extern size_t z_free_page_count; + +/* Convenience macro for iterating over all page frames */ +#define Z_PAGE_FRAME_FOREACH(_phys, _pageframe) \ + for (_phys = Z_PHYS_RAM_START, _pageframe = z_page_frames; \ + _phys < Z_PHYS_RAM_END; \ + _phys += CONFIG_MMU_PAGE_SIZE, _pageframe++) + +#ifdef CONFIG_DEMAND_PAGING +/* We reserve a virtual page as a scratch area for page-ins/outs at the end + * of the address space + */ +#define Z_VM_RESERVED CONFIG_MMU_PAGE_SIZE +#define Z_SCRATCH_PAGE ((void *)((uintptr_t)CONFIG_KERNEL_VM_BASE + \ + (uintptr_t)CONFIG_KERNEL_VM_SIZE - \ + CONFIG_MMU_PAGE_SIZE)) +#else +#define Z_VM_RESERVED 0 +#endif + +#ifdef CONFIG_DEMAND_PAGING +/* + * Eviction algorihm APIs + */ + +/** + * Select a page frame for eviction + * + * The kernel will invoke this to choose a page frame to evict if there + * are no free page frames. + * + * This function will never be called before the initial z_eviction_init(). + * + * This function is invoked with interrupts locked. + * + * @param [out] Whether the page to evict is dirty + * @return The page frame to evict + */ +struct z_page_frame *z_eviction_select(bool *dirty); + +/** + * Initialization function + * + * Called at POST_KERNEL to perform any necessary initialization tasks for the + * eviction algorithm. z_eviction_select() is guaranteed to never be called + * until this has returned, and this will only be called once. + */ +void z_eviction_init(void); + +/* + * Backing store APIs + */ + +/** + * Reserve or fetch a storage location for a data page loaded into a page frame + * + * The returned location token must be unique to the mapped virtual address. + * This location will be used in the backing store to page out data page + * contents for later retrieval. The location value must be page-aligned. + * + * This function may be called multiple times on the same data page. If its + * page frame has its Z_PAGE_FRAME_BACKED bit set, it is expected to return + * the previous backing store location for the data page containing a cached + * clean copy. This clean copy may be updated on page-out, or used to + * discard clean pages without needing to write out their contents. + * + * If the backing store is full, some other backing store location which caches + * a loaded data page may be selected, in which case its associated page frame + * will have the Z_PAGE_FRAME_BACKED bit cleared (as it is no longer cached). + * + * pf->addr will indicate the virtual address the page is currently mapped to. + * Large, sparse backing stores which can contain the entire address space + * may simply generate location tokens purely as a function of pf->addr with no + * other management necessary. + * + * This function distinguishes whether it was called on behalf of a page + * fault. A free backing store location must always be reserved in order for + * page faults to succeed. If the page_fault parameter is not set, this + * function should return -ENOMEM even if one location is available. + * + * This function is invoked with interrupts locked. + * + * @param addr Virtual address to obtain a storage location + * @param [out] location storage location token + * @param page_fault Whether this request was for a page fault + * @return 0 Success + * @return -ENOMEM Backing store is full + */ +int z_backing_store_location_get(struct z_page_frame *pf, uintptr_t *location, + bool page_fault); + +/** + * Free a backing store location + * + * Any stored data may be discarded, and the location token associated with + * this address may be re-used for some other data page. + * + * This function is invoked with interrupts locked. + * + * @param location Location token to free + */ +void z_backing_store_location_free(uintptr_t location); + +/** + * Copy a data page from Z_SCRATCH_PAGE to the specified location + * + * Immediately before this is called, Z_SCRATCH_PAGE will be mapped read-write + * to the intended source page frame for the calling context. + * + * Calls to this and z_backing_store_page_in() will always be serialized, + * but interrupts may be enabled. + * + * @param location Location token for the data page, for later retrieval + */ +void z_backing_store_page_out(uintptr_t location); + +/** + * Copy a data page from the provided location to Z_SCRATCH_PAGE. + * + * Immediately before this is called, Z_SCRATCH_PAGE will be mapped read-write + * to the intended destination page frame for the calling context. + * + * Calls to this and z_backing_store_page_out() will always be serialized, + * but interrupts may be enabled. + * + * @param location Location token for the data page + */ +void z_backing_store_page_in(uintptr_t location); + +/** + * Update internal accounting after a page-in + * + * This is invoked after z_backing_store_page_in() and interrupts have been + * re-locked, making it safe to access the z_page_frame data. The location + * value will be the same passed to z_backing_store_page_in(). + * + * The primary use-case for this is to update custom fields for the backing + * store in the page frame, to reflect where the data should be evicted to + * if it is paged out again. This may be a no-op in some implementations. + * + * If the backing store caches paged-in data pages, this is the appropriate + * time to set the Z_PAGE_FRAME_BACKED bit. The kernel only skips paging + * out clean data pages if they are noted as clean in the page tables and the + * Z_PAGE_FRAME_BACKED bit is set in their associated page frame. + * + * @param pf Page frame that was loaded in + * @param location Location of where the loaded data page was retrieved + */ +void z_backing_store_page_finalize(struct z_page_frame *pf, uintptr_t location); + +/** + * Backing store initialization function. + * + * The implementation may expect to receive page in/out calls as soon as this + * returns, but not before that. Called at POST_KERNEL. + * + * This function is expected to do two things: + * - Initialize any internal data structures and accounting for the backing + * store. + * - If the backing store already contains all or some loaded kernel data pages + * at boot time, Z_PAGE_FRAME_BACKED should be appropriately set for their + * associated page frames, and any internal accounting set up appropriately. + */ +void z_backing_store_init(void); + +/* + * Core kernel demand paging APIs + */ + +/** + * Number of page faults since system startup + * + * Counts only those page faults that were handled successfully by the demand + * paging mechanism and were not errors. + * + * @return Number of successful page faults + */ +unsigned long z_num_pagefaults_get(void); + +/** + * Free a page frame physical address by evicting its contents + * + * The indicated page frame, if it contains a data page, will have that + * data page evicted to the backing store. The page frame will then be + * marked as available for mappings or page-ins. + * + * This is useful for freeing up entire memory banks so that they may be + * deactivated to save power. + * + * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be + * called by ISRs as the backing store may be in-use. + * + * @param phys Page frame physical address + * @retval 0 Success + * @retval -ENOMEM Insufficient backing store space + */ +int z_page_frame_evict(uintptr_t phys); + +/** + * Handle a page fault for a virtual data page + * + * This is invoked from the architecture page fault handler. + * + * If a valid page fault, the core kernel will obtain a page frame, + * populate it with the data page that was evicted to the backing store, + * update page tables, and return so that the faulting instruction may be + * re-tried. + * + * The architecture must not call this function if the page was mapped and + * not paged out at the time the exception was triggered (i.e. a protection + * violation for a mapped page). + * + * If the faulting context had interrupts disabled when the page fault was + * triggered, the entire page fault handling path must have interrupts + * disabled, including the invocation of this function. + * + * Otherwise, interrupts may be enabled and the page fault handler may be + * preemptible. Races to page-in will be appropriately handled by the kernel. + * + * @param addr Faulting virtual address + * @retval true Page fault successfully handled, or nothing needed to be done. + * The arch layer should retry the faulting instruction. + * @retval false This page fault was from an un-mapped page, should + * be treated as an error, and not re-tried. + */ +bool z_page_fault(void *addr); +#endif /* CONFIG_DEMAND_PAGING */ +#endif /* CONFIG_MMU */ +#endif /* KERNEL_INCLUDE_MMU_H */ diff --git a/kernel/init.c b/kernel/init.c index 9e2956a6b1de..b12db355d86b 100644 --- a/kernel/init.c +++ b/kernel/init.c @@ -136,6 +136,14 @@ static void bg_thread_main(void *unused1, void *unused2, void *unused3) ARG_UNUSED(unused2); ARG_UNUSED(unused3); +#ifdef CONFIG_MMU + /* Invoked here such that backing store or eviction algorithms may + * initialize kernel objects, and that all POST_KERNEL and later tasks + * may perform memory management tasks (except for z_phys_map() which + * is allowed at any time) + */ + z_mem_manage_init(); +#endif /* CONFIG_MMU */ z_sys_post_kernel = true; z_sys_init_run_level(_SYS_INIT_LEVEL_POST_KERNEL); @@ -383,7 +391,9 @@ FUNC_NORETURN void z_cstart(void) z_dummy_thread_init(&dummy_thread); #endif - +#if defined(CONFIG_MMU) && defined(CONFIG_USERSPACE) + z_kernel_map_fixup(); +#endif /* perform basic hardware initialization */ z_sys_init_run_level(_SYS_INIT_LEVEL_PRE_KERNEL_1); z_sys_init_run_level(_SYS_INIT_LEVEL_PRE_KERNEL_2); diff --git a/kernel/mmu.c b/kernel/mmu.c index b6180614f20b..88c40025873d 100644 --- a/kernel/mmu.c +++ b/kernel/mmu.c @@ -9,28 +9,151 @@ #include #include #include +#include +#include +#include +#include #include LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); +/* + * General terminology: + * - A page frame is a page-sized physical memory region in RAM. It is a + * container where a data page may be placed. It is always referred to by + * physical address. We have a convention of using uintptr_t for physical + * addresses. We instantiate a struct z_page_frame to store metadata for + * every page frame. + * + * - A data page is a page-sized region of data. It may exist in a page frame, + * or be paged out to some backing store. Its location can always be looked + * up in the CPU's page tables (or equivalent) by virtual address. + * The data type will always be void * or in some cases uint8_t * when we + * want to do pointer arithmetic. + */ + /* Spinlock to protect any globals in this file and serialize page table * updates in arch code */ -static struct k_spinlock mm_lock; +struct k_spinlock z_mm_lock; + +/* + * General page frame management + */ + +/* Database of all RAM page frames */ +struct z_page_frame z_page_frames[Z_NUM_PAGE_FRAMES]; + +#if __ASSERT_ON +/* Indicator that z_page_frames has been initialized, many of these APIs do + * not work before POST_KERNEL + */ +static bool page_frames_initialized; +#endif + +/* Add colors to page table dumps to indicate mapping type */ +#define COLOR_PAGE_FRAMES 1 + +#if COLOR_PAGE_FRAMES +#define ANSI_DEFAULT "\x1B[0m" +#define ANSI_RED "\x1B[1;31m" +#define ANSI_GREEN "\x1B[1;32m" +#define ANSI_YELLOW "\x1B[1;33m" +#define ANSI_BLUE "\x1B[1;34m" +#define ANSI_MAGENTA "\x1B[1;35m" +#define ANSI_CYAN "\x1B[1;36m" +#define ANSI_GREY "\x1B[1;90m" + +#define COLOR(x) printk(_CONCAT(ANSI_, x)) +#else +#define COLOR(x) do { } while (0) +#endif + +static void page_frame_dump(struct z_page_frame *pf) +{ + if (z_page_frame_is_reserved(pf)) { + COLOR(CYAN); + printk("R"); + } else if (z_page_frame_is_busy(pf)) { + COLOR(MAGENTA); + printk("B"); + } else if (z_page_frame_is_pinned(pf)) { + COLOR(YELLOW); + printk("P"); + } else if (z_page_frame_is_available(pf)) { + COLOR(GREY); + printk("."); + } else if (z_page_frame_is_mapped(pf)) { + COLOR(DEFAULT); + printk("M"); + } else { + COLOR(RED); + printk("?"); + } +} + +void z_page_frames_dump(void) +{ + int column = 0; + + __ASSERT(page_frames_initialized, "%s called too early", __func__); + printk("Physical memory from 0x%lx to 0x%lx\n", + Z_PHYS_RAM_START, Z_PHYS_RAM_END); + + for (int i = 0; i < Z_NUM_PAGE_FRAMES; i++) { + struct z_page_frame *pf = &z_page_frames[i]; + + page_frame_dump(pf); + + column++; + if (column == 64) { + column = 0; + printk("\n"); + } + } + + COLOR(DEFAULT); + if (column != 0) { + printk("\n"); + } +} + +#define VIRT_FOREACH(_base, _size, _pos) \ + for (_pos = _base; \ + _pos < ((uint8_t *)_base + _size); _pos += CONFIG_MMU_PAGE_SIZE) + +#define PHYS_FOREACH(_base, _size, _pos) \ + for (_pos = _base; \ + _pos < ((uintptr_t)_base + _size); _pos += CONFIG_MMU_PAGE_SIZE) + /* - * Overall virtual memory map. When the kernel starts, it is expected that all - * memory regions are mapped into one large virtual region at the beginning of - * CONFIG_KERNEL_VM_BASE. Unused virtual memory up to the limit noted by - * CONFIG_KERNEL_VM_SIZE may be used for runtime memory mappings. + * Virtual address space management + * + * Call all of these functions with z_mm_lock held. + * + * Overall virtual memory map: When the kernel starts, it resides in + * virtual memory in the region Z_KERNEL_VIRT_START to + * Z_KERNEL_VIRT_END. Unused virtual memory past this, up to the limit + * noted by CONFIG_KERNEL_VM_SIZE may be used for runtime memory mappings. + * + * If CONFIG_ARCH_MAPS_ALL_RAM is set, we do not just map the kernel image, + * but have a mapping for all RAM in place. This is for special architectural + * purposes and does not otherwise affect page frame accounting or flags; + * the only guarantee is that such RAM mapping outside of the Zephyr image + * won't be disturbed by subsequent memory mapping calls. * - * +--------------+ <- CONFIG_KERNEL_VM_BASE + * +--------------+ <- Z_VIRT_ADDR_START + * | Undefined VM | <- May contain ancillary regions like x86_64's locore + * +--------------+ <- Z_KERNEL_VIRT_START (often == Z_VIRT_ADDR_START) * | Mapping for | - * | all RAM | + * | main kernel | + * | image | + * | | + * | | + * +--------------+ <- Z_FREE_VM_START * | | - * | | - * +--------------+ <- CONFIG_KERNEL_VM_BASE + CONFIG_KERNEL_RAM_SIZE - * | Available | also the mapping limit as mappings grown downward - * | virtual mem | + * | Unused, | + * | Available VM | * | | * |..............| <- mapping_pos (grows downward as more mappings are made) * | Mapping | @@ -40,98 +163,836 @@ static struct k_spinlock mm_lock; * | ... | * +--------------+ * | Mapping | - * +--------------+ <- CONFIG_KERNEL_VM_BASE + CONFIG_KERNEL_VM_SIZE + * +--------------+ <- mappings start here + * | Reserved | <- special purpose virtual page(s) of size Z_VM_RESERVED + * +--------------+ <- Z_VIRT_RAM_END * - * At the moment we just have one area for mappings and they are permanent. - * This is under heavy development and may change. + * At the moment we just have one downward-growing area for mappings. + * There is currently no support for un-mapping memory, see #28900. */ +static uint8_t *mapping_pos = Z_VIRT_RAM_END - Z_VM_RESERVED; + +/* Get a chunk of virtual memory and mark it as being in-use. + * + * This may be called from arch early boot code before z_cstart() is invoked. + * Data will be copied and BSS zeroed, but this must not rely on any + * initialization functions being called prior to work correctly. + */ +static void *virt_region_get(size_t size) +{ + uint8_t *dest_addr; - /* Current position for memory mappings in kernel memory. - * At the moment, all kernel memory mappings are permanent. - * Memory mappings start at the end of the address space, and grow - * downward. - * - * All of this is under heavy development and is subject to change. - */ -static uint8_t *mapping_pos = - (uint8_t *)((uintptr_t)CONFIG_KERNEL_VM_BASE + - (uintptr_t)CONFIG_KERNEL_VM_SIZE); + if ((mapping_pos - size) < Z_FREE_VM_START) { + LOG_ERR("insufficient virtual address space (requested %zu)", + size); + return NULL; + } + + mapping_pos -= size; + dest_addr = mapping_pos; + + return dest_addr; +} -/* Lower-limit of virtual address mapping. Immediately below this is the - * permanent identity mapping for all SRAM. +/* + * Free page frames management + * + * Call all of these functions with z_mm_lock held. */ -static uint8_t *mapping_limit = - (uint8_t *)((uintptr_t)CONFIG_KERNEL_VM_BASE + - (size_t)CONFIG_KERNEL_RAM_SIZE); -size_t k_mem_region_align(uintptr_t *aligned_addr, size_t *aligned_size, - uintptr_t phys_addr, size_t size, size_t align) +/* Linked list of unused and available page frames. + * + * TODO: This is very simple and treats all free page frames as being equal. + * However, there are use-cases to consolidate free pages such that entire + * SRAM banks can be switched off to save power, and so obtaining free pages + * may require a more complex ontology which prefers page frames in RAM banks + * which are still active. + * + * This implies in the future there may be multiple slists managing physical + * pages. Each page frame will still just have one snode link. + */ +static sys_slist_t free_page_frame_list; + +/* Number of unused and available free page frames */ +size_t z_free_page_count; + +#define PF_ASSERT(pf, expr, fmt, ...) \ + __ASSERT(expr, "page frame 0x%lx: " fmt, z_page_frame_to_phys(pf), \ + ##__VA_ARGS__) + +/* Get an unused page frame. don't care which one, or NULL if there are none */ +static struct z_page_frame *free_page_frame_list_get(void) { - size_t addr_offset; + sys_snode_t *node; + struct z_page_frame *pf = NULL; - /* The actual mapped region must be page-aligned. Round down the - * physical address and pad the region size appropriately + node = sys_slist_get(&free_page_frame_list); + if (node != NULL) { + z_free_page_count--; + pf = CONTAINER_OF(node, struct z_page_frame, node); + PF_ASSERT(pf, z_page_frame_is_available(pf), + "unavailable but somehow on free list"); + } + + return pf; +} + +/* Release a page frame back into the list of free pages */ +static void free_page_frame_list_put(struct z_page_frame *pf) +{ + PF_ASSERT(pf, z_page_frame_is_available(pf), + "unavailable page put on free list"); + sys_slist_append(&free_page_frame_list, &pf->node); + z_free_page_count++; +} + +static void free_page_frame_list_init(void) +{ + sys_slist_init(&free_page_frame_list); +} + +/* + * Memory Mapping + */ + +/* Called after the frame is mapped in the arch layer, to update our + * local ontology (and do some assertions while we're at it) + */ +static void frame_mapped_set(struct z_page_frame *pf, void *addr) +{ + PF_ASSERT(pf, !z_page_frame_is_reserved(pf), + "attempted to map a reserved page frame"); + + /* We do allow multiple mappings for pinned page frames + * since we will never need to reverse map them. + * This is uncommon, use-cases are for things like the + * Zephyr equivalent of VSDOs */ - *aligned_addr = ROUND_DOWN(phys_addr, align); - addr_offset = phys_addr - *aligned_addr; - *aligned_size = ROUND_UP(size + addr_offset, align); + PF_ASSERT(pf, !z_page_frame_is_mapped(pf) || z_page_frame_is_pinned(pf), + "non-pinned and already mapped to %p", pf->addr); - return addr_offset; + pf->flags |= Z_PAGE_FRAME_MAPPED; + pf->addr = addr; +} + +#ifdef CONFIG_DEMAND_PAGING +static int page_frame_prepare_locked(struct z_page_frame *pf, bool *dirty_ptr, + bool page_in, uintptr_t *location_ptr); +#endif /* CONFIG_DEMAND_PAGING */ + +/* Allocate a free page frame, and map it to a specified virtual address + * + * TODO: Add optional support for copy-on-write mappings to a zero page instead + * of allocating, in which case page frames will be allocated lazily as + * the mappings to the zero page get touched. This will avoid expensive + * page-ins as memory is mapped and physical RAM or backing store storage will + * not be used if the mapped memory is unused. The cost is an empty physical + * page of zeroes. + */ +static int map_anon_page(void *addr, uint32_t flags) +{ + struct z_page_frame *pf; + uintptr_t phys; + bool lock = (flags & K_MEM_MAP_LOCK) != 0; + bool uninit = (flags & K_MEM_MAP_UNINIT) != 0; + + pf = free_page_frame_list_get(); + if (pf == NULL) { +#ifdef CONFIG_DEMAND_PAGING + uintptr_t location; + bool dirty; + int ret; + + pf = z_eviction_select(&dirty); + __ASSERT(pf != NULL, "failed to get a page frame"); + LOG_DBG("evicting %p at 0x%lx", pf->addr, + z_page_frame_to_phys(pf)); + ret = page_frame_prepare_locked(pf, &dirty, false, &location); + if (ret != 0) { + return -ENOMEM; + } + if (dirty) { + z_backing_store_page_out(location); + } + pf->flags = 0; +#else + return -ENOMEM; +#endif /* CONFIG_DEMAND_PAGING */ + } + + phys = z_page_frame_to_phys(pf); + arch_mem_map(addr, phys, CONFIG_MMU_PAGE_SIZE, flags | K_MEM_CACHE_WB); + + if (lock) { + pf->flags |= Z_PAGE_FRAME_PINNED; + } + frame_mapped_set(pf, addr); + + LOG_DBG("memory mapping anon page %p -> 0x%lx", addr, phys); + + if (!uninit) { + /* If we later implement mappings to a copy-on-write + * zero page, won't need this step + */ + memset(addr, 0, CONFIG_MMU_PAGE_SIZE); + } + + return 0; +} + +void *k_mem_map(size_t size, uint32_t flags) +{; + uint8_t *dst; + size_t total_size = size; + int ret; + k_spinlock_key_t key; + bool guard = (flags & K_MEM_MAP_GUARD) != 0; + uint8_t *pos; + + __ASSERT(!(((flags & K_MEM_PERM_USER) != 0) && + ((flags & K_MEM_MAP_UNINIT) != 0)), + "user access to anonymous uninitialized pages is forbidden"); + __ASSERT(size % CONFIG_MMU_PAGE_SIZE == 0, + "unaligned size %zu passed to %s", size, __func__); + __ASSERT(size != 0, "zero sized memory mapping"); + __ASSERT(page_frames_initialized, "%s called too early", __func__); + __ASSERT((flags & K_MEM_CACHE_MASK) == 0, + "%s does not support explicit cache settings", __func__); + + key = k_spin_lock(&z_mm_lock); + + if (guard) { + /* Need extra virtual page for the guard which we + * won't map + */ + total_size += CONFIG_MMU_PAGE_SIZE; + } + + dst = virt_region_get(total_size); + if (dst == NULL) { + /* Address space has no free region */ + goto out; + } + if (guard) { + /* Skip over the guard page in returned address. */ + dst += CONFIG_MMU_PAGE_SIZE; + } + + VIRT_FOREACH(dst, size, pos) { + ret = map_anon_page(pos, flags); + + if (ret != 0) { + /* TODO: call k_mem_unmap(dst, pos - dst) when + * implmented in #28990 and release any guard virtual + * page as well. + */ + dst = NULL; + goto out; + } + } +out: + k_spin_unlock(&z_mm_lock, key); + return dst; } +size_t k_mem_free_get(void) +{ + size_t ret; + k_spinlock_key_t key; + + __ASSERT(page_frames_initialized, "%s called too early", __func__); + + key = k_spin_lock(&z_mm_lock); + ret = z_free_page_count; + k_spin_unlock(&z_mm_lock, key); + + return ret * CONFIG_MMU_PAGE_SIZE; +} + +/* This may be called from arch early boot code before z_cstart() is invoked. + * Data will be copied and BSS zeroed, but this must not rely on any + * initialization functions being called prior to work correctly. + */ void z_phys_map(uint8_t **virt_ptr, uintptr_t phys, size_t size, uint32_t flags) { - uintptr_t aligned_addr, addr_offset; + uintptr_t aligned_phys, addr_offset; size_t aligned_size; - int ret; k_spinlock_key_t key; - uint8_t *dest_virt; + uint8_t *dest_addr; - addr_offset = k_mem_region_align(&aligned_addr, &aligned_size, + addr_offset = k_mem_region_align(&aligned_phys, &aligned_size, phys, size, CONFIG_MMU_PAGE_SIZE); + __ASSERT(aligned_size != 0, "0-length mapping at 0x%lx", aligned_phys); + __ASSERT(aligned_phys < (aligned_phys + (aligned_size - 1)), + "wraparound for physical address 0x%lx (size %zu)", + aligned_phys, aligned_size); - key = k_spin_lock(&mm_lock); - - /* Carve out some unused virtual memory from the top of the - * address space - */ - if ((mapping_pos - aligned_size) < mapping_limit) { - LOG_ERR("insufficient kernel virtual address space"); + key = k_spin_lock(&z_mm_lock); + /* Obtain an appropriately sized chunk of virtual memory */ + dest_addr = virt_region_get(aligned_size); + if (!dest_addr) { goto fail; } - mapping_pos -= aligned_size; - dest_virt = mapping_pos; - LOG_DBG("arch_mem_map(%p, 0x%lx, %zu, %x) offset %lu\n", dest_virt, - aligned_addr, aligned_size, flags, addr_offset); - __ASSERT(dest_virt != NULL, "NULL page memory mapping"); - __ASSERT(aligned_size != 0, "0-length mapping at 0x%lx", aligned_addr); - __ASSERT((uintptr_t)dest_virt < - ((uintptr_t)dest_virt + (aligned_size - 1)), + /* If this fails there's something amiss with virt_region_get */ + __ASSERT((uintptr_t)dest_addr < + ((uintptr_t)dest_addr + (size - 1)), "wraparound for virtual address %p (size %zu)", - dest_virt, size); - __ASSERT(aligned_addr < (aligned_addr + (size - 1)), - "wraparound for physical address 0x%lx (size %zu)", - aligned_addr, size); + dest_addr, size); - ret = arch_mem_map(dest_virt, aligned_addr, aligned_size, flags); - k_spin_unlock(&mm_lock, key); + LOG_DBG("arch_mem_map(%p, 0x%lx, %zu, %x) offset %lu", dest_addr, + aligned_phys, aligned_size, flags, addr_offset); - if (ret == 0) { - *virt_ptr = dest_virt + addr_offset; - } else { - /* This happens if there is an insurmountable problem - * with the selected cache modes or access flags - * with no safe fallback - */ + arch_mem_map(dest_addr, aligned_phys, aligned_size, flags); + k_spin_unlock(&z_mm_lock, key); - LOG_ERR("arch_mem_map() to %p returned %d", dest_virt, ret); - goto fail; - } + *virt_ptr = dest_addr + addr_offset; return; fail: + /* May re-visit this in the future, but for now running out of + * virtual address space or failing the arch_mem_map() call is + * an unrecoverable situation. + * + * Other problems not related to resource exhaustion we leave as + * assertions since they are clearly programming mistakes. + */ LOG_ERR("memory mapping 0x%lx (size %zu, flags 0x%x) failed", phys, size, flags); k_panic(); } + +/* + * Miscellaneous + */ + +size_t k_mem_region_align(uintptr_t *aligned_phys, size_t *aligned_size, + uintptr_t phys_addr, size_t size, size_t align) +{ + size_t addr_offset; + + /* The actual mapped region must be page-aligned. Round down the + * physical address and pad the region size appropriately + */ + *aligned_phys = ROUND_DOWN(phys_addr, align); + addr_offset = phys_addr - *aligned_phys; + *aligned_size = ROUND_UP(size + addr_offset, align); + + return addr_offset; +} + + +#ifdef CONFIG_USERSPACE +void z_kernel_map_fixup(void) +{ + /* XXX: Gperf kernel object data created at build time will not have + * visibility in zephyr_prebuilt.elf. There is a possibility that this + * data would not be memory-mapped if it shifts z_mapped_end between + * builds. Ensure this area is mapped. + * + * A third build phase for page tables would solve this. + */ + uint8_t *kobject_page_begin = + (uint8_t *)ROUND_DOWN((uintptr_t)&z_kobject_data_begin, + CONFIG_MMU_PAGE_SIZE); + size_t kobject_size = (size_t)(Z_KERNEL_VIRT_END - kobject_page_begin); + + if (kobject_size != 0) { + arch_mem_map(kobject_page_begin, + Z_BOOT_VIRT_TO_PHYS(kobject_page_begin), + kobject_size, K_MEM_PERM_RW | K_MEM_CACHE_WB); + } +} +#endif /* CONFIG_USERSPACE */ + +void z_mem_manage_init(void) +{ + uintptr_t phys; + uint8_t *addr; + struct z_page_frame *pf; + k_spinlock_key_t key = k_spin_lock(&z_mm_lock); + + free_page_frame_list_init(); + +#ifdef CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES + /* If some page frames are unavailable for use as memory, arch + * code will mark Z_PAGE_FRAME_RESERVED in their flags + */ + arch_reserved_pages_update(); +#endif /* CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES */ + + /* All pages composing the Zephyr image are mapped at boot in a + * predictable way. This can change at runtime. + */ + VIRT_FOREACH(Z_KERNEL_VIRT_START, Z_KERNEL_VIRT_SIZE, addr) + { + pf = z_phys_to_page_frame(Z_BOOT_VIRT_TO_PHYS(addr)); + frame_mapped_set(pf, addr); + + /* TODO: for now we pin the whole Zephyr image. Demand paging + * currently tested with anonymously-mapped pages which are not + * pinned. + * + * We will need to setup linker regions for a subset of kernel + * code/data pages which are pinned in memory and + * may not be evicted. This will contain critical CPU data + * structures, and any code used to perform page fault + * handling, page-ins, etc. + */ + pf->flags |= Z_PAGE_FRAME_PINNED; + } + + /* Any remaining pages that aren't mapped, reserved, or pinned get + * added to the free pages list + */ + Z_PAGE_FRAME_FOREACH(phys, pf) { + if (z_page_frame_is_available(pf)) { + free_page_frame_list_put(pf); + } + } + LOG_DBG("free page frames: %zu", z_free_page_count); + +#ifdef CONFIG_DEMAND_PAGING + z_backing_store_init(); + z_eviction_init(); +#endif +#if __ASSERT_ON + page_frames_initialized = true; +#endif + k_spin_unlock(&z_mm_lock, key); +} + +#ifdef CONFIG_DEMAND_PAGING +static unsigned long z_num_pagefaults; + +/* Current implementation relies on interrupt locking to any prevent page table + * access, which falls over if other CPUs are active. Addressing this is not + * as simple as using spinlocks as regular memory reads/writes constitute + * "access" in this sense. + * + * Current needs for demand paging are on uniprocessor systems. + */ +BUILD_ASSERT(!IS_ENABLED(CONFIG_SMP)); + +static void virt_region_foreach(void *addr, size_t size, + void (*func)(void *)) +{ + z_mem_assert_virtual_region(addr, size); + + for (size_t offset = 0; offset < size; offset += CONFIG_MMU_PAGE_SIZE) { + func((uint8_t *)addr + offset); + } +} + +static void page_frame_free_locked(struct z_page_frame *pf) +{ + pf->flags = 0; + free_page_frame_list_put(pf); +} + +/* + * Perform some preparatory steps before paging out. The provided page frame + * must be evicted to the backing store immediately after this is called + * with a call to z_backing_store_page_out() if it contains a data page. + * + * - Map page frame to scratch area if requested. This always is true if we're + * doing a page fault, but is only set on manual evictions if the page is + * dirty. + * - If mapped: + * - obtain backing store location and populate location parameter + * - Update page tables with location + * - Mark page frame as busy + * + * Returns -ENOMEM if the backing store is full + */ +static int page_frame_prepare_locked(struct z_page_frame *pf, bool *dirty_ptr, + bool page_fault, uintptr_t *location_ptr) +{ + uintptr_t phys; + int ret; + bool dirty = *dirty_ptr; + + phys = z_page_frame_to_phys(pf); + __ASSERT(!z_page_frame_is_pinned(pf), "page frame 0x%lx is pinned", + phys); + + /* If the backing store doesn't have a copy of the page, even if it + * wasn't modified, treat as dirty. This can happen for a few + * reasons: + * 1) Page has never been swapped out before, and the backing store + * wasn't pre-populated with this data page. + * 2) Page was swapped out before, but the page contents were not + * preserved after swapping back in. + * 3) Page contents were preserved when swapped back in, but were later + * evicted from the backing store to make room for other evicted + * pages. + */ + if (z_page_frame_is_mapped(pf)) { + dirty = dirty || !z_page_frame_is_backed(pf); + } + + if (dirty || page_fault) { + arch_mem_scratch(phys); + } + + if (z_page_frame_is_mapped(pf)) { + ret = z_backing_store_location_get(pf, location_ptr, + page_fault); + if (ret != 0) { + LOG_ERR("out of backing store memory"); + return -ENOMEM; + } + arch_mem_page_out(pf->addr, *location_ptr); + } else { + /* Shouldn't happen unless this function is mis-used */ + __ASSERT(!dirty, "un-mapped page determined to be dirty"); + } +#ifdef CONFIG_DEMAND_PAGING_ALLOW_IRQ + /* Mark as busy so that z_page_frame_is_evictable() returns false */ + __ASSERT(!z_page_frame_is_busy(pf), "page frame 0x%lx is already busy", + phys); + pf->flags |= Z_PAGE_FRAME_BUSY; +#endif + /* Update dirty parameter, since we set to true if it wasn't backed + * even if otherwise clean + */ + *dirty_ptr = dirty; + + return 0; +} + +static int do_mem_evict(void *addr) +{ + bool dirty; + struct z_page_frame *pf; + uintptr_t location; + int key, ret; + uintptr_t flags, phys; + +#if CONFIG_DEMAND_PAGING_ALLOW_IRQ + __ASSERT(!k_is_in_isr(), + "%s is unavailable in ISRs with CONFIG_DEMAND_PAGING_ALLOW_IRQ", + __func__); + k_sched_lock(); +#endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */ + key = irq_lock(); + flags = arch_page_info_get(addr, &phys, false); + __ASSERT((flags & ARCH_DATA_PAGE_NOT_MAPPED) == 0, + "address %p isn't mapped", addr); + if ((flags & ARCH_DATA_PAGE_LOADED) == 0) { + /* Un-mapped or already evicted. Nothing to do */ + ret = 0; + goto out; + } + + dirty = (flags & ARCH_DATA_PAGE_DIRTY) != 0; + pf = z_phys_to_page_frame(phys); + __ASSERT(pf->addr == addr, "page frame address mismatch"); + ret = page_frame_prepare_locked(pf, &dirty, false, &location); + if (ret != 0) { + goto out; + } + + __ASSERT(ret == 0, "failed to prepare page frame"); +#ifdef CONFIG_DEMAND_PAGING_ALLOW_IRQ + irq_unlock(key); +#endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */ + if (dirty) { + z_backing_store_page_out(location); + } +#ifdef CONFIG_DEMAND_PAGING_ALLOW_IRQ + key = irq_lock(); +#endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */ + page_frame_free_locked(pf); +out: + irq_unlock(key); +#ifdef CONFIG_DEMAND_PAGING_ALLOW_IRQ + k_sched_unlock(); +#endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */ + return ret; +} + +int k_mem_page_out(void *addr, size_t size) +{ + __ASSERT(page_frames_initialized, "%s called on %p too early", __func__, + addr); + z_mem_assert_virtual_region(addr, size); + + for (size_t offset = 0; offset < size; offset += CONFIG_MMU_PAGE_SIZE) { + void *pos = (uint8_t *)addr + offset; + int ret; + + ret = do_mem_evict(pos); + if (ret != 0) { + return ret; + } + } + + return 0; +} + +int z_page_frame_evict(uintptr_t phys) +{ + int key, ret; + struct z_page_frame *pf; + bool dirty; + uintptr_t flags; + uintptr_t location; + + __ASSERT(page_frames_initialized, "%s called on 0x%lx too early", + __func__, phys); + + /* Implementation is similar to do_page_fault() except there is no + * data page to page-in, see comments in that function. + */ + +#ifdef CONFIG_DEMAND_PAGING_ALLOW_IRQ + __ASSERT(!k_is_in_isr(), + "%s is unavailable in ISRs with CONFIG_DEMAND_PAGING_ALLOW_IRQ", + __func__); + k_sched_lock(); +#endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */ + key = irq_lock(); + pf = z_phys_to_page_frame(phys); + if (!z_page_frame_is_mapped(pf)) { + /* Nothing to do, free page */ + ret = 0; + goto out; + } + flags = arch_page_info_get(pf->addr, NULL, false); + /* Shouldn't ever happen */ + __ASSERT((flags & ARCH_DATA_PAGE_LOADED) != 0, "data page not loaded"); + dirty = (flags & ARCH_DATA_PAGE_DIRTY) != 0; + ret = page_frame_prepare_locked(pf, &dirty, false, &location); + if (ret != 0) { + goto out; + } + +#ifdef CONFIG_DEMAND_PAGING_ALLOW_IRQ + irq_unlock(key); +#endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */ + if (dirty) { + z_backing_store_page_out(location); + } +#ifdef CONFIG_DEMAND_PAGING_ALLOW_IRQ + key = irq_lock(); +#endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */ + page_frame_free_locked(pf); +out: + irq_unlock(key); +#ifdef CONFIG_DEMAND_PAGING_ALLOW_IRQ + k_sched_unlock(); +#endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */ + return ret; +} + +static bool do_page_fault(void *addr, bool pin) +{ + struct z_page_frame *pf; + int key, ret; + uintptr_t page_in_location, page_out_location; + enum arch_page_location status; + bool result; + bool dirty = false; + + __ASSERT(page_frames_initialized, "page fault at %p happened too early", + addr); + + LOG_DBG("page fault at %p", addr); + + /* + * TODO: Add performance accounting: + * - Number of pagefaults + * * gathered on a per-thread basis: + * . Pagefaults with IRQs locked in faulting thread (bad) + * . Pagefaults with IRQs unlocked in faulting thread + * * Pagefaults in ISRs (if allowed) + * - z_eviction_select() metrics + * * Clean vs dirty page eviction counts + * * execution time histogram + * * periodic timer execution time histogram (if implemented) + * - z_backing_store_page_out() execution time histogram + * - z_backing_store_page_in() execution time histogram + */ + +#ifdef CONFIG_DEMAND_PAGING_ALLOW_IRQ + /* We lock the scheduler so that other threads are never scheduled + * during the page-in/out operation. + * + * We do however re-enable interrupts during the page-in/page-out + * operation iff interrupts were enabled when the exception was taken; + * in this configuration page faults in an ISR are a bug; all their + * code/data must be pinned. + * + * If interrupts were disabled when the exception was taken, the + * arch code is responsible for keeping them that way when entering + * this function. + * + * If this is not enabled, then interrupts are always locked for the + * entire operation. This is far worse for system interrupt latency + * but requires less pinned pages and ISRs may also take page faults. + * + * Support for allowing z_backing_store_page_out() and + * z_backing_store_page_in() to also sleep and allow other threads to + * run (such as in the case where the transfer is async DMA) is not + * implemented. Even if limited to thread context, arbitrary memory + * access triggering exceptions that put a thread to sleep on a + * contended page fault operation will break scheduling assumptions of + * cooperative threads or threads that implement crticial sections with + * spinlocks or disabling IRQs. + */ + k_sched_lock(); + __ASSERT(!k_is_in_isr(), "ISR page faults are forbidden"); +#endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */ + + key = irq_lock(); + status = arch_page_location_get(addr, &page_in_location); + if (status == ARCH_PAGE_LOCATION_BAD) { + /* Return false to treat as a fatal error */ + result = false; + goto out; + } + result = true; + if (status == ARCH_PAGE_LOCATION_PAGED_IN) { + if (pin) { + /* It's a physical memory address */ + uintptr_t phys = page_in_location; + + pf = z_phys_to_page_frame(phys); + pf->flags |= Z_PAGE_FRAME_PINNED; + } + /* We raced before locking IRQs, re-try */ + goto out; + } + __ASSERT(status == ARCH_PAGE_LOCATION_PAGED_OUT, + "unexpected status value %d", status); + + pf = free_page_frame_list_get(); + if (pf == NULL) { + /* Need to evict a page frame */ + pf = z_eviction_select(&dirty); + __ASSERT(pf != NULL, "failed to get a page frame"); + LOG_DBG("evicting %p at 0x%lx", pf->addr, + z_page_frame_to_phys(pf)); + } + ret = page_frame_prepare_locked(pf, &dirty, true, &page_out_location); + __ASSERT(ret == 0, "failed to prepare page frame"); + +#ifdef CONFIG_DEMAND_PAGING_ALLOW_IRQ + irq_unlock(key); + /* Interrupts are now unlocked if they were not locked when we entered + * this function, and we may service ISRs. The scheduler is still + * locked. + */ +#endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */ + if (dirty) { + z_backing_store_page_out(page_out_location); + } + z_backing_store_page_in(page_in_location); + +#ifdef CONFIG_DEMAND_PAGING_ALLOW_IRQ + key = irq_lock(); + pf->flags &= ~Z_PAGE_FRAME_BUSY; +#endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */ + if (pin) { + pf->flags |= Z_PAGE_FRAME_PINNED; + } + pf->flags |= Z_PAGE_FRAME_MAPPED; + pf->addr = addr; + arch_mem_page_in(addr, z_page_frame_to_phys(pf)); + z_backing_store_page_finalize(pf, page_in_location); +out: + irq_unlock(key); +#ifdef CONFIG_DEMAND_PAGING_ALLOW_IRQ + k_sched_unlock(); +#endif /* CONFIG_DEMAND_PAGING_ALLOW_IRQ */ + + return result; +} + +static void do_page_in(void *addr) +{ + bool ret; + + ret = do_page_fault(addr, false); + __ASSERT(ret, "unmapped memory address %p", addr); + (void)ret; +} + +void k_mem_page_in(void *addr, size_t size) +{ + __ASSERT(!IS_ENABLED(CONFIG_DEMAND_PAGING_ALLOW_IRQ) || !k_is_in_isr(), + "%s may not be called in ISRs if CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled", + __func__); + virt_region_foreach(addr, size, do_page_in); +} + +static void do_mem_pin(void *addr) +{ + bool ret; + + ret = do_page_fault(addr, true); + __ASSERT(ret, "unmapped memory address %p", addr); + (void)ret; +} + +void k_mem_pin(void *addr, size_t size) +{ + __ASSERT(!IS_ENABLED(CONFIG_DEMAND_PAGING_ALLOW_IRQ) || !k_is_in_isr(), + "%s may not be called in ISRs if CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled", + __func__); + virt_region_foreach(addr, size, do_mem_pin); +} + +bool z_page_fault(void *addr) +{ + bool ret; + + ret = do_page_fault(addr, false); + if (ret) { + /* Wasn't an error, increment page fault count */ + int key; + + key = irq_lock(); + z_num_pagefaults++; + irq_unlock(key); + } + return ret; +} + +unsigned long z_num_pagefaults_get(void) +{ + unsigned long ret; + int key; + + key = irq_lock(); + ret = z_num_pagefaults; + irq_unlock(key); + + return ret; +} + +static void do_mem_unpin(void *addr) +{ + struct z_page_frame *pf; + int key; + uintptr_t flags, phys; + + key = irq_lock(); + flags = arch_page_info_get(addr, &phys, false); + __ASSERT((flags & ARCH_DATA_PAGE_NOT_MAPPED) == 0, + "invalid data page at %p", addr); + if ((flags & ARCH_DATA_PAGE_LOADED) != 0) { + pf = z_phys_to_page_frame(phys); + pf->flags &= ~Z_PAGE_FRAME_PINNED; + } + irq_unlock(key); +} + +void k_mem_unpin(void *addr, size_t size) +{ + __ASSERT(page_frames_initialized, "%s called on %p too early", __func__, + addr); + virt_region_foreach(addr, size, do_mem_unpin); +} +#endif /* CONFIG_DEMAND_PAGING */ diff --git a/lib/libc/Kconfig b/lib/libc/Kconfig index d48eef3ec9c4..39c7d42c42b7 100644 --- a/lib/libc/Kconfig +++ b/lib/libc/Kconfig @@ -52,6 +52,16 @@ config NEWLIB_LIBC_NANO The newlib-nano library for ARM embedded processors is a part of the GNU Tools for ARM Embedded Processors. +config NEWLIB_LIBC_MAX_MAPPED_REGION_SIZE + int "Maximum memory mapped for newlib heap" + depends on MMU + default 1048576 + help + On MMU-based systems, indicates the maximum amount of memory which + will be used for the newlib malloc() heap. The actual amount of + memory used will be the minimum of this value and the amount of + free physical memory at kernel boot. + config NEWLIB_LIBC_ALIGNED_HEAP_SIZE int "Newlib aligned heap size" depends on MPU_REQUIRES_POWER_OF_TWO_ALIGNMENT diff --git a/lib/libc/newlib/libc-hooks.c b/lib/libc/newlib/libc-hooks.c index 0b5cb9c61367..a620792aab9b 100644 --- a/lib/libc/newlib/libc-hooks.c +++ b/lib/libc/newlib/libc-hooks.c @@ -16,91 +16,112 @@ #include #include #include +#include #define LIBC_BSS K_APP_BMEM(z_libc_partition) #define LIBC_DATA K_APP_DMEM(z_libc_partition) -#if CONFIG_NEWLIB_LIBC_ALIGNED_HEAP_SIZE -K_APPMEM_PARTITION_DEFINE(z_malloc_partition); -#define MALLOC_BSS K_APP_BMEM(z_malloc_partition) - -/* Compiler will throw an error if the provided value isn't a power of two */ -MALLOC_BSS static unsigned char __aligned(CONFIG_NEWLIB_LIBC_ALIGNED_HEAP_SIZE) - heap_base[CONFIG_NEWLIB_LIBC_ALIGNED_HEAP_SIZE]; -#define MAX_HEAP_SIZE CONFIG_NEWLIB_LIBC_ALIGNED_HEAP_SIZE - -#else /* CONFIG_NEWLIB_LIBC_ALIGNED_HEAP_SIZE */ -/* Heap base and size are determined based on the available unused SRAM, - * in the interval from a properly aligned address after the linker symbol - * `_end`, to the end of SRAM +/* + * End result of this thorny set of ifdefs is to define: + * + * - HEAP_BASE base address of the heap arena + * - MAX_HEAP_SIZE size of the heap arena */ -#define USED_RAM_END_ADDR POINTER_TO_UINT(&_end) -#ifdef Z_MALLOC_PARTITION_EXISTS -/* Need to be able to program a memory protection region from HEAP_BASE - * to the end of RAM so that user threads can get at it. - * Implies that the base address needs to be suitably aligned since the - * bounds have to go in a k_mem_partition. - */ #ifdef CONFIG_MMU -/* Linker script may already have done this, but just to be safe */ -#define HEAP_BASE ROUND_UP(USED_RAM_END_ADDR, CONFIG_MMU_PAGE_SIZE) -#else /* MPU-based systems */ -/* TODO: Need a generic Kconfig for the MPU region granularity */ -#if defined(CONFIG_ARM) -#define HEAP_BASE ROUND_UP(USED_RAM_END_ADDR, \ + #ifdef CONFIG_USERSPACE + struct k_mem_partition z_malloc_partition; + #endif + + LIBC_BSS static unsigned char *heap_base; + LIBC_BSS static size_t max_heap_size; + + #define HEAP_BASE heap_base + #define MAX_HEAP_SIZE max_heap_size + #define USE_MALLOC_PREPARE 1 +#elif CONFIG_NEWLIB_LIBC_ALIGNED_HEAP_SIZE + /* Arena size expressed in Kconfig, due to power-of-two size/align + * requirements of certain MPUs. + * + * We use an automatic memory partition instead of setting this up + * in malloc_prepare(). + */ + K_APPMEM_PARTITION_DEFINE(z_malloc_partition); + #define MALLOC_BSS K_APP_BMEM(z_malloc_partition) + + /* Compiler will throw an error if the provided value isn't a + * power of two + */ + MALLOC_BSS static unsigned char + __aligned(CONFIG_NEWLIB_LIBC_ALIGNED_HEAP_SIZE) + heap_base[CONFIG_NEWLIB_LIBC_ALIGNED_HEAP_SIZE]; + #define MAX_HEAP_SIZE CONFIG_NEWLIB_LIBC_ALIGNED_HEAP_SIZE + #define HEAP_BASE heap_base +#else /* Not MMU or CONFIG_NEWLIB_LIBC_ALIGNED_HEAP_SIZE */ + #define USED_RAM_END_ADDR POINTER_TO_UINT(&_end) + + #ifdef Z_MALLOC_PARTITION_EXISTS + /* Start of malloc arena needs to be aligned per MPU + * requirements + */ + struct k_mem_partition z_malloc_partition; + + #if defined(CONFIG_ARM) + #define HEAP_BASE ROUND_UP(USED_RAM_END_ADDR, \ CONFIG_ARM_MPU_REGION_MIN_ALIGN_AND_SIZE) -#elif defined(CONFIG_ARC) -#define HEAP_BASE ROUND_UP(USED_RAM_END_ADDR, Z_ARC_MPU_ALIGN) -#else -#error "Unsupported platform" -#endif /* CONFIG_ */ -#endif /* !CONFIG_MMU */ -#else /* !Z_MALLOC_PARTITION_EXISTS */ -/* No partition, heap can just start wherever _end is */ -#define HEAP_BASE USED_RAM_END_ADDR -#endif /* Z_MALLOC_PARTITION_EXISTS */ - -#ifdef CONFIG_MMU -/* Currently a placeholder, we're just setting up all unused RAM pages - * past the end of the kernel as the heap arena. SRAM_BASE_ADDRESS is - * a physical address so we can't use that. - * - * Later, we should do this much more like other VM-enabled operating systems: - * - Set up a core kernel ontology of free physical pages - * - Allow anonymous memoory mappings drawing from the pool of free physical - * pages - * - Have _sbrk() map anonymous pages into the heap arena as needed - * - See: https://github.com/zephyrproject-rtos/zephyr/issues/29526 - */ -#define MAX_HEAP_SIZE (CONFIG_KERNEL_RAM_SIZE - (HEAP_BASE - \ - CONFIG_KERNEL_VM_BASE)) -#elif defined(CONFIG_XTENSA) -extern void *_heap_sentry; -#define MAX_HEAP_SIZE (POINTER_TO_UINT(&_heap_sentry) - HEAP_BASE) -#else -#define MAX_HEAP_SIZE (KB(CONFIG_SRAM_SIZE) - (HEAP_BASE - \ - CONFIG_SRAM_BASE_ADDRESS)) -#endif /* CONFIG_MMU */ - -#if Z_MALLOC_PARTITION_EXISTS -struct k_mem_partition z_malloc_partition; + #elif defined(CONFIG_ARC) + #define HEAP_BASE ROUND_UP(USED_RAM_END_ADDR, \ + Z_ARC_MPU_ALIGN) + #else + #error "Unsupported platform" + #endif /* CONFIG_ */ + #define USE_MALLOC_PREPARE 1 + #else + /* End of kernel image */ + #define HEAP_BASE USED_RAM_END_ADDR + #endif + + /* End of the malloc arena is the end of physical memory */ + #if defined(CONFIG_XTENSA) + /* TODO: Why is xtensa a special case? */ + extern void *_heap_sentry; + #define MAX_HEAP_SIZE (POINTER_TO_UINT(&_heap_sentry) - \ + HEAP_BASE) + #else + #define MAX_HEAP_SIZE (KB(CONFIG_SRAM_SIZE) - (HEAP_BASE - \ + CONFIG_SRAM_BASE_ADDRESS)) + #endif /* CONFIG_XTENSA */ +#endif +#ifdef USE_MALLOC_PREPARE static int malloc_prepare(const struct device *unused) { ARG_UNUSED(unused); - z_malloc_partition.start = HEAP_BASE; - z_malloc_partition.size = MAX_HEAP_SIZE; +#ifdef CONFIG_MMU + max_heap_size = MIN(CONFIG_NEWLIB_LIBC_MAX_MAPPED_REGION_SIZE, + k_mem_free_get()); + + if (max_heap_size != 0) { + heap_base = k_mem_map(max_heap_size, K_MEM_PERM_RW); + __ASSERT(heap_base != NULL, + "failed to allocate heap of size %zu", max_heap_size); + + } +#endif /* CONFIG_MMU */ +#ifdef Z_MALLOC_PARTITION_EXISTS + z_malloc_partition.start = (uintptr_t)HEAP_BASE; + z_malloc_partition.size = (size_t)MAX_HEAP_SIZE; z_malloc_partition.attr = K_MEM_PARTITION_P_RW_U_RW; +#endif /* Z_MALLOC_PARTITION_EXISTS */ return 0; } SYS_INIT(malloc_prepare, APPLICATION, CONFIG_KERNEL_INIT_PRIORITY_DEFAULT); -#endif /* CONFIG_USERSPACE */ -#endif /* CONFIG_NEWLIB_LIBC_ALIGNED_HEAP_SIZE */ +#endif /* USE_MALLOC_PREPARE */ -LIBC_BSS static unsigned int heap_sz; +/* Current offset from HEAP_BASE of unused memory */ +LIBC_BSS static size_t heap_sz; static int _stdout_hook_default(int c) { @@ -248,18 +269,13 @@ __weak void _exit(int status) static LIBC_DATA SYS_SEM_DEFINE(heap_sem, 1, 1); -void *_sbrk(int count) +void *_sbrk(intptr_t count) { void *ret, *ptr; /* coverity[CHECKED_RETURN] */ sys_sem_take(&heap_sem, K_FOREVER); - -#if CONFIG_NEWLIB_LIBC_ALIGNED_HEAP_SIZE - ptr = heap_base + heap_sz; -#else ptr = ((char *)HEAP_BASE) + heap_sz; -#endif if ((heap_sz + count) < MAX_HEAP_SIZE) { heap_sz += count; diff --git a/soc/x86/apollo_lake/Kconfig.soc b/soc/x86/apollo_lake/Kconfig.soc index acde2242e09f..5440ce3e704d 100644 --- a/soc/x86/apollo_lake/Kconfig.soc +++ b/soc/x86/apollo_lake/Kconfig.soc @@ -10,3 +10,4 @@ config SOC_APOLLO_LAKE select PCIE_MSI select DYNAMIC_INTERRUPTS select X86_MMU + select ARCH_HAS_RESERVED_PAGE_FRAMES diff --git a/soc/x86/atom/Kconfig.soc b/soc/x86/atom/Kconfig.soc index 4dde328ccc2f..ac396eec8802 100644 --- a/soc/x86/atom/Kconfig.soc +++ b/soc/x86/atom/Kconfig.soc @@ -6,3 +6,4 @@ config SOC_ATOM select CPU_ATOM select X86_MMU select ARCH_HAS_USERSPACE + select ARCH_HAS_RESERVED_PAGE_FRAMES diff --git a/soc/x86/ia32/Kconfig.soc b/soc/x86/ia32/Kconfig.soc index 66cd91c8f92e..8b00c5bf3e58 100644 --- a/soc/x86/ia32/Kconfig.soc +++ b/soc/x86/ia32/Kconfig.soc @@ -4,3 +4,4 @@ config SOC_IA32 bool "Generic IA32 SoC" select X86 select CPU_MINUTEIA + select ARCH_HAS_RESERVED_PAGE_FRAMES if SRAM_BASE_ADDRESS = 0 diff --git a/subsys/CMakeLists.txt b/subsys/CMakeLists.txt index 80d7a838b35b..9608ddec21ca 100644 --- a/subsys/CMakeLists.txt +++ b/subsys/CMakeLists.txt @@ -26,3 +26,4 @@ add_subdirectory(tracing) add_subdirectory_ifdef(CONFIG_JWT jwt) add_subdirectory(canbus) add_subdirectory_ifdef(CONFIG_TIMING_FUNCTIONS timing) +add_subdirectory_ifdef(CONFIG_DEMAND_PAGING demand_paging) diff --git a/subsys/Kconfig b/subsys/Kconfig index 6c5a95d671f0..e6b60b9c67ac 100644 --- a/subsys/Kconfig +++ b/subsys/Kconfig @@ -58,4 +58,6 @@ source "subsys/timing/Kconfig" source "subsys/tracing/Kconfig" +source "subsys/demand_paging/Kconfig" + endmenu diff --git a/subsys/demand_paging/CMakeLists.txt b/subsys/demand_paging/CMakeLists.txt new file mode 100644 index 000000000000..486e9f9ad2ed --- /dev/null +++ b/subsys/demand_paging/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +add_subdirectory(eviction) +add_subdirectory(backing_store) diff --git a/subsys/demand_paging/Kconfig b/subsys/demand_paging/Kconfig new file mode 100644 index 000000000000..3e1626f989ec --- /dev/null +++ b/subsys/demand_paging/Kconfig @@ -0,0 +1,11 @@ +# Copyright (c) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +menu "Demand Paging modules" + depends on DEMAND_PAGING + +source "subsys/demand_paging/eviction/Kconfig" + +source "subsys/demand_paging/backing_store/Kconfig" + +endmenu diff --git a/subsys/demand_paging/backing_store/CMakeLists.txt b/subsys/demand_paging/backing_store/CMakeLists.txt new file mode 100644 index 000000000000..bbeb3320834d --- /dev/null +++ b/subsys/demand_paging/backing_store/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright (c) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +add_definitions(-D__ZEPHYR_SUPERVISOR__) + +include_directories( + ${ZEPHYR_BASE}/kernel/include + ${ARCH_DIR}/${ARCH}/include + ) + +if(NOT DEFINED CONFIG_BACKING_STORE_CUSTOM) + zephyr_library() + zephyr_library_sources_ifdef(CONFIG_BACKING_STORE_RAM ram.c) +endif() diff --git a/subsys/demand_paging/backing_store/Kconfig b/subsys/demand_paging/backing_store/Kconfig new file mode 100644 index 000000000000..bdbc4c9d3e58 --- /dev/null +++ b/subsys/demand_paging/backing_store/Kconfig @@ -0,0 +1,32 @@ +# Copyright (c) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +choice BACKING_STORE_CHOICE + prompt "Backing store algorithms" + default BACKING_STORE_CUSTOM + +config BACKING_STORE_CUSTOM + bool "Custom backing store implementation" + help + This option is chosen when the backing store will be implemented in + the application. This will be typical as these tend to be very + hardware-dependent. + +config BACKING_STORE_RAM + bool "RAM-based test backing store" + help + This implements a backing store using physical RAM pages that the + Zephyr kernel is otherwise unaware of. It is intended for + demonstration and testing of the demand paging feature. +endchoice + +if BACKING_STORE_RAM +config BACKING_STORE_RAM_PAGES + int "Number of pages for RAM backing store" + default 16 + help + Number of pages of backing store memory to reserve in RAM. All test + cases for demand paging assume that there are at least 16 pages of + backing store storage available. + +endif # BACKING_STORE_RAM diff --git a/subsys/demand_paging/backing_store/ram.c b/subsys/demand_paging/backing_store/ram.c new file mode 100644 index 000000000000..fd324396ff08 --- /dev/null +++ b/subsys/demand_paging/backing_store/ram.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2020 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + * + * RAM-based memory buffer backing store implementation for demo purposes + */ +#include +#include +#include + +/* + * TODO: + * + * This is a demonstration backing store for testing the kernel side of the + * demand paging feature. In production there are basically two types of + * backing stores: + * + * 1) A large, sparse backing store that is big enough to capture the entire + * address space. Implementation of these is very simple; the location + * token is just a function of the evicted virtual address and no space + * management is necessary. Clean copies of paged-in data pages may be kept + * indefinitely. + * + * 2) A backing store that has limited storage space, and is not sufficiently + * large to hold clean copies of all mapped memory. + * + * This backing store is an example of the latter case. However, locations + * are freed as soon as pages are paged in, in z_backing_store_page_finalize(). + * This implies that all data pages are treated as dirty as + * Z_PAGE_FRAME_BACKED is never set, even if the data page was paged out before + * and not modified since then. + * + * An optimization a real backing store will want is have + * z_backing_store_page_finalize() note the storage location of a paged-in + * data page in a custom field of its associated z_page_frame, and set the + * Z_PAGE_FRAME_BACKED bit. Invocations of z_backing_store_location_get() will + * have logic to return the previous clean page location instead of allocating + * a new one if Z_PAGE_FRAME_BACKED is set. + * + * This will, however, require the implementation of a clean page + * eviction algorithm, to free backing store locations for loaded data pages + * as the backing store fills up, and clear the Z_PAGE_FRAME_BACKED bit + * appropriately. + * + * All of this logic is local to the backing store implementation; from the + * core kernel's perspective the only change is that Z_PAGE_FRAME_BACKED + * starts getting set for certain page frames after a page-in (and possibly + * cleared at a later time). + */ +static char backing_store[CONFIG_MMU_PAGE_SIZE * + CONFIG_BACKING_STORE_RAM_PAGES]; +static struct k_mem_slab backing_slabs; +static unsigned int free_slabs; + +static void *location_to_slab(uintptr_t location) +{ + __ASSERT(location % CONFIG_MMU_PAGE_SIZE == 0, + "unaligned location 0x%lx", location); + __ASSERT(location < + (CONFIG_BACKING_STORE_RAM_PAGES * CONFIG_MMU_PAGE_SIZE), + "bad location 0x%lx, past bounds of backing store", location); + + return backing_store + location; +} + +static uintptr_t slab_to_location(void *slab) +{ + char *pos = slab; + uintptr_t offset; + + __ASSERT(pos >= backing_store && + pos < backing_store + ARRAY_SIZE(backing_store), + "bad slab pointer %p", slab); + offset = pos - backing_store; + __ASSERT(offset % CONFIG_MMU_PAGE_SIZE == 0, + "unaligned slab pointer %p", slab); + + return offset; +} + +int z_backing_store_location_get(struct z_page_frame *pf, uintptr_t *location, + bool page_fault) +{ + int ret; + void *slab; + + if ((!page_fault && free_slabs == 1) || free_slabs == 0) { + return -ENOMEM; + } + + ret = k_mem_slab_alloc(&backing_slabs, &slab, K_NO_WAIT); + __ASSERT(ret == 0, "slab count mismatch"); + (void)ret; + *location = slab_to_location(slab); + free_slabs--; + + return 0; +} + +void z_backing_store_location_free(uintptr_t location) +{ + void *slab = location_to_slab(location); + + k_mem_slab_free(&backing_slabs, &slab); + free_slabs++; +} + +void z_backing_store_page_out(uintptr_t location) +{ + (void)memcpy(location_to_slab(location), Z_SCRATCH_PAGE, + CONFIG_MMU_PAGE_SIZE); +} + +void z_backing_store_page_in(uintptr_t location) +{ + (void)memcpy(Z_SCRATCH_PAGE, location_to_slab(location), + CONFIG_MMU_PAGE_SIZE); +} + +void z_backing_store_page_finalize(struct z_page_frame *pf, uintptr_t location) +{ + z_backing_store_location_free(location); +} + +void z_backing_store_init(void) +{ + k_mem_slab_init(&backing_slabs, backing_store, CONFIG_MMU_PAGE_SIZE, + CONFIG_BACKING_STORE_RAM_PAGES); + free_slabs = CONFIG_BACKING_STORE_RAM_PAGES; +} diff --git a/subsys/demand_paging/eviction/CMakeLists.txt b/subsys/demand_paging/eviction/CMakeLists.txt new file mode 100644 index 000000000000..4b28d8ebe252 --- /dev/null +++ b/subsys/demand_paging/eviction/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright (c) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +add_definitions(-D__ZEPHYR_SUPERVISOR__) + +include_directories( + ${ZEPHYR_BASE}/kernel/include + ${ARCH_DIR}/${ARCH}/include + ) + +if(NOT DEFINED CONFIG_EVICTION_CUSTOM) + zephyr_library() + zephyr_library_sources_ifdef(CONFIG_EVICTION_NRU nru.c) +endif() diff --git a/subsys/demand_paging/eviction/Kconfig b/subsys/demand_paging/eviction/Kconfig new file mode 100644 index 000000000000..05f8b8d3334d --- /dev/null +++ b/subsys/demand_paging/eviction/Kconfig @@ -0,0 +1,40 @@ +# Copyright (c) 2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# Demand paging sample eviction algorithms + +choice EVICTION_CHOICE + prompt "Page frame eviction algorithms" + default EVICTION_NRU + depends on DEMAND_PAGING + +config EVICTION_CUSTOM + bool "Custom eviction algorithm" + help + This option is chosen when the eviction algorithm will be implemented + by the application, instead of using one included in Zephyr. + +config EVICTION_NRU + bool "Not Recently Used (NRU) page eviction algorithm" + help + This implements a Not Recently Used page eviction algorithm. + A periodic timer will clear the accessed state of all virtual pages. + When a page frame needs to be evicted, the algorithm will prefer to + evict page frames using an ascending order of priority: + + - recently accessed, dirty + - recently accessed, clean + - not recently accessed, dirty + - not recently accessed, clean + +endchoice + +if EVICTION_NRU +config EVICTION_NRU_PERIOD + int "Recently accessed period, in milliseconds" + default 100 + help + A periodic timer will fire that clears the accessed state of all virtual + pages that are capable of being paged out. At eviction time, if a page + still has the accessed property, it will be considered as recently used. +endif # EVICTION_NRU diff --git a/subsys/demand_paging/eviction/nru.c b/subsys/demand_paging/eviction/nru.c new file mode 100644 index 000000000000..bfdb9ea2bf9e --- /dev/null +++ b/subsys/demand_paging/eviction/nru.c @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2020 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + * + * Not Recently Used (NRU) eviction algorithm for demand paging + */ +#include +#include +#include +#include + +/* The accessed and dirty states of each page frame are used to create + * a hierarchy with a numerical value. When evicting a page, try to evict + * page with the highest value (we prefer clean, not accessed pages). + * + * In this ontology, "accessed" means "recently accessed" and gets cleared + * during the periodic update. + * + * 0 not accessed, clean + * 1 not accessed, dirty + * 2 accessed, clean + * 3 accessed, dirty + */ +static void nru_periodic_update(struct k_timer *timer) +{ + uintptr_t phys; + struct z_page_frame *pf; + int key = irq_lock(); + + Z_PAGE_FRAME_FOREACH(phys, pf) { + if (!z_page_frame_is_evictable(pf)) { + continue; + } + + /* Clear accessed bit in page tables */ + (void)arch_page_info_get(pf->addr, NULL, true); + } + + irq_unlock(key); +} + +struct z_page_frame *z_eviction_select(bool *dirty_ptr) +{ + unsigned int last_prec = 4U; + struct z_page_frame *last_pf = NULL, *pf; + bool accessed; + bool dirty = false; + uintptr_t flags, phys; + + Z_PAGE_FRAME_FOREACH(phys, pf) { + unsigned int prec; + + if (!z_page_frame_is_evictable(pf)) { + continue; + } + + flags = arch_page_info_get(pf->addr, NULL, false); + accessed = (flags & ARCH_DATA_PAGE_ACCESSED) != 0UL; + dirty = (flags & ARCH_DATA_PAGE_DIRTY) != 0UL; + + /* Implies a mismatch with page frame ontology and page + * tables + */ + __ASSERT((flags & ARCH_DATA_PAGE_LOADED) != 0U, + "non-present page, %s", + ((flags & ARCH_DATA_PAGE_NOT_MAPPED) != 0U) ? + "un-mapped" : "paged out"); + + prec = (dirty ? 1U : 0U) + (accessed ? 2U : 0U); + if (prec == 0) { + /* If we find a not accessed, clean page we're done */ + last_pf = pf; + break; + } + + if (prec < last_prec) { + last_prec = prec; + last_pf = pf; + } + } + /* Shouldn't ever happen unless every page is pinned */ + __ASSERT(last_pf != NULL, "no page to evict"); + + *dirty_ptr = dirty; + + return last_pf; +} + +static K_TIMER_DEFINE(nru_timer, nru_periodic_update, NULL); + +void z_eviction_init(void) +{ + k_timer_start(&nru_timer, K_NO_WAIT, + K_MSEC(CONFIG_EVICTION_NRU_PERIOD)); +} diff --git a/tests/arch/x86/pagetables/src/main.c b/tests/arch/x86/pagetables/src/main.c index 3ca9f04f6a15..a2ae3323b75e 100644 --- a/tests/arch/x86/pagetables/src/main.c +++ b/tests/arch/x86/pagetables/src/main.c @@ -15,11 +15,9 @@ #include #include #include +#include #include "main.h" -#define VM_BASE ((uint8_t *)CONFIG_KERNEL_VM_BASE) -#define VM_LIMIT (VM_BASE + CONFIG_KERNEL_RAM_SIZE) - #ifdef CONFIG_X86_64 #define PT_LEVEL 3 #elif CONFIG_X86_PAE @@ -45,6 +43,10 @@ extern char _locore_start[]; extern char _locore_size[]; extern char _lorodata_start[]; extern char _lorodata_size[]; +extern char _lodata_end[]; + +#define LOCORE_START ((uint8_t *)&_locore_start) +#define LOCORE_END ((uint8_t *)&_lodata_end) #endif #ifdef CONFIG_COVERAGE_GCOV @@ -52,6 +54,21 @@ extern char __gcov_bss_start[]; extern char __gcov_bss_size[]; #endif +static pentry_t get_entry(pentry_t *flags, void *addr) +{ + int level; + pentry_t entry; + + z_x86_pentry_get(&level, &entry, z_x86_page_tables_get(), addr); + + zassert_true((entry & MMU_P) != 0, + "non-present RAM entry"); + zassert_equal(level, PT_LEVEL, "bigpage found"); + *flags = entry & FLAGS_MASK; + + return entry; +} + /** * Test that MMU flags on RAM virtual address range are set properly * @@ -61,21 +78,16 @@ void test_ram_perms(void) { uint8_t *pos; - for (pos = VM_BASE; pos < VM_LIMIT; pos += CONFIG_MMU_PAGE_SIZE) { - int level; - pentry_t entry, flags, expected; + pentry_t entry, flags, expected; + for (pos = Z_KERNEL_VIRT_START; pos < Z_KERNEL_VIRT_END; + pos += CONFIG_MMU_PAGE_SIZE) { if (pos == NULL) { /* We have another test specifically for NULL page */ continue; } - z_x86_pentry_get(&level, &entry, z_x86_page_tables_get(), pos); - - zassert_true((entry & MMU_P) != 0, - "non-present RAM entry"); - zassert_equal(level, PT_LEVEL, "bigpage found"); - flags = entry & FLAGS_MASK; + entry = get_entry(&flags, pos); if (!IS_ENABLED(CONFIG_SRAM_REGION_PERMISSIONS)) { expected = MMU_P | MMU_RW; @@ -87,21 +99,8 @@ void test_ram_perms(void) } else if (IN_REGION(__gcov_bss, pos)) { expected = MMU_P | MMU_RW | MMU_US | MMU_XD; #endif -#ifdef CONFIG_X86_64 - } else if (IN_REGION(_locore, pos)) { - if (IS_ENABLED(CONFIG_X86_KPTI)) { - expected = MMU_P | MMU_US; - } else { - expected = MMU_P; - } - } else if (IN_REGION(_lorodata, pos)) { - if (IS_ENABLED(CONFIG_X86_KPTI)) { - expected = MMU_P | MMU_US | MMU_XD; - } else { - expected = MMU_P | MMU_XD; - } -#endif /* CONFIG_X86_64 */ -#if !defined(CONFIG_X86_KPTI) && !defined(CONFIG_X86_COMMON_PAGE_TABLE) +#if !defined(CONFIG_X86_KPTI) && !defined(CONFIG_X86_COMMON_PAGE_TABLE) && \ + defined(CONFIG_USERSPACE) } else if (IN_REGION(_app_smem, pos)) { /* If KPTI is not enabled, then the default memory * domain affects our page tables even though we are @@ -124,12 +123,58 @@ void test_ram_perms(void) */ expected = MMU_P | MMU_RW | MMU_XD; } + zassert_equal(flags, expected, + "bad flags " PRI_ENTRY " at %p, expected " + PRI_ENTRY, flags, pos, expected); + } +#ifdef CONFIG_X86_64 + /* Check the locore too */ + for (pos = LOCORE_START; pos < LOCORE_END; + pos += CONFIG_MMU_PAGE_SIZE) { + if (pos == NULL) { + /* We have another test specifically for NULL page */ + continue; + } + + entry = get_entry(&flags, pos); + + if (IN_REGION(_locore, pos)) { + if (IS_ENABLED(CONFIG_X86_KPTI)) { + expected = MMU_P | MMU_US; + } else { + expected = MMU_P; + } + } else if (IN_REGION(_lorodata, pos)) { + if (IS_ENABLED(CONFIG_X86_KPTI)) { + expected = MMU_P | MMU_US | MMU_XD; + } else { + expected = MMU_P | MMU_XD; + } + } else { + expected = MMU_P | MMU_RW | MMU_XD; + } zassert_equal(flags, expected, "bad flags " PRI_ENTRY " at %p, expected " PRI_ENTRY, flags, pos, expected); } +#endif /* CONFIG_X86_64 */ +#ifdef CONFIG_ARCH_MAPS_ALL_RAM + /* All RAM page frame entries aside from 0x0 must have a mapping. + * We currently identity-map on x86, no conversion necessary other than a cast + */ + for (pos = (uint8_t *)Z_PHYS_RAM_START; pos < (uint8_t *)Z_PHYS_RAM_END; + pos += CONFIG_MMU_PAGE_SIZE) { + if (pos == NULL) { + continue; + } + + entry = get_entry(&flags, pos); + zassert_true((flags & MMU_P) != 0, + "address %p isn't mapped", pos); + } +#endif } /** @@ -155,11 +200,13 @@ void z_impl_dump_my_ptables(void) z_x86_dump_page_tables(z_x86_thread_page_tables_get(cur)); } +#ifdef CONFIG_USERSPACE void z_vrfy_dump_my_ptables(void) { z_impl_dump_my_ptables(); } #include +#endif /* CONFIG_USERSPACE */ /** * Dump kernel's page tables to console diff --git a/tests/kernel/context/testcase.yaml b/tests/kernel/context/testcase.yaml index d3854ccfa8fc..0476346b781d 100644 --- a/tests/kernel/context/testcase.yaml +++ b/tests/kernel/context/testcase.yaml @@ -1,3 +1,5 @@ tests: - kernel.common: + kernel.context: tags: kernel + # Bug is #31333 + filter: not CONFIG_DEMAND_PAGING diff --git a/tests/kernel/mem_protect/demand_paging/CMakeLists.txt b/tests/kernel/mem_protect/demand_paging/CMakeLists.txt new file mode 100644 index 000000000000..740ecd78d7ec --- /dev/null +++ b/tests/kernel/mem_protect/demand_paging/CMakeLists.txt @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.13.1) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(mem_map) + +target_include_directories(app PRIVATE + ${ZEPHYR_BASE}/kernel/include + ${ZEPHYR_BASE}/arch/${ARCH}/include + ) + +FILE(GLOB app_sources src/*.c) +target_sources(app PRIVATE ${app_sources}) diff --git a/tests/kernel/mem_protect/demand_paging/prj.conf b/tests/kernel/mem_protect/demand_paging/prj.conf new file mode 100644 index 000000000000..9467c2926896 --- /dev/null +++ b/tests/kernel/mem_protect/demand_paging/prj.conf @@ -0,0 +1 @@ +CONFIG_ZTEST=y diff --git a/tests/kernel/mem_protect/demand_paging/src/main.c b/tests/kernel/mem_protect/demand_paging/src/main.c new file mode 100644 index 000000000000..432b781ca929 --- /dev/null +++ b/tests/kernel/mem_protect/demand_paging/src/main.c @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2021 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#ifdef CONFIG_BACKING_STORE_RAM_PAGES +#define EXTRA_PAGES (CONFIG_BACKING_STORE_RAM_PAGES - 1) +#else +#error "Unsupported configuration" +#endif + +size_t arena_size; +char *arena; + +static bool expect_fault; + +void k_sys_fatal_error_handler(unsigned int reason, const z_arch_esf_t *pEsf) +{ + printk("Caught system error -- reason %d\n", reason); + + if (expect_fault && reason == 0) { + expect_fault = false; + ztest_test_pass(); + } else { + printk("Unexpected fault during test"); + k_fatal_halt(reason); + } +} + +/* The mapped anonymous area will be free RAM plus half of the available + * frames in the backing store. + */ +#define HALF_PAGES (EXTRA_PAGES / 2) +#define HALF_BYTES (HALF_PAGES * CONFIG_MMU_PAGE_SIZE) +static const char *nums = "0123456789"; + +void test_map_anon_pages(void) +{ + arena_size = k_mem_free_get() + HALF_BYTES; + arena = k_mem_map(arena_size, K_MEM_PERM_RW); + + zassert_not_null(arena, "failed to map anonymous memory arena size %zu", + arena_size); + printk("Anonymous memory arena %p size %zu\n", arena, arena_size); + z_page_frames_dump(); +} + +void test_touch_anon_pages(void) +{ + unsigned long faults; + + faults = z_num_pagefaults_get(); + + printk("checking zeroes\n"); + /* The mapped area should have started out zeroed. Check this. */ + for (size_t i = 0; i < arena_size; i++) { + zassert_equal(arena[i], '\x00', + "page not zeroed got 0x%hhx at index %d", + arena[i], i); + } + + printk("writing data\n"); + /* Write a pattern of data to the whole arena */ + for (size_t i = 0; i < arena_size; i++) { + arena[i] = nums[i % 10]; + } + + printk("reading data\n"); + /* And ensure it can be read back */ + for (size_t i = 0; i < arena_size; i++) { + zassert_equal(arena[i], nums[i % 10], + "arena corrupted at index %d (%p): got 0x%hhx expected 0x%hhx", + i, &arena[i], arena[i], nums[i % 10]); + arena[i] = 0; + } + + faults = z_num_pagefaults_get() - faults; + + /* Specific number depends on how much RAM we have but shouldn't be 0 */ + zassert_not_equal(faults, 0UL, "no page faults handled?"); + printk("Kernel handled %lu page faults\n", faults); +} + +void test_k_mem_page_out(void) +{ + unsigned long faults; + int key, ret; + + /* Lock IRQs to prevent other pagefaults from happening while we + * are measuring stuff + */ + key = irq_lock(); + faults = z_num_pagefaults_get(); + ret = k_mem_page_out(arena, HALF_BYTES); + zassert_equal(ret, 0, "k_mem_page_out failed with %d", ret); + + /* Write to the supposedly evicted region */ + for (size_t i = 0; i < HALF_BYTES; i++) { + arena[i] = nums[i % 10]; + } + faults = z_num_pagefaults_get() - faults; + irq_unlock(key); + + zassert_equal(faults, HALF_PAGES, + "unexpected num pagefaults expected %lu got %d", + HALF_PAGES, faults); + + ret = k_mem_page_out(arena, arena_size); + zassert_equal(ret, -ENOMEM, "k_mem_page_out should have failed"); + +} + +void test_k_mem_page_in(void) +{ + unsigned long faults; + int key, ret; + + /* Lock IRQs to prevent other pagefaults from happening while we + * are measuring stuff + */ + key = irq_lock(); + + ret = k_mem_page_out(arena, HALF_BYTES); + zassert_equal(ret, 0, "k_mem_page_out failed with %d", ret); + + k_mem_page_in(arena, HALF_BYTES); + + faults = z_num_pagefaults_get(); + /* Write to the supposedly evicted region */ + for (size_t i = 0; i < HALF_BYTES; i++) { + arena[i] = nums[i % 10]; + } + faults = z_num_pagefaults_get() - faults; + irq_unlock(key); + + zassert_equal(faults, 0, "%d page faults when 0 expected", + faults); +} + +void test_k_mem_pin(void) +{ + unsigned long faults; + int key; + + k_mem_pin(arena, HALF_BYTES); + + /* Write to the rest of the arena */ + for (size_t i = HALF_BYTES; i < arena_size; i++) { + arena[i] = nums[i % 10]; + } + + key = irq_lock(); + /* Show no faults writing to the pinned area */ + faults = z_num_pagefaults_get(); + for (size_t i = 0; i < HALF_BYTES; i++) { + arena[i] = nums[i % 10]; + } + faults = z_num_pagefaults_get() - faults; + irq_unlock(key); + + zassert_equal(faults, 0, "%d page faults when 0 expected", + faults); + + /* Clean up */ + k_mem_unpin(arena, HALF_BYTES); +} + +void test_k_mem_unpin(void) +{ + /* Pin the memory (which we know works from prior test) */ + k_mem_pin(arena, HALF_BYTES); + + /* Now un-pin it */ + k_mem_unpin(arena, HALF_BYTES); + + /* repeat the page_out scenario, which should work */ + test_k_mem_page_out(); +} + +/* Show that even if we map enough anonymous memory to fill the backing + * store, we can still handle pagefaults. + * This eats up memory so should be last in the suite. + */ +void test_backing_store_capacity(void) +{ + char *mem, *ret; + int key; + unsigned long faults; + size_t size = (((CONFIG_BACKING_STORE_RAM_PAGES - 1) - HALF_PAGES) * + CONFIG_MMU_PAGE_SIZE); + + /* Consume the rest of memory */ + mem = k_mem_map(size, K_MEM_PERM_RW); + zassert_not_null(mem, "k_mem_map failed"); + + /* Show no memory is left */ + ret = k_mem_map(CONFIG_MMU_PAGE_SIZE, K_MEM_PERM_RW); + zassert_is_null(ret, "k_mem_map shouldn't have succeeded"); + + key = irq_lock(); + faults = z_num_pagefaults_get(); + /* Poke all anonymous memory */ + for (size_t i = 0; i < HALF_BYTES; i++) { + arena[i] = nums[i % 10]; + } + for (size_t i = 0; i < size; i++) { + mem[i] = nums[i % 10]; + } + faults = z_num_pagefaults_get() - faults; + irq_unlock(key); + + zassert_not_equal(faults, 0, "should have had some pagefaults"); +} + +/* ztest main entry*/ +void test_main(void) +{ + ztest_test_suite(test_demand_paging, + ztest_unit_test(test_map_anon_pages), + ztest_unit_test(test_touch_anon_pages), + ztest_unit_test(test_k_mem_page_out), + ztest_unit_test(test_k_mem_page_in), + ztest_unit_test(test_k_mem_pin), + ztest_unit_test(test_k_mem_unpin), + ztest_unit_test(test_backing_store_capacity)); + + ztest_run_test_suite(test_demand_paging); +} diff --git a/tests/kernel/mem_protect/demand_paging/testcase.yaml b/tests/kernel/mem_protect/demand_paging/testcase.yaml new file mode 100644 index 000000000000..72917e8d6886 --- /dev/null +++ b/tests/kernel/mem_protect/demand_paging/testcase.yaml @@ -0,0 +1,4 @@ +tests: + kernel.memory_protection.demand_paging: + tags: kernel mmu demand_paging ignore_faults + filter: CONFIG_DEMAND_PAGING diff --git a/tests/kernel/mem_protect/mem_map/src/main.c b/tests/kernel/mem_protect/mem_map/src/main.c index 065ed374ae8e..75881272304f 100644 --- a/tests/kernel/mem_protect/mem_map/src/main.c +++ b/tests/kernel/mem_protect/mem_map/src/main.c @@ -165,13 +165,62 @@ void test_z_phys_map_side_effect(void) ztest_test_fail(); } +/** + * Basic k_mem_map() functionality + * + * Does not exercise K_MEM_MAP_* control flags, just default behavior + */ +void test_k_mem_map(void) +{ + size_t free_mem, free_mem_after_map; + char *mapped; + int i; + + free_mem = k_mem_free_get(); + zassert_not_equal(free_mem, 0, "no free memory"); + printk("Free memory: %zu\n", free_mem); + + mapped = k_mem_map(CONFIG_MMU_PAGE_SIZE, K_MEM_PERM_RW); + zassert_not_null(mapped, "failed to map memory"); + printk("mapped a page to %p\n", mapped); + + /* Page should be zeroed */ + for (i = 0; i < CONFIG_MMU_PAGE_SIZE; i++) { + zassert_equal(mapped[i], '\x00', "page not zeroed"); + } + + free_mem_after_map = k_mem_free_get(); + printk("Free memory now: %zu\n", free_mem_after_map); + zassert_equal(free_mem, free_mem_after_map + CONFIG_MMU_PAGE_SIZE, + "incorrect free memory accounting"); + + /* Show we can write to page without exploding */ + (void)memset(mapped, '\xFF', CONFIG_MMU_PAGE_SIZE); + for (i = 0; i < CONFIG_MMU_PAGE_SIZE; i++) { + zassert_true(mapped[i] == '\xFF', + "incorrect value 0x%hhx read at index %d", + mapped[i], i); + } + + /* TODO: Un-map the mapped page to clean up, once we have that + * capability + */ +} + /* ztest main entry*/ void test_main(void) { +#ifdef CONFIG_DEMAND_PAGING + /* This test sets up multiple mappings of RAM pages, which is only + * allowed for pinned memory + */ + k_mem_pin(test_page, sizeof(test_page)); +#endif ztest_test_suite(test_mem_map, ztest_unit_test(test_z_phys_map_rw), ztest_unit_test(test_z_phys_map_exec), - ztest_unit_test(test_z_phys_map_side_effect) + ztest_unit_test(test_z_phys_map_side_effect), + ztest_unit_test(test_k_mem_map) ); ztest_run_test_suite(test_mem_map); } diff --git a/tests/kernel/mem_protect/syscalls/CMakeLists.txt b/tests/kernel/mem_protect/syscalls/CMakeLists.txt index a34f3f7a8375..55684803bba1 100644 --- a/tests/kernel/mem_protect/syscalls/CMakeLists.txt +++ b/tests/kernel/mem_protect/syscalls/CMakeLists.txt @@ -4,5 +4,10 @@ cmake_minimum_required(VERSION 3.13.1) find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) project(syscalls) +target_include_directories(app PRIVATE + ${ZEPHYR_BASE}/kernel/include + ${ZEPHYR_BASE}/arch/${ARCH}/include + ) + FILE(GLOB app_sources src/*.c) target_sources(app PRIVATE ${app_sources}) diff --git a/tests/kernel/mem_protect/syscalls/src/main.c b/tests/kernel/mem_protect/syscalls/src/main.c index 83d88766cc7a..c6d2cfef58aa 100644 --- a/tests/kernel/mem_protect/syscalls/src/main.c +++ b/tests/kernel/mem_protect/syscalls/src/main.c @@ -7,7 +7,9 @@ #include #include #include +#include #include "test_syscalls.h" +#include #define BUF_SIZE 32 #define SLEEP_MS_LONG 15000 @@ -16,8 +18,8 @@ || defined(CONFIG_BOARD_NUCLEO_L073RZ) #define FAULTY_ADDRESS 0x0FFFFFFF #elif CONFIG_MMU -/* Just past the permanent RAM mapping should be a non-present page */ -#define FAULTY_ADDRESS (CONFIG_KERNEL_VM_BASE + CONFIG_KERNEL_RAM_SIZE) +/* Just past the zephyr image mapping should be a non-present page */ +#define FAULTY_ADDRESS Z_FREE_VM_START #else #define FAULTY_ADDRESS 0xFFFFFFF0 #endif