rulimine/common/mm/vmm.c

306 lines
10 KiB
C

#include <stdint.h>
#include <stddef.h>
#include <mm/vmm.h>
#include <mm/pmm.h>
#include <lib/blib.h>
#include <lib/print.h>
#include <sys/cpu.h>
#define PT_SIZE ((uint64_t)0x1000)
typedef uint64_t pt_entry_t;
static pt_entry_t *get_next_level(pagemap_t pagemap, pt_entry_t *current_level,
uint64_t virt, enum page_size desired_sz,
size_t level_idx, size_t entry);
#if defined (__x86_64__) || defined (__i386__)
#define PT_FLAG_VALID ((uint64_t)1 << 0)
#define PT_FLAG_WRITE ((uint64_t)1 << 1)
#define PT_FLAG_USER ((uint64_t)1 << 2)
#define PT_FLAG_LARGE ((uint64_t)1 << 7)
#define PT_FLAG_NX ((uint64_t)1 << 63)
#define PT_PADDR_MASK ((uint64_t)0x0000FFFFFFFFF000)
#define PT_TABLE_FLAGS (PT_FLAG_VALID | PT_FLAG_WRITE | PT_FLAG_USER)
#define PT_IS_TABLE(x) (((x) & (PT_FLAG_VALID | PT_FLAG_LARGE)) == PT_FLAG_VALID)
#define PT_IS_LARGE(x) (((x) & (PT_FLAG_VALID | PT_FLAG_LARGE)) == (PT_FLAG_VALID | PT_FLAG_LARGE))
#define PT_TO_VMM_FLAGS(x) ((x) & (PT_FLAG_WRITE | PT_FLAG_NX))
void vmm_assert_nx(void) {
uint32_t a, b, c, d;
if (!cpuid(0x80000001, 0, &a, &b, &c, &d) || !(d & (1 << 20))) {
panic(false, "vmm: NX functionality not available on this CPU.");
}
}
pagemap_t new_pagemap(int lv) {
pagemap_t pagemap;
pagemap.levels = lv;
pagemap.top_level = ext_mem_alloc(PT_SIZE);
return pagemap;
}
static bool is_1gib_page_supported(void) {
// Cache the cpuid result :^)
static bool CACHE_INIT = false;
static bool CACHE = false;
if (!CACHE_INIT) {
// Check if 1GiB pages are supported:
uint32_t eax, ebx, ecx, edx;
CACHE = cpuid(0x80000001, 0, &eax, &ebx, &ecx, &edx) && ((edx & 1 << 26) == 1 << 26);
CACHE_INIT = true;
printv("paging: 1GiB pages are %s!\n", CACHE ? "supported" : "not supported");
}
return CACHE;
}
void map_page(pagemap_t pagemap, uint64_t virt_addr, uint64_t phys_addr, uint64_t flags, enum page_size pg_size) {
// Calculate the indices in the various tables using the virtual address
size_t pml5_entry = (virt_addr & ((uint64_t)0x1ff << 48)) >> 48;
size_t pml4_entry = (virt_addr & ((uint64_t)0x1ff << 39)) >> 39;
size_t pml3_entry = (virt_addr & ((uint64_t)0x1ff << 30)) >> 30;
size_t pml2_entry = (virt_addr & ((uint64_t)0x1ff << 21)) >> 21;
size_t pml1_entry = (virt_addr & ((uint64_t)0x1ff << 12)) >> 12;
pt_entry_t *pml5, *pml4, *pml3, *pml2, *pml1;
flags |= PT_FLAG_VALID; // Always present
// Paging levels
switch (pagemap.levels) {
case 5:
pml5 = pagemap.top_level;
goto level5;
case 4:
pml4 = pagemap.top_level;
goto level4;
default:
__builtin_unreachable();
}
level5:
pml4 = get_next_level(pagemap, pml5, virt_addr, pg_size, 4, pml5_entry);
level4:
pml3 = get_next_level(pagemap, pml4, virt_addr, pg_size, 3, pml4_entry);
if (pg_size == Size1GiB) {
// Check if 1GiB pages are avaliable.
if (is_1gib_page_supported()) {
pml3[pml3_entry] = (pt_entry_t)(phys_addr | flags | PT_FLAG_LARGE);
} else {
// If 1GiB pages are not supported then emulate it by splitting them into
// 2MiB pages.
for (uint64_t i = 0; i < 0x40000000; i += 0x200000) {
map_page(pagemap, virt_addr + i, phys_addr + i, flags, Size2MiB);
}
}
return;
}
pml2 = get_next_level(pagemap, pml3, virt_addr, pg_size, 2, pml3_entry);
if (pg_size == Size2MiB) {
pml2[pml2_entry] = (pt_entry_t)(phys_addr | flags | PT_FLAG_LARGE);
return;
}
pml1 = get_next_level(pagemap, pml2, virt_addr, pg_size, 1, pml2_entry);
pml1[pml1_entry] = (pt_entry_t)(phys_addr | flags);
}
#elif defined (__aarch64__)
// Here we operate under the assumption that 4K pages are supported by the CPU.
// This appears to be guaranteed by UEFI, as section 2.3.6 "AArch64 Platforms"
// states that the primary processor core configuration includes 4K translation
// granules (TCR_EL1.TG0 = 0).
// Support for 4K pages also implies 2M, 1G and 512G blocks.
// Sanity check that 4K pages are supported.
void vmm_assert_4k_pages(void) {
uint64_t aa64mmfr0;
asm volatile ("mrs %0, id_aa64mmfr0_el1" : "=r"(aa64mmfr0));
if (((aa64mmfr0 >> 28) & 0b1111) == 0b1111) {
panic(false, "vmm: CPU does not support 4K pages, please make a bug report about this.");
}
}
#define PT_FLAG_VALID ((uint64_t)1 << 0)
#define PT_FLAG_TABLE ((uint64_t)1 << 1)
#define PT_FLAG_4K_PAGE ((uint64_t)1 << 1)
#define PT_FLAG_BLOCK ((uint64_t)0 << 1)
#define PT_FLAG_USER ((uint64_t)1 << 6)
#define PT_FLAG_READONLY ((uint64_t)1 << 7)
#define PT_FLAG_INNER_SH ((uint64_t)3 << 8)
#define PT_FLAG_ACCESS ((uint64_t)1 << 10)
#define PT_FLAG_XN ((uint64_t)1 << 54)
#define PT_FLAG_WB ((uint64_t)0 << 2)
#define PT_FLAG_FB ((uint64_t)1 << 2)
#define PT_PADDR_MASK ((uint64_t)0x0000FFFFFFFFF000)
#define PT_TABLE_FLAGS (PT_FLAG_VALID | PT_FLAG_TABLE)
#define PT_IS_TABLE(x) (((x) & (PT_FLAG_VALID | PT_FLAG_TABLE)) == (PT_FLAG_VALID | PT_FLAG_TABLE))
#define PT_IS_LARGE(x) (((x) & (PT_FLAG_VALID | PT_FLAG_TABLE)) == PT_FLAG_VALID)
#define PT_TO_VMM_FLAGS(x) (pt_to_vmm_flags_internal(x))
static uint64_t pt_to_vmm_flags_internal(pt_entry_t entry) {
uint64_t flags = 0;
if (!(entry & PT_FLAG_READONLY))
flags |= VMM_FLAG_WRITE;
if (entry & PT_FLAG_XN)
flags |= VMM_FLAG_NOEXEC;
if (entry & PT_FLAG_FB)
flags |= VMM_FLAG_FB;
return flags;
}
pagemap_t new_pagemap(int lv) {
pagemap_t pagemap;
pagemap.levels = lv;
pagemap.top_level[0] = ext_mem_alloc(PT_SIZE);
pagemap.top_level[1] = ext_mem_alloc(PT_SIZE);
return pagemap;
}
void map_page(pagemap_t pagemap, uint64_t virt_addr, uint64_t phys_addr, uint64_t flags, enum page_size pg_size) {
// Calculate the indices in the various tables using the virtual address
size_t pml5_entry = (virt_addr & ((uint64_t)0xf << 48)) >> 48;
size_t pml4_entry = (virt_addr & ((uint64_t)0x1ff << 39)) >> 39;
size_t pml3_entry = (virt_addr & ((uint64_t)0x1ff << 30)) >> 30;
size_t pml2_entry = (virt_addr & ((uint64_t)0x1ff << 21)) >> 21;
size_t pml1_entry = (virt_addr & ((uint64_t)0x1ff << 12)) >> 12;
pt_entry_t *pml5, *pml4, *pml3, *pml2, *pml1;
bool is_higher_half = virt_addr & ((uint64_t)1 << 63);
uint64_t real_flags = PT_FLAG_VALID | PT_FLAG_INNER_SH | PT_FLAG_ACCESS | PT_FLAG_WB;
if (!(flags & VMM_FLAG_WRITE))
real_flags |= PT_FLAG_READONLY;
if (flags & VMM_FLAG_NOEXEC)
real_flags |= PT_FLAG_XN;
if (flags & VMM_FLAG_FB)
real_flags |= PT_FLAG_FB;
// Paging levels
switch (pagemap.levels) {
case 5:
pml5 = pagemap.top_level[is_higher_half];
goto level5;
case 4:
pml4 = pagemap.top_level[is_higher_half];
goto level4;
default:
__builtin_unreachable();
}
level5:
pml4 = get_next_level(pagemap, pml5, virt_addr, pg_size, 4, pml5_entry);
level4:
pml3 = get_next_level(pagemap, pml4, virt_addr, pg_size, 3, pml4_entry);
if (pg_size == Size1GiB) {
pml3[pml3_entry] = (pt_entry_t)(phys_addr | real_flags | PT_FLAG_BLOCK);
return;
}
pml2 = get_next_level(pagemap, pml3, virt_addr, pg_size, 2, pml3_entry);
if (pg_size == Size2MiB) {
pml2[pml2_entry] = (pt_entry_t)(phys_addr | real_flags | PT_FLAG_BLOCK);
return;
}
pml1 = get_next_level(pagemap, pml2, virt_addr, pg_size, 1, pml2_entry);
pml1[pml1_entry] = (pt_entry_t)(phys_addr | real_flags | PT_FLAG_4K_PAGE);
}
#else
#error Unknown architecture
#endif
static pt_entry_t *get_next_level(pagemap_t pagemap, pt_entry_t *current_level,
uint64_t virt, enum page_size desired_sz,
size_t level_idx, size_t entry) {
pt_entry_t *ret;
if (PT_IS_TABLE(current_level[entry])) {
ret = (pt_entry_t *)(size_t)(current_level[entry] & PT_PADDR_MASK);
} else {
if (PT_IS_LARGE(current_level[entry])) {
// We are replacing an existing large page with a smaller page.
// Split the previous mapping into mappings of the newly requested size
// before performing the requested map operation.
uint64_t old_page_size, new_page_size;
switch (level_idx) {
case 2:
old_page_size = 0x40000000;
break;
case 1:
old_page_size = 0x200000;
break;
default:
panic(false, "Unexpected level in get_next_level");
}
switch (desired_sz) {
case Size1GiB:
new_page_size = 0x40000000;
break;
case Size2MiB:
new_page_size = 0x200000;
break;
case Size4KiB:
new_page_size = 0x1000;
break;
default:
panic(false, "Unexpected page size in get_next_level");
}
// Save all the information from the old entry at this level
uint64_t old_flags = PT_TO_VMM_FLAGS(current_level[entry]);
uint64_t old_phys = current_level[entry] & PT_PADDR_MASK;
uint64_t old_virt = virt & ~(old_page_size - 1);
if (old_phys & (old_page_size - 1))
panic(false, "Unexpected page table entry address in get_next_level");
// Allocate a table for the next level
ret = ext_mem_alloc(PT_SIZE);
current_level[entry] = (pt_entry_t)(size_t)ret | PT_TABLE_FLAGS;
// Recreate the old mapping with smaller pages
for (uint64_t i = 0; i < old_page_size; i += new_page_size) {
map_page(pagemap, old_virt + i, old_phys + i, old_flags, desired_sz);
}
} else {
// Allocate a table for the next level
ret = ext_mem_alloc(PT_SIZE);
current_level[entry] = (pt_entry_t)(size_t)ret | PT_TABLE_FLAGS;
}
}
return ret;
}