From 4a4409e1b25426e1117c5358f591822cab5c9a6b Mon Sep 17 00:00:00 2001 From: mintsuki Date: Sat, 24 Sep 2022 03:57:55 +0200 Subject: [PATCH] pmm: Undo most of what was done between 85603ec8 and 74a3a1c6 --- common/lib/misc.c | 96 ++++++++++++++++ common/mm/pmm.h | 7 +- common/mm/pmm.s2.c | 280 +++++++++++++++++++++++++++++---------------- 3 files changed, 284 insertions(+), 99 deletions(-) diff --git a/common/lib/misc.c b/common/lib/misc.c index 70ed9d6f..4ed97bfe 100644 --- a/common/lib/misc.c +++ b/common/lib/misc.c @@ -160,6 +160,102 @@ retry: #error Unknown architecture #endif + // Go through new EFI memmap and free up bootloader entries + size_t entry_count = efi_mmap_size / efi_desc_size; + + EFI_MEMORY_DESCRIPTOR *efi_copy = ext_mem_alloc(EFI_COPY_MAX_ENTRIES * efi_desc_size); + size_t efi_copy_i = 0; + + for (size_t i = 0; i < entry_count; i++) { + EFI_MEMORY_DESCRIPTOR *orig_entry = (void *)efi_mmap + i * efi_desc_size; + EFI_MEMORY_DESCRIPTOR *new_entry = (void *)efi_copy + efi_copy_i * efi_desc_size; + + memcpy(new_entry, orig_entry, efi_desc_size); + + uint64_t base = orig_entry->PhysicalStart; + uint64_t length = orig_entry->NumberOfPages * 4096; + uint64_t top = base + length; + + // Find for a match in the untouched memory map + for (size_t j = 0; j < untouched_memmap_entries; j++) { + if (untouched_memmap[j].type != MEMMAP_USABLE) + continue; + + if (top > untouched_memmap[j].base && top <= untouched_memmap[j].base + untouched_memmap[j].length) { + if (untouched_memmap[j].base < base) { + new_entry->NumberOfPages = (base - untouched_memmap[j].base) / 4096; + + efi_copy_i++; + if (efi_copy_i == EFI_COPY_MAX_ENTRIES) { + panic(false, "efi: New memory map exhausted"); + } + new_entry = (void *)efi_copy + efi_copy_i * efi_desc_size; + memcpy(new_entry, orig_entry, efi_desc_size); + + new_entry->NumberOfPages -= (base - untouched_memmap[j].base) / 4096; + new_entry->PhysicalStart = base; + new_entry->VirtualStart = new_entry->PhysicalStart; + + length = new_entry->NumberOfPages * 4096; + top = base + length; + } + + if (untouched_memmap[j].base > base) { + new_entry->NumberOfPages = (untouched_memmap[j].base - base) / 4096; + + efi_copy_i++; + if (efi_copy_i == EFI_COPY_MAX_ENTRIES) { + panic(false, "efi: New memory map exhausted"); + } + new_entry = (void *)efi_copy + efi_copy_i * efi_desc_size; + memcpy(new_entry, orig_entry, efi_desc_size); + + new_entry->NumberOfPages -= (untouched_memmap[j].base - base) / 4096; + new_entry->PhysicalStart = untouched_memmap[j].base; + new_entry->VirtualStart = new_entry->PhysicalStart; + + base = new_entry->PhysicalStart; + length = new_entry->NumberOfPages * 4096; + top = base + length; + } + + if (length < untouched_memmap[j].length) { + panic(false, "efi: Memory map corruption"); + } + + new_entry->Type = EfiConventionalMemory; + + if (length == untouched_memmap[j].length) { + // It's a perfect match! + break; + } + + new_entry->NumberOfPages = untouched_memmap[j].length / 4096; + + efi_copy_i++; + if (efi_copy_i == EFI_COPY_MAX_ENTRIES) { + panic(false, "efi: New memory map exhausted"); + } + new_entry = (void *)efi_copy + efi_copy_i * efi_desc_size; + memcpy(new_entry, orig_entry, efi_desc_size); + + new_entry->NumberOfPages = (length - untouched_memmap[j].length) / 4096; + new_entry->PhysicalStart = base + untouched_memmap[j].length; + new_entry->VirtualStart = new_entry->PhysicalStart; + + break; + } + } + + efi_copy_i++; + if (efi_copy_i == EFI_COPY_MAX_ENTRIES) { + panic(false, "efi: New memory map exhausted"); + } + } + + efi_mmap = efi_copy; + efi_mmap_size = efi_copy_i * efi_desc_size; + efi_boot_services_exited = true; printv("efi: Exited boot services.\n"); diff --git a/common/mm/pmm.h b/common/mm/pmm.h index dc12315a..55421ae0 100644 --- a/common/mm/pmm.h +++ b/common/mm/pmm.h @@ -21,6 +21,7 @@ struct memmap_entry { #define MEMMAP_KERNEL_AND_MODULES 0x1001 #define MEMMAP_FRAMEBUFFER 0x1002 #define MEMMAP_EFI_RECLAIMABLE 0x2000 +#define MEMMAP_EFI_BOOTSERVICES 0x2001 struct meminfo { size_t uppermem; @@ -37,6 +38,9 @@ extern size_t memmap_entries; #if defined (UEFI) extern struct memmap_entry *memmap; extern size_t memmap_entries; + +extern struct memmap_entry *untouched_memmap; +extern size_t untouched_memmap_entries; #endif extern bool allocations_disallowed; @@ -45,8 +49,6 @@ void init_memmap(void); struct memmap_entry *get_memmap(size_t *entries); struct memmap_entry *get_raw_memmap(size_t *entry_count); void print_memmap(struct memmap_entry *mm, size_t size); -bool memmap_alloc_range_in(struct memmap_entry *m, size_t *_count, - uint64_t base, uint64_t length, uint32_t type, uint32_t overlay_type, bool do_panic, bool simulation, bool new_entry); bool memmap_alloc_range(uint64_t base, uint64_t length, uint32_t type, uint32_t overlay_type, bool panic, bool simulation, bool new_entry); void pmm_randomise_memory(void); @@ -59,6 +61,7 @@ void *conv_mem_alloc(size_t count); void pmm_free(void *ptr, size_t length); #if defined (UEFI) +void pmm_reclaim_uefi_mem(void); void pmm_release_uefi_mem(void); #endif diff --git a/common/mm/pmm.s2.c b/common/mm/pmm.s2.c index cb690cad..c2936930 100644 --- a/common/mm/pmm.s2.c +++ b/common/mm/pmm.s2.c @@ -60,8 +60,8 @@ static size_t memmap_max_entries; struct memmap_entry *memmap; size_t memmap_entries = 0; -static struct memmap_entry *untouched_memmap; -static size_t untouched_memmap_entries = 0; +struct memmap_entry *untouched_memmap; +size_t untouched_memmap_entries = 0; #endif static const char *memmap_type(uint32_t type) { @@ -84,6 +84,8 @@ static const char *memmap_type(uint32_t type) { return "Kernel/Modules"; case MEMMAP_EFI_RECLAIMABLE: return "EFI reclaimable"; + case MEMMAP_EFI_BOOTSERVICES: + return "EFI boot services"; default: return "???"; } @@ -116,6 +118,10 @@ static bool align_entry(uint64_t *base, uint64_t *length) { return true; } +static bool sanitiser_keep_first_page = false; + +#define MEMMAP_DROP_LATER ((uint32_t)-1) + static void sanitise_entries(struct memmap_entry *m, size_t *_count, bool align_entries) { size_t count = *_count; @@ -138,6 +144,14 @@ static void sanitise_entries(struct memmap_entry *m, size_t *_count, bool align_ if ( (res_base >= base && res_base < top) && (res_top >= base && res_top < top) ) { + // Drop the entry entirely if usable + if (m[j].type == MEMMAP_USABLE) { + m[j].type = MEMMAP_DROP_LATER; + } + if (m[j].type == MEMMAP_DROP_LATER) { + continue; + } + // TODO actually handle splitting off usable chunks panic(false, "A non-usable memory map entry is inside a usable section."); } @@ -162,12 +176,22 @@ static void sanitise_entries(struct memmap_entry *m, size_t *_count, bool align_ } } + // Collect "drop later" entries + for (size_t i = 0; i < count; i++) { + if (m[i].type != MEMMAP_DROP_LATER) { + continue; + } + + m[i] = m[count - 1]; + count--; i--; + } + // Remove 0 length usable entries and usable entries below 0x1000 for (size_t i = 0; i < count; i++) { if (m[i].type != MEMMAP_USABLE) continue; - if (m[i].base < 0x1000) { + if (!sanitiser_keep_first_page && m[i].base < 0x1000) { if (m[i].base + m[i].length <= 0x1000) { goto del_mm1; } @@ -221,13 +245,9 @@ del_mm1: *_count = count; } -#if defined (UEFI) -static void pmm_reclaim_uefi_mem(struct memmap_entry *m, size_t *_count); -#endif - struct memmap_entry *get_memmap(size_t *entries) { #if defined (UEFI) - pmm_reclaim_uefi_mem(memmap, &memmap_entries); + pmm_reclaim_uefi_mem(); #endif sanitise_entries(memmap, &memmap_entries, true); @@ -336,7 +356,7 @@ void init_memmap(void) { our_type = MEMMAP_RESERVED; break; case EfiBootServicesCode: case EfiBootServicesData: - our_type = MEMMAP_EFI_RECLAIMABLE; break; + our_type = MEMMAP_EFI_BOOTSERVICES; break; case EfiACPIReclaimMemory: our_type = MEMMAP_ACPI_RECLAIMABLE; break; case EfiACPIMemoryNVS: @@ -416,83 +436,121 @@ fail: panic(false, "pmm: Failure initialising memory map"); } -static void pmm_reclaim_uefi_mem(struct memmap_entry *m, size_t *_count) { - size_t count = *_count; - +void pmm_reclaim_uefi_mem(void) { size_t recl_i = 0; - for (size_t i = 0; i < count; i++) { - if (m[i].type == MEMMAP_EFI_RECLAIMABLE) { + for (size_t i = 0; i < memmap_entries; i++) { + if (memmap[i].type == MEMMAP_EFI_RECLAIMABLE) { recl_i++; } } struct memmap_entry *recl = ext_mem_alloc(recl_i * sizeof(struct memmap_entry)); - for (size_t i = 0, j = 0; i < count; i++) { - if (m[i].type == MEMMAP_EFI_RECLAIMABLE) { - recl[j++] = m[i]; + { + size_t recl_j = 0; + for (size_t i = 0; i < memmap_entries; i++) { + if (memmap[i].type == MEMMAP_EFI_RECLAIMABLE) { + recl[recl_j++] = memmap[i]; + } } } - for (size_t ri = 0; ri < recl_i; ri++) { - struct memmap_entry *r = &recl[ri]; +another_recl:; + // Punch holes in our EFI reclaimable entry for every EFI area which is + // boot services or conventional that fits within + size_t efi_mmap_entry_count = efi_mmap_size / efi_desc_size; + for (size_t i = 0; i < efi_mmap_entry_count; i++) { + EFI_MEMORY_DESCRIPTOR *entry = (void *)efi_mmap + i * efi_desc_size; - // Punch holes in our EFI reclaimable entry for every EFI area which is - // boot services or conventional that fits within - size_t efi_mmap_entry_count = efi_mmap_size / efi_desc_size; - for (size_t i = 0; i < efi_mmap_entry_count; i++) { - EFI_MEMORY_DESCRIPTOR *entry = (void *)efi_mmap + i * efi_desc_size; + uint64_t base = recl->base; + uint64_t top = base + recl->length; + uint64_t efi_base = entry->PhysicalStart; + uint64_t efi_size = entry->NumberOfPages * 4096; - uint64_t base = r->base; - uint64_t top = base + r->length; - uint64_t efi_base = entry->PhysicalStart; - uint64_t efi_size = entry->NumberOfPages * 4096; - - if (efi_base < base) { - if (efi_size <= base - efi_base) - continue; - efi_size -= base - efi_base; - efi_base = base; - } - - uint64_t efi_top = efi_base + efi_size; - - if (efi_top > top) { - if (efi_size <= efi_top - top) - continue; - efi_size -= efi_top - top; - efi_top = top; - } - - // Sanity check - if (!(efi_base >= base && efi_base < top - && efi_top > base && efi_top <= top)) + if (efi_base < base) { + if (efi_size <= base - efi_base) continue; + efi_size -= base - efi_base; + efi_base = base; + } + + uint64_t efi_top = efi_base + efi_size; + + if (efi_top > top) { + if (efi_size <= efi_top - top) + continue; + efi_size -= efi_top - top; + efi_top = top; + } + + // Sanity check + if (!(efi_base >= base && efi_base < top + && efi_top > base && efi_top <= top)) + continue; + + uint32_t our_type; + switch (entry->Type) { + case EfiBootServicesCode: + case EfiBootServicesData: + case EfiConventionalMemory: + our_type = MEMMAP_USABLE; break; + case EfiACPIReclaimMemory: + our_type = MEMMAP_ACPI_RECLAIMABLE; break; + case EfiACPIMemoryNVS: + our_type = MEMMAP_ACPI_NVS; break; + default: + our_type = MEMMAP_RESERVED; break; + } + + memmap_alloc_range(efi_base, efi_size, our_type, false, true, false, true); + } + + if (--recl_i > 0) { + recl++; + goto another_recl; + } + + // Ensure the boot services are still boot services, or free, in + // the EFI memmap, and disallow allocations since our stack and page tables + // are placed in this newly freed memory. + for (size_t i = 0; i < memmap_entries; i++) { + if (memmap[i].type != MEMMAP_EFI_BOOTSERVICES) + continue; + + // Go through EFI memmap and ensure this entry fits within a boot services + // or conventional entry + size_t entry_count = efi_mmap_size / efi_desc_size; + + for (size_t j = 0; j < entry_count; j++) { + EFI_MEMORY_DESCRIPTOR *entry = (void *)efi_mmap + j * efi_desc_size; - uint32_t our_type; switch (entry->Type) { case EfiBootServicesCode: case EfiBootServicesData: case EfiConventionalMemory: - our_type = MEMMAP_USABLE; break; - case EfiACPIReclaimMemory: - our_type = MEMMAP_ACPI_RECLAIMABLE; break; - case EfiACPIMemoryNVS: - our_type = MEMMAP_ACPI_NVS; break; + break; default: - our_type = MEMMAP_RESERVED; break; + continue; } - memmap_alloc_range_in(m, &count, efi_base, efi_size, our_type, 0, true, false, false); + uintptr_t base = memmap[i].base; + uintptr_t top = base + memmap[i].length; + uintptr_t efi_base = entry->PhysicalStart; + uintptr_t efi_size = entry->NumberOfPages * 4096; + uintptr_t efi_top = efi_base + efi_size; + + if (!(base >= efi_base && base < efi_top + && top > efi_base && top <= efi_top)) + continue; + + memmap[i].type = MEMMAP_USABLE; } } allocations_disallowed = true; - sanitise_entries(m, &count, false); - - *_count = count; + sanitise_entries(memmap, &memmap_entries, false); } void pmm_release_uefi_mem(void) { @@ -524,9 +582,49 @@ struct memmap_entry *get_raw_memmap(size_t *entry_count) { #if defined (UEFI) struct memmap_entry *get_raw_memmap(size_t *entry_count) { - pmm_reclaim_uefi_mem(untouched_memmap, &untouched_memmap_entries); - *entry_count = untouched_memmap_entries; - return untouched_memmap; + size_t mmap_count = efi_mmap_size / efi_desc_size; + size_t mmap_len = mmap_count * sizeof(struct memmap_entry); + + struct memmap_entry *mmap = ext_mem_alloc(mmap_len); + + for (size_t i = 0; i < mmap_count; i++) { + EFI_MEMORY_DESCRIPTOR *entry = (void *)efi_mmap + i * efi_desc_size; + + uint32_t our_type; + switch (entry->Type) { + case EfiReservedMemoryType: + case EfiRuntimeServicesCode: + case EfiRuntimeServicesData: + case EfiUnusableMemory: + case EfiMemoryMappedIO: + case EfiMemoryMappedIOPortSpace: + case EfiPalCode: + case EfiLoaderCode: + case EfiLoaderData: + default: + our_type = MEMMAP_RESERVED; break; + case EfiACPIReclaimMemory: + our_type = MEMMAP_ACPI_RECLAIMABLE; break; + case EfiACPIMemoryNVS: + our_type = MEMMAP_ACPI_NVS; break; + case EfiBootServicesCode: + case EfiBootServicesData: + case EfiConventionalMemory: + our_type = MEMMAP_USABLE; break; + } + + mmap[i].base = entry->PhysicalStart; + mmap[i].length = entry->NumberOfPages * 4096; + mmap[i].type = our_type; + } + + bool s_old = sanitiser_keep_first_page; + sanitiser_keep_first_page = true; + sanitise_entries(mmap, &mmap_count, false); + sanitiser_keep_first_page = s_old; + + *entry_count = mmap_count; + return mmap; } #endif @@ -617,22 +715,19 @@ struct meminfo mmap_get_info(size_t mmap_count, struct memmap_entry *mmap) { return info; } -static bool pmm_new_entry(struct memmap_entry *m, size_t *_count, - uint64_t base, uint64_t length, uint32_t type) { - size_t count = *_count; - +static bool pmm_new_entry(uint64_t base, uint64_t length, uint32_t type) { uint64_t top = base + length; // Handle overlapping new entries. - for (size_t i = 0; i < count; i++) { - uint64_t entry_base = m[i].base; - uint64_t entry_top = m[i].base + m[i].length; + for (size_t i = 0; i < memmap_entries; i++) { + uint64_t entry_base = memmap[i].base; + uint64_t entry_top = memmap[i].base + memmap[i].length; // Full overlap if (base <= entry_base && top >= entry_top) { // Remove overlapped entry - m[i] = m[count - 1]; - count--; + memmap[i] = memmap[memmap_entries - 1]; + memmap_entries--; i--; continue; } @@ -640,30 +735,30 @@ static bool pmm_new_entry(struct memmap_entry *m, size_t *_count, // Partial overlap (bottom) if (base <= entry_base && top < entry_top && top > entry_base) { // Entry gets bottom shaved off - m[i].base += top - entry_base; - m[i].length -= top - entry_base; + memmap[i].base += top - entry_base; + memmap[i].length -= top - entry_base; continue; } // Partial overlap (top) if (base > entry_base && base < entry_top && top >= entry_top) { // Entry gets top shaved off - m[i].length -= entry_top - base; + memmap[i].length -= entry_top - base; continue; } // Nested (pain) if (base > entry_base && top < entry_top) { // Entry gets top shaved off first - m[i].length -= entry_top - base; + memmap[i].length -= entry_top - base; // Now we need to create a new entry - if (count >= memmap_max_entries) + if (memmap_entries >= memmap_max_entries) panic(false, "Memory map exhausted."); - struct memmap_entry *new_entry = &m[count++]; + struct memmap_entry *new_entry = &memmap[memmap_entries++]; - new_entry->type = m[i].type; + new_entry->type = memmap[i].type; new_entry->base = top; new_entry->length = entry_top - top; @@ -671,23 +766,19 @@ static bool pmm_new_entry(struct memmap_entry *m, size_t *_count, } } - if (count >= memmap_max_entries) + if (memmap_entries >= memmap_max_entries) panic(false, "Memory map exhausted."); - struct memmap_entry *target = &m[count++]; + struct memmap_entry *target = &memmap[memmap_entries++]; target->type = type; target->base = base; target->length = length; - *_count = count; return true; } -bool memmap_alloc_range_in(struct memmap_entry *m, size_t *_count, - uint64_t base, uint64_t length, uint32_t type, uint32_t overlay_type, bool do_panic, bool simulation, bool new_entry) { - size_t count = *_count; - +bool memmap_alloc_range(uint64_t base, uint64_t length, uint32_t type, uint32_t overlay_type, bool do_panic, bool simulation, bool new_entry) { if (length == 0) return true; @@ -697,18 +788,18 @@ bool memmap_alloc_range_in(struct memmap_entry *m, size_t *_count, uint64_t top = base + length; - for (size_t i = 0; i < count; i++) { - if (overlay_type != 0 && m[i].type != overlay_type) + for (size_t i = 0; i < memmap_entries; i++) { + if (overlay_type != 0 && memmap[i].type != overlay_type) continue; - uint64_t entry_base = m[i].base; - uint64_t entry_top = m[i].base + m[i].length; + uint64_t entry_base = memmap[i].base; + uint64_t entry_top = memmap[i].base + memmap[i].length; if (base >= entry_base && base < entry_top && top <= entry_top) { if (simulation) return true; - if (pmm_new_entry(m, &count, base, length, type) == true) { + if (pmm_new_entry(base, length, type) == true) { goto success; } } @@ -721,16 +812,11 @@ bool memmap_alloc_range_in(struct memmap_entry *m, size_t *_count, return false; } - if (pmm_new_entry(m, &count, base, length, type) == false) { + if (pmm_new_entry(base, length, type) == false) { return false; } success: - sanitise_entries(m, &count, false); - *_count = count; + sanitise_entries(memmap, &memmap_entries, false); return true; } - -bool memmap_alloc_range(uint64_t base, uint64_t length, uint32_t type, uint32_t overlay_type, bool do_panic, bool simulation, bool new_entry) { - return memmap_alloc_range_in(memmap, &memmap_entries, base, length, type, overlay_type, do_panic, simulation, new_entry); -}