From 1c7fda897e52591c6f2ada9a2a16f33e6e127b32 Mon Sep 17 00:00:00 2001 From: Ingo Weinhold Date: Tue, 8 Jun 2010 20:35:53 +0000 Subject: [PATCH] * Moved X86VMTranslationMap32Bit to its own source file. * Made all helper function operating on paging structures static methods of X86PagingMethod32Bit. git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@37064 a95241bf-73f2-0310-859d-f6bbb57e9c96 --- src/system/kernel/arch/x86/Jamfile | 1 + .../x86/paging/32bit/X86PagingMethod32Bit.cpp | 1138 ++--------------- .../x86/paging/32bit/X86PagingMethod32Bit.h | 100 ++ .../paging/32bit/X86VMTranslationMap32Bit.cpp | 943 ++++++++++++++ .../kernel/arch/x86/paging/32bit/paging.h | 42 - 5 files changed, 1142 insertions(+), 1082 deletions(-) create mode 100644 src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp diff --git a/src/system/kernel/arch/x86/Jamfile b/src/system/kernel/arch/x86/Jamfile index a15d462252..5694c0895f 100644 --- a/src/system/kernel/arch/x86/Jamfile +++ b/src/system/kernel/arch/x86/Jamfile @@ -52,6 +52,7 @@ KernelMergeObject kernel_arch_x86.o : # paging/32bit X86PagingMethod32Bit.cpp X86PagingStructures32Bit.cpp + X86VMTranslationMap32Bit.cpp x86_apic.cpp x86_hpet.cpp diff --git a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp index 5c39f82459..50164002c8 100644 --- a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp +++ b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp @@ -18,14 +18,8 @@ #include #include #include -#include -#include -#include -#include -#include -#include +#include #include -#include #include "paging/32bit/X86PagingStructures32Bit.h" #include "paging/32bit/X86VMTranslationMap32Bit.h" @@ -47,1034 +41,6 @@ using X86LargePhysicalPageMapper::PhysicalPageSlot; static const size_t kPageTableAlignment = 1024 * B_PAGE_SIZE; -// #pragma mark - - - -//! TODO: currently assumes this translation map is active -static status_t -early_query(addr_t va, phys_addr_t *_physicalAddress) -{ - X86PagingMethod32Bit* method = X86PagingMethod32Bit::Method(); - if ((method->PageHolePageDir()[VADDR_TO_PDENT(va)] & X86_PDE_PRESENT) - == 0) { - // no pagetable here - return B_ERROR; - } - - page_table_entry* pentry = method->PageHole() + va / B_PAGE_SIZE; - if ((*pentry & X86_PTE_PRESENT) == 0) { - // page mapping not valid - return B_ERROR; - } - - *_physicalAddress = *pentry & X86_PTE_ADDRESS_MASK; - return B_OK; -} - - -static inline uint32 -memory_type_to_pte_flags(uint32 memoryType) -{ - // ATM we only handle the uncacheable and write-through type explicitly. For - // all other types we rely on the MTRRs to be set up correctly. Since we set - // the default memory type to write-back and since the uncacheable type in - // the PTE overrides any MTRR attribute (though, as per the specs, that is - // not recommended for performance reasons), this reduces the work we - // actually *have* to do with the MTRRs to setting the remaining types - // (usually only write-combining for the frame buffer). - switch (memoryType) { - case B_MTR_UC: - return X86_PTE_CACHING_DISABLED | X86_PTE_WRITE_THROUGH; - - case B_MTR_WC: - // X86_PTE_WRITE_THROUGH would be closer, but the combination with - // MTRR WC is "implementation defined" for Pentium Pro/II. - return 0; - - case B_MTR_WT: - return X86_PTE_WRITE_THROUGH; - - case B_MTR_WP: - case B_MTR_WB: - default: - return 0; - } -} - - -static void -put_page_table_entry_in_pgtable(page_table_entry* entry, - phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType, - bool globalPage) -{ - page_table_entry page = (physicalAddress & X86_PTE_ADDRESS_MASK) - | X86_PTE_PRESENT | (globalPage ? X86_PTE_GLOBAL : 0) - | memory_type_to_pte_flags(memoryType); - - // if the page is user accessible, it's automatically - // accessible in kernel space, too (but with the same - // protection) - if ((attributes & B_USER_PROTECTION) != 0) { - page |= X86_PTE_USER; - if ((attributes & B_WRITE_AREA) != 0) - page |= X86_PTE_WRITABLE; - } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) - page |= X86_PTE_WRITABLE; - - // put it in the page table - *(volatile page_table_entry*)entry = page; -} - - -// #pragma mark - - - -void -x86_put_pgtable_in_pgdir(page_directory_entry *entry, - phys_addr_t pgtablePhysical, uint32 attributes) -{ - *entry = (pgtablePhysical & X86_PDE_ADDRESS_MASK) - | X86_PDE_PRESENT - | X86_PDE_WRITABLE - | X86_PDE_USER; - // TODO: we ignore the attributes of the page table - for compatibility - // with BeOS we allow having user accessible areas in the kernel address - // space. This is currently being used by some drivers, mainly for the - // frame buffer. Our current real time data implementation makes use of - // this fact, too. - // We might want to get rid of this possibility one day, especially if - // we intend to port it to a platform that does not support this. -} - - -void -x86_early_prepare_page_tables(page_table_entry* pageTables, addr_t address, - size_t size) -{ - memset(pageTables, 0, B_PAGE_SIZE * (size / (B_PAGE_SIZE * 1024))); - - // put the array of pgtables directly into the kernel pagedir - // these will be wired and kept mapped into virtual space to be easy to get - // to - { - addr_t virtualTable = (addr_t)pageTables; - - page_directory_entry* pageHolePageDir - = X86PagingMethod32Bit::Method()->PageHolePageDir(); - - for (size_t i = 0; i < (size / (B_PAGE_SIZE * 1024)); - i++, virtualTable += B_PAGE_SIZE) { - phys_addr_t physicalTable = 0; - early_query(virtualTable, &physicalTable); - page_directory_entry* entry = &pageHolePageDir[ - (address / (B_PAGE_SIZE * 1024)) + i]; - x86_put_pgtable_in_pgdir(entry, physicalTable, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - } - } -} - - -// #pragma mark - VM ops - - -X86VMTranslationMap32Bit::X86VMTranslationMap32Bit() - : - fPagingStructures(NULL) -{ -} - - -X86VMTranslationMap32Bit::~X86VMTranslationMap32Bit() -{ - if (fPagingStructures == NULL) - return; - - if (fPageMapper != NULL) - fPageMapper->Delete(); - - if (fPagingStructures->pgdir_virt != NULL) { - // cycle through and free all of the user space pgtables - for (uint32 i = VADDR_TO_PDENT(USER_BASE); - i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) { - if ((fPagingStructures->pgdir_virt[i] & X86_PDE_PRESENT) != 0) { - addr_t address = fPagingStructures->pgdir_virt[i] - & X86_PDE_ADDRESS_MASK; - vm_page* page = vm_lookup_page(address / B_PAGE_SIZE); - if (!page) - panic("destroy_tmap: didn't find pgtable page\n"); - DEBUG_PAGE_ACCESS_START(page); - vm_page_set_state(page, PAGE_STATE_FREE); - } - } - } - - fPagingStructures->RemoveReference(); -} - - -status_t -X86VMTranslationMap32Bit::Init(bool kernel) -{ - TRACE("X86VMTranslationMap32Bit::Init()\n"); - - X86VMTranslationMap::Init(kernel); - - fPagingStructures = new(std::nothrow) X86PagingStructures32Bit; - if (fPagingStructures == NULL) - return B_NO_MEMORY; - - X86PagingMethod32Bit* method = X86PagingMethod32Bit::Method(); - - if (!kernel) { - // user - // allocate a physical page mapper - status_t error = method->PhysicalPageMapper() - ->CreateTranslationMapPhysicalPageMapper(&fPageMapper); - if (error != B_OK) - return error; - - // allocate the page directory - page_directory_entry* virtualPageDir = (page_directory_entry*)memalign( - B_PAGE_SIZE, B_PAGE_SIZE); - if (virtualPageDir == NULL) - return B_NO_MEMORY; - - // look up the page directory's physical address - phys_addr_t physicalPageDir; - vm_get_page_mapping(VMAddressSpace::KernelID(), - (addr_t)virtualPageDir, &physicalPageDir); - - fPagingStructures->Init(virtualPageDir, physicalPageDir, - method->KernelVirtualPageDirectory()); - } else { - // kernel - // get the physical page mapper - fPageMapper = method->KernelPhysicalPageMapper(); - - // we already know the kernel pgdir mapping - fPagingStructures->Init(method->KernelVirtualPageDirectory(), - method->KernelPhysicalPageDirectory(), NULL); - } - - return B_OK; -} - - -size_t -X86VMTranslationMap32Bit::MaxPagesNeededToMap(addr_t start, addr_t end) const -{ - // If start == 0, the actual base address is not yet known to the caller and - // we shall assume the worst case. - if (start == 0) { - // offset the range so it has the worst possible alignment - start = 1023 * B_PAGE_SIZE; - end += 1023 * B_PAGE_SIZE; - } - - return VADDR_TO_PDENT(end) + 1 - VADDR_TO_PDENT(start); -} - - -status_t -X86VMTranslationMap32Bit::Map(addr_t va, phys_addr_t pa, uint32 attributes, - uint32 memoryType, vm_page_reservation* reservation) -{ - TRACE("map_tmap: entry pa 0x%lx va 0x%lx\n", pa, va); - -/* - dprintf("pgdir at 0x%x\n", pgdir); - dprintf("index is %d\n", va / B_PAGE_SIZE / 1024); - dprintf("final at 0x%x\n", &pgdir[va / B_PAGE_SIZE / 1024]); - dprintf("value is 0x%x\n", *(int *)&pgdir[va / B_PAGE_SIZE / 1024]); - dprintf("present bit is %d\n", pgdir[va / B_PAGE_SIZE / 1024].present); - dprintf("addr is %d\n", pgdir[va / B_PAGE_SIZE / 1024].addr); -*/ - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - // check to see if a page table exists for this range - uint32 index = VADDR_TO_PDENT(va); - if ((pd[index] & X86_PDE_PRESENT) == 0) { - phys_addr_t pgtable; - vm_page *page; - - // we need to allocate a pgtable - page = vm_page_allocate_page(reservation, - PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); - - DEBUG_PAGE_ACCESS_END(page); - - pgtable = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE; - - TRACE("map_tmap: asked for free page for pgtable. 0x%lx\n", pgtable); - - // put it in the pgdir - x86_put_pgtable_in_pgdir(&pd[index], pgtable, attributes - | ((attributes & B_USER_PROTECTION) != 0 - ? B_WRITE_AREA : B_KERNEL_WRITE_AREA)); - - // update any other page directories, if it maps kernel space - if (index >= FIRST_KERNEL_PGDIR_ENT - && index < (FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS)) { - X86PagingStructures32Bit::UpdateAllPageDirs(index, pd[index]); - } - - fMapCount++; - } - - // now, fill in the pentry - struct thread* thread = thread_get_current_thread(); - ThreadCPUPinner pinner(thread); - - page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - index = VADDR_TO_PTENT(va); - - ASSERT_PRINT((pt[index] & X86_PTE_PRESENT) == 0, - "virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx32, va, - pt[index]); - - put_page_table_entry_in_pgtable(&pt[index], pa, attributes, memoryType, - fIsKernelMap); - - pinner.Unlock(); - - // Note: We don't need to invalidate the TLB for this address, as previously - // the entry was not present and the TLB doesn't cache those entries. - - fMapCount++; - - return 0; -} - - -status_t -X86VMTranslationMap32Bit::Unmap(addr_t start, addr_t end) -{ - page_directory_entry *pd = fPagingStructures->pgdir_virt; - - start = ROUNDDOWN(start, B_PAGE_SIZE); - end = ROUNDUP(end, B_PAGE_SIZE); - - TRACE("unmap_tmap: asked to free pages 0x%lx to 0x%lx\n", start, end); - -restart: - if (start >= end) - return B_OK; - - int index = VADDR_TO_PDENT(start); - if ((pd[index] & X86_PDE_PRESENT) == 0) { - // no pagetable here, move the start up to access the next page table - start = ROUNDUP(start + 1, B_PAGE_SIZE * 1024); - if (start == 0) - return B_OK; - goto restart; - } - - struct thread* thread = thread_get_current_thread(); - ThreadCPUPinner pinner(thread); - - page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - - for (index = VADDR_TO_PTENT(start); (index < 1024) && (start < end); - index++, start += B_PAGE_SIZE) { - if ((pt[index] & X86_PTE_PRESENT) == 0) { - // page mapping not valid - continue; - } - - TRACE("unmap_tmap: removing page 0x%lx\n", start); - - page_table_entry oldEntry = clear_page_table_entry_flags(&pt[index], - X86_PTE_PRESENT); - fMapCount--; - - if ((oldEntry & X86_PTE_ACCESSED) != 0) { - // Note, that we only need to invalidate the address, if the - // accessed flags was set, since only then the entry could have been - // in any TLB. - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = start; - - fInvalidPagesCount++; - } - } - - pinner.Unlock(); - - goto restart; -} - - -/*! Caller must have locked the cache of the page to be unmapped. - This object shouldn't be locked. -*/ -status_t -X86VMTranslationMap32Bit::UnmapPage(VMArea* area, addr_t address, - bool updatePageQueue) -{ - ASSERT(address % B_PAGE_SIZE == 0); - - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - TRACE("X86VMTranslationMap32Bit::UnmapPage(%#" B_PRIxADDR ")\n", address); - - RecursiveLocker locker(fLock); - - int index = VADDR_TO_PDENT(address); - if ((pd[index] & X86_PDE_PRESENT) == 0) - return B_ENTRY_NOT_FOUND; - - ThreadCPUPinner pinner(thread_get_current_thread()); - - page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - - index = VADDR_TO_PTENT(address); - page_table_entry oldEntry = clear_page_table_entry(&pt[index]); - - pinner.Unlock(); - - if ((oldEntry & X86_PTE_PRESENT) == 0) { - // page mapping not valid - return B_ENTRY_NOT_FOUND; - } - - fMapCount--; - - if ((oldEntry & X86_PTE_ACCESSED) != 0) { - // Note, that we only need to invalidate the address, if the - // accessed flags was set, since only then the entry could have been - // in any TLB. - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = address; - - fInvalidPagesCount++; - - Flush(); - - // NOTE: Between clearing the page table entry and Flush() other - // processors (actually even this processor with another thread of the - // same team) could still access the page in question via their cached - // entry. We can obviously lose a modified flag in this case, with the - // effect that the page looks unmodified (and might thus be recycled), - // but is actually modified. - // In most cases this is harmless, but for vm_remove_all_page_mappings() - // this is actually a problem. - // Interestingly FreeBSD seems to ignore this problem as well - // (cf. pmap_remove_all()), unless I've missed something. - } - - if (area->cache_type == CACHE_TYPE_DEVICE) - return B_OK; - - // get the page - vm_page* page = vm_lookup_page( - (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); - ASSERT(page != NULL); - - // transfer the accessed/dirty flags to the page - if ((oldEntry & X86_PTE_ACCESSED) != 0) - page->accessed = true; - if ((oldEntry & X86_PTE_DIRTY) != 0) - page->modified = true; - - // remove the mapping object/decrement the wired_count of the page - vm_page_mapping* mapping = NULL; - if (area->wiring == B_NO_LOCK) { - vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); - while ((mapping = iterator.Next()) != NULL) { - if (mapping->area == area) { - area->mappings.Remove(mapping); - page->mappings.Remove(mapping); - break; - } - } - - ASSERT(mapping != NULL); - } else - page->wired_count--; - - locker.Unlock(); - - if (page->wired_count == 0 && page->mappings.IsEmpty()) { - atomic_add(&gMappedPagesCount, -1); - - if (updatePageQueue) { - if (page->Cache()->temporary) - vm_page_set_state(page, PAGE_STATE_INACTIVE); - else if (page->modified) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - else - vm_page_set_state(page, PAGE_STATE_CACHED); - } - } - - if (mapping != NULL) { - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - object_cache_free(gPageMappingsObjectCache, mapping, - CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0)); - } - - return B_OK; -} - - -void -X86VMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size, - bool updatePageQueue) -{ - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - addr_t start = base; - addr_t end = base + size; - - TRACE("X86VMTranslationMap32Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#" - B_PRIxADDR ")\n", area, start, end); - - VMAreaMappings queue; - - RecursiveLocker locker(fLock); - - while (start < end) { - int index = VADDR_TO_PDENT(start); - if ((pd[index] & X86_PDE_PRESENT) == 0) { - // no page table here, move the start up to access the next page - // table - start = ROUNDUP(start + 1, B_PAGE_SIZE * 1024); - if (start == 0) - break; - continue; - } - - struct thread* thread = thread_get_current_thread(); - ThreadCPUPinner pinner(thread); - - page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - - for (index = VADDR_TO_PTENT(start); (index < 1024) && (start < end); - index++, start += B_PAGE_SIZE) { - page_table_entry oldEntry = clear_page_table_entry(&pt[index]); - if ((oldEntry & X86_PTE_PRESENT) == 0) - continue; - - fMapCount--; - - if ((oldEntry & X86_PTE_ACCESSED) != 0) { - // Note, that we only need to invalidate the address, if the - // accessed flags was set, since only then the entry could have - // been in any TLB. - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = start; - - fInvalidPagesCount++; - } - - if (area->cache_type != CACHE_TYPE_DEVICE) { - // get the page - vm_page* page = vm_lookup_page( - (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); - ASSERT(page != NULL); - - DEBUG_PAGE_ACCESS_START(page); - - // transfer the accessed/dirty flags to the page - if ((oldEntry & X86_PTE_ACCESSED) != 0) - page->accessed = true; - if ((oldEntry & X86_PTE_DIRTY) != 0) - page->modified = true; - - // remove the mapping object/decrement the wired_count of the - // page - if (area->wiring == B_NO_LOCK) { - vm_page_mapping* mapping = NULL; - vm_page_mappings::Iterator iterator - = page->mappings.GetIterator(); - while ((mapping = iterator.Next()) != NULL) { - if (mapping->area == area) - break; - } - - ASSERT(mapping != NULL); - - area->mappings.Remove(mapping); - page->mappings.Remove(mapping); - queue.Add(mapping); - } else - page->wired_count--; - - if (page->wired_count == 0 && page->mappings.IsEmpty()) { - atomic_add(&gMappedPagesCount, -1); - - if (updatePageQueue) { - if (page->Cache()->temporary) - vm_page_set_state(page, PAGE_STATE_INACTIVE); - else if (page->modified) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - else - vm_page_set_state(page, PAGE_STATE_CACHED); - } - } - - DEBUG_PAGE_ACCESS_END(page); - } - } - - Flush(); - // flush explicitly, since we directly use the lock - - pinner.Unlock(); - } - - // TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not - // really critical here, as in all cases this method is used, the unmapped - // area range is unmapped for good (resized/cut) and the pages will likely - // be freed. - - locker.Unlock(); - - // free removed mappings - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); - while (vm_page_mapping* mapping = queue.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); -} - - -void -X86VMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace, - bool ignoreTopCachePageFlags) -{ - if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { - X86VMTranslationMap32Bit::UnmapPages(area, area->Base(), area->Size(), - true); - return; - } - - bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; - - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - RecursiveLocker locker(fLock); - - VMAreaMappings mappings; - mappings.MoveFrom(&area->mappings); - - for (VMAreaMappings::Iterator it = mappings.GetIterator(); - vm_page_mapping* mapping = it.Next();) { - vm_page* page = mapping->page; - page->mappings.Remove(mapping); - - VMCache* cache = page->Cache(); - - bool pageFullyUnmapped = false; - if (page->wired_count == 0 && page->mappings.IsEmpty()) { - atomic_add(&gMappedPagesCount, -1); - pageFullyUnmapped = true; - } - - if (unmapPages || cache != area->cache) { - addr_t address = area->Base() - + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); - - int index = VADDR_TO_PDENT(address); - if ((pd[index] & X86_PDE_PRESENT) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page dir entry", page, area, address); - continue; - } - - ThreadCPUPinner pinner(thread_get_current_thread()); - - page_table_entry* pt - = (page_table_entry*)fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - page_table_entry oldEntry = clear_page_table_entry( - &pt[VADDR_TO_PTENT(address)]); - - pinner.Unlock(); - - if ((oldEntry & X86_PTE_PRESENT) == 0) { - panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " - "has no page table entry", page, area, address); - continue; - } - - // transfer the accessed/dirty flags to the page and invalidate - // the mapping, if necessary - if ((oldEntry & X86_PTE_ACCESSED) != 0) { - page->accessed = true; - - if (!deletingAddressSpace) { - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = address; - - fInvalidPagesCount++; - } - } - - if ((oldEntry & X86_PTE_DIRTY) != 0) - page->modified = true; - - if (pageFullyUnmapped) { - DEBUG_PAGE_ACCESS_START(page); - - if (cache->temporary) - vm_page_set_state(page, PAGE_STATE_INACTIVE); - else if (page->modified) - vm_page_set_state(page, PAGE_STATE_MODIFIED); - else - vm_page_set_state(page, PAGE_STATE_CACHED); - - DEBUG_PAGE_ACCESS_END(page); - } - } - - fMapCount--; - } - - Flush(); - // flush explicitely, since we directly use the lock - - locker.Unlock(); - - bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); - uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY - | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); - while (vm_page_mapping* mapping = mappings.RemoveHead()) - object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); -} - - -status_t -X86VMTranslationMap32Bit::Query(addr_t va, phys_addr_t *_physical, - uint32 *_flags) -{ - // default the flags to not present - *_flags = 0; - *_physical = 0; - - int index = VADDR_TO_PDENT(va); - page_directory_entry *pd = fPagingStructures->pgdir_virt; - if ((pd[index] & X86_PDE_PRESENT) == 0) { - // no pagetable here - return B_OK; - } - - struct thread* thread = thread_get_current_thread(); - ThreadCPUPinner pinner(thread); - - page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - page_table_entry entry = pt[VADDR_TO_PTENT(va)]; - - *_physical = entry & X86_PDE_ADDRESS_MASK; - - // read in the page state flags - if ((entry & X86_PTE_USER) != 0) { - *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0) - | B_READ_AREA; - } - - *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0) - | B_KERNEL_READ_AREA - | ((entry & X86_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0) - | ((entry & X86_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0) - | ((entry & X86_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0); - - pinner.Unlock(); - - TRACE("query_tmap: returning pa 0x%lx for va 0x%lx\n", *_physical, va); - - return B_OK; -} - - -status_t -X86VMTranslationMap32Bit::QueryInterrupt(addr_t va, phys_addr_t *_physical, - uint32 *_flags) -{ - *_flags = 0; - *_physical = 0; - - int index = VADDR_TO_PDENT(va); - page_directory_entry* pd = fPagingStructures->pgdir_virt; - if ((pd[index] & X86_PDE_PRESENT) == 0) { - // no pagetable here - return B_OK; - } - - // map page table entry - page_table_entry* pt = (page_table_entry*)X86PagingMethod32Bit::Method() - ->PhysicalPageMapper()->InterruptGetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - page_table_entry entry = pt[VADDR_TO_PTENT(va)]; - - *_physical = entry & X86_PDE_ADDRESS_MASK; - - // read in the page state flags - if ((entry & X86_PTE_USER) != 0) { - *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0) - | B_READ_AREA; - } - - *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0) - | B_KERNEL_READ_AREA - | ((entry & X86_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0) - | ((entry & X86_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0) - | ((entry & X86_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0); - - return B_OK; -} - - -status_t -X86VMTranslationMap32Bit::Protect(addr_t start, addr_t end, uint32 attributes, - uint32 memoryType) -{ - page_directory_entry *pd = fPagingStructures->pgdir_virt; - - start = ROUNDDOWN(start, B_PAGE_SIZE); - - TRACE("protect_tmap: pages 0x%lx to 0x%lx, attributes %lx\n", start, end, - attributes); - - // compute protection flags - uint32 newProtectionFlags = 0; - if ((attributes & B_USER_PROTECTION) != 0) { - newProtectionFlags = X86_PTE_USER; - if ((attributes & B_WRITE_AREA) != 0) - newProtectionFlags |= X86_PTE_WRITABLE; - } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) - newProtectionFlags = X86_PTE_WRITABLE; - -restart: - if (start >= end) - return B_OK; - - int index = VADDR_TO_PDENT(start); - if ((pd[index] & X86_PDE_PRESENT) == 0) { - // no pagetable here, move the start up to access the next page table - start = ROUNDUP(start + 1, B_PAGE_SIZE * 1024); - if (start == 0) - return B_OK; - goto restart; - } - - struct thread* thread = thread_get_current_thread(); - ThreadCPUPinner pinner(thread); - - page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - - for (index = VADDR_TO_PTENT(start); index < 1024 && start < end; - index++, start += B_PAGE_SIZE) { - page_table_entry entry = pt[index]; - if ((entry & X86_PTE_PRESENT) == 0) { - // page mapping not valid - continue; - } - - TRACE("protect_tmap: protect page 0x%lx\n", start); - - // set the new protection flags -- we want to do that atomically, - // without changing the accessed or dirty flag - page_table_entry oldEntry; - while (true) { - oldEntry = test_and_set_page_table_entry(&pt[index], - (entry & ~(X86_PTE_PROTECTION_MASK | X86_PTE_MEMORY_TYPE_MASK)) - | newProtectionFlags | memory_type_to_pte_flags(memoryType), - entry); - if (oldEntry == entry) - break; - entry = oldEntry; - } - - if ((oldEntry & X86_PTE_ACCESSED) != 0) { - // Note, that we only need to invalidate the address, if the - // accessed flag was set, since only then the entry could have been - // in any TLB. - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = start; - - fInvalidPagesCount++; - } - } - - pinner.Unlock(); - - goto restart; -} - - -status_t -X86VMTranslationMap32Bit::ClearFlags(addr_t va, uint32 flags) -{ - int index = VADDR_TO_PDENT(va); - page_directory_entry* pd = fPagingStructures->pgdir_virt; - if ((pd[index] & X86_PDE_PRESENT) == 0) { - // no pagetable here - return B_OK; - } - - uint32 flagsToClear = ((flags & PAGE_MODIFIED) ? X86_PTE_DIRTY : 0) - | ((flags & PAGE_ACCESSED) ? X86_PTE_ACCESSED : 0); - - struct thread* thread = thread_get_current_thread(); - ThreadCPUPinner pinner(thread); - - page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - index = VADDR_TO_PTENT(va); - - // clear out the flags we've been requested to clear - page_table_entry oldEntry - = clear_page_table_entry_flags(&pt[index], flagsToClear); - - pinner.Unlock(); - - if ((oldEntry & flagsToClear) != 0) { - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = va; - - fInvalidPagesCount++; - } - - return B_OK; -} - - -bool -X86VMTranslationMap32Bit::ClearAccessedAndModified(VMArea* area, addr_t address, - bool unmapIfUnaccessed, bool& _modified) -{ - ASSERT(address % B_PAGE_SIZE == 0); - - page_directory_entry* pd = fPagingStructures->pgdir_virt; - - TRACE("X86VMTranslationMap32Bit::ClearAccessedAndModified(%#" B_PRIxADDR - ")\n", address); - - RecursiveLocker locker(fLock); - - int index = VADDR_TO_PDENT(address); - if ((pd[index] & X86_PDE_PRESENT) == 0) - return false; - - ThreadCPUPinner pinner(thread_get_current_thread()); - - page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( - pd[index] & X86_PDE_ADDRESS_MASK); - - index = VADDR_TO_PTENT(address); - - // perform the deed - page_table_entry oldEntry; - - if (unmapIfUnaccessed) { - while (true) { - oldEntry = pt[index]; - if ((oldEntry & X86_PTE_PRESENT) == 0) { - // page mapping not valid - return false; - } - - if (oldEntry & X86_PTE_ACCESSED) { - // page was accessed -- just clear the flags - oldEntry = clear_page_table_entry_flags(&pt[index], - X86_PTE_ACCESSED | X86_PTE_DIRTY); - break; - } - - // page hasn't been accessed -- unmap it - if (test_and_set_page_table_entry(&pt[index], 0, oldEntry) - == oldEntry) { - break; - } - - // something changed -- check again - } - } else { - oldEntry = clear_page_table_entry_flags(&pt[index], - X86_PTE_ACCESSED | X86_PTE_DIRTY); - } - - pinner.Unlock(); - - _modified = (oldEntry & X86_PTE_DIRTY) != 0; - - if ((oldEntry & X86_PTE_ACCESSED) != 0) { - // Note, that we only need to invalidate the address, if the - // accessed flags was set, since only then the entry could have been - // in any TLB. - if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) - fInvalidPages[fInvalidPagesCount] = address; - - fInvalidPagesCount++; - - Flush(); - - return true; - } - - if (!unmapIfUnaccessed) - return false; - - // We have unmapped the address. Do the "high level" stuff. - - fMapCount--; - - if (area->cache_type == CACHE_TYPE_DEVICE) - return false; - - // get the page - vm_page* page = vm_lookup_page( - (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); - ASSERT(page != NULL); - - // remove the mapping object/decrement the wired_count of the page - vm_page_mapping* mapping = NULL; - if (area->wiring == B_NO_LOCK) { - vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); - while ((mapping = iterator.Next()) != NULL) { - if (mapping->area == area) { - area->mappings.Remove(mapping); - page->mappings.Remove(mapping); - break; - } - } - - ASSERT(mapping != NULL); - } else - page->wired_count--; - - locker.Unlock(); - - if (page->wired_count == 0 && page->mappings.IsEmpty()) - atomic_add(&gMappedPagesCount, -1); - - if (mapping != NULL) { - object_cache_free(gPageMappingsObjectCache, mapping, - CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE); - // Since this is called by the page daemon, we never want to lock - // the kernel address space. - } - - return false; -} - - -X86PagingStructures* -X86VMTranslationMap32Bit::PagingStructures() const -{ - return fPagingStructures; -} - - // #pragma mark - X86PagingMethod32Bit::PhysicalPageSlotPool @@ -1136,8 +102,7 @@ X86PagingMethod32Bit::PhysicalPageSlotPool::InitInitial(kernel_args* args) areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, false); // prepare the page table - x86_early_prepare_page_tables(pageTable, virtualBase, - 1024 * B_PAGE_SIZE); + _EarlyPreparePageTables(pageTable, virtualBase, 1024 * B_PAGE_SIZE); // init the pool structure and add the initial pool Init(-1, pageTable, -1, (addr_t)virtualBase); @@ -1263,7 +228,7 @@ X86PagingMethod32Bit::PhysicalPageSlotPool::AllocatePool( int32 index = (addr_t)virtualBase / (B_PAGE_SIZE * 1024); page_directory_entry* entry = &map->PagingStructures32Bit()->pgdir_virt[index]; - x86_put_pgtable_in_pgdir(entry, physicalTable, + PutPageTableInPageDir(entry, physicalTable, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); X86PagingStructures32Bit::UpdateAllPageDirs(index, *entry); @@ -1427,7 +392,7 @@ X86PagingMethod32Bit::MapEarly(kernel_args* args, addr_t virtualAddress, // put it in the pgdir e = &fPageHolePageDir[index]; - x86_put_pgtable_in_pgdir(e, pgtable, attributes); + PutPageTableInPageDir(e, pgtable, attributes); // zero it out in it's new mapping memset((unsigned int*)((addr_t)fPageHole @@ -1442,7 +407,7 @@ X86PagingMethod32Bit::MapEarly(kernel_args* args, addr_t virtualAddress, fPageHole[virtualAddress / B_PAGE_SIZE]); // now, fill in the pentry - put_page_table_entry_in_pgtable(fPageHole + virtualAddress / B_PAGE_SIZE, + PutPageTableEntryInTable(fPageHole + virtualAddress / B_PAGE_SIZE, physicalAddress, attributes, 0, IS_KERNEL_ADDRESS(virtualAddress)); return B_OK; @@ -1510,3 +475,96 @@ X86PagingMethod32Bit::IsKernelPageAccessible(addr_t virtualAddress, return (protection & B_KERNEL_WRITE_AREA) == 0 || (pageTableEntry & X86_PTE_WRITABLE) != 0; } + + +/*static*/ void +X86PagingMethod32Bit::PutPageTableInPageDir(page_directory_entry* entry, + phys_addr_t pgtablePhysical, uint32 attributes) +{ + *entry = (pgtablePhysical & X86_PDE_ADDRESS_MASK) + | X86_PDE_PRESENT + | X86_PDE_WRITABLE + | X86_PDE_USER; + // TODO: we ignore the attributes of the page table - for compatibility + // with BeOS we allow having user accessible areas in the kernel address + // space. This is currently being used by some drivers, mainly for the + // frame buffer. Our current real time data implementation makes use of + // this fact, too. + // We might want to get rid of this possibility one day, especially if + // we intend to port it to a platform that does not support this. +} + + +/*static*/ void +X86PagingMethod32Bit::PutPageTableEntryInTable(page_table_entry* entry, + phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType, + bool globalPage) +{ + page_table_entry page = (physicalAddress & X86_PTE_ADDRESS_MASK) + | X86_PTE_PRESENT | (globalPage ? X86_PTE_GLOBAL : 0) + | MemoryTypeToPageTableEntryFlags(memoryType); + + // if the page is user accessible, it's automatically + // accessible in kernel space, too (but with the same + // protection) + if ((attributes & B_USER_PROTECTION) != 0) { + page |= X86_PTE_USER; + if ((attributes & B_WRITE_AREA) != 0) + page |= X86_PTE_WRITABLE; + } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) + page |= X86_PTE_WRITABLE; + + // put it in the page table + *(volatile page_table_entry*)entry = page; +} + + +/*static*/ void +X86PagingMethod32Bit::_EarlyPreparePageTables(page_table_entry* pageTables, + addr_t address, size_t size) +{ + memset(pageTables, 0, B_PAGE_SIZE * (size / (B_PAGE_SIZE * 1024))); + + // put the array of pgtables directly into the kernel pagedir + // these will be wired and kept mapped into virtual space to be easy to get + // to + { + addr_t virtualTable = (addr_t)pageTables; + + page_directory_entry* pageHolePageDir + = X86PagingMethod32Bit::Method()->PageHolePageDir(); + + for (size_t i = 0; i < (size / (B_PAGE_SIZE * 1024)); + i++, virtualTable += B_PAGE_SIZE) { + phys_addr_t physicalTable = 0; + _EarlyQuery(virtualTable, &physicalTable); + page_directory_entry* entry = &pageHolePageDir[ + (address / (B_PAGE_SIZE * 1024)) + i]; + PutPageTableInPageDir(entry, physicalTable, + B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); + } + } +} + + +//! TODO: currently assumes this translation map is active +/*static*/ status_t +X86PagingMethod32Bit::_EarlyQuery(addr_t virtualAddress, + phys_addr_t *_physicalAddress) +{ + X86PagingMethod32Bit* method = X86PagingMethod32Bit::Method(); + int index = VADDR_TO_PDENT(virtualAddress); + if ((method->PageHolePageDir()[index] & X86_PDE_PRESENT) == 0) { + // no pagetable here + return B_ERROR; + } + + page_table_entry* entry = method->PageHole() + virtualAddress / B_PAGE_SIZE; + if ((*entry & X86_PTE_PRESENT) == 0) { + // page mapping not valid + return B_ERROR; + } + + *_physicalAddress = *entry & X86_PTE_ADDRESS_MASK; + return B_OK; +} diff --git a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.h b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.h index f208aaf2b7..92951f5b39 100644 --- a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.h +++ b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.h @@ -51,8 +51,40 @@ public: static X86PagingMethod32Bit* Method(); + static void PutPageTableInPageDir( + page_directory_entry* entry, + phys_addr_t pgtablePhysical, + uint32 attributes); + static void PutPageTableEntryInTable( + page_table_entry* entry, + phys_addr_t physicalAddress, + uint32 attributes, uint32 memoryType, + bool globalPage); + static page_table_entry SetPageTableEntry(page_table_entry* entry, + page_table_entry newEntry); + static page_table_entry SetPageTableEntryFlags(page_table_entry* entry, + uint32 flags); + static page_table_entry TestAndSetPageTableEntry( + page_table_entry* entry, + page_table_entry newEntry, + page_table_entry oldEntry); + static page_table_entry ClearPageTableEntry(page_table_entry* entry); + static page_table_entry ClearPageTableEntryFlags( + page_table_entry* entry, uint32 flags); + + static uint32 MemoryTypeToPageTableEntryFlags( + uint32 memoryType); + private: struct PhysicalPageSlotPool; + friend struct PhysicalPageSlotPool; + +private: + static void _EarlyPreparePageTables( + page_table_entry* pageTables, + addr_t address, size_t size); + static status_t _EarlyQuery(addr_t virtualAddress, + phys_addr_t *_physicalAddress); private: page_table_entry* fPageHole; @@ -72,4 +104,72 @@ X86PagingMethod32Bit::Method() } +/*static*/ inline page_table_entry +X86PagingMethod32Bit::SetPageTableEntry(page_table_entry* entry, + page_table_entry newEntry) +{ + return atomic_set((int32*)entry, newEntry); +} + + +/*static*/ inline page_table_entry +X86PagingMethod32Bit::SetPageTableEntryFlags(page_table_entry* entry, + uint32 flags) +{ + return atomic_or((int32*)entry, flags); +} + + +/*static*/ inline page_table_entry +X86PagingMethod32Bit::TestAndSetPageTableEntry(page_table_entry* entry, + page_table_entry newEntry, page_table_entry oldEntry) +{ + return atomic_test_and_set((int32*)entry, newEntry, oldEntry); +} + + +/*static*/ inline page_table_entry +X86PagingMethod32Bit::ClearPageTableEntry(page_table_entry* entry) +{ + return SetPageTableEntry(entry, 0); +} + + +/*static*/ inline page_table_entry +X86PagingMethod32Bit::ClearPageTableEntryFlags(page_table_entry* entry, uint32 flags) +{ + return atomic_and((int32*)entry, ~flags); +} + + +/*static*/ inline uint32 +X86PagingMethod32Bit::MemoryTypeToPageTableEntryFlags(uint32 memoryType) +{ + // ATM we only handle the uncacheable and write-through type explicitly. For + // all other types we rely on the MTRRs to be set up correctly. Since we set + // the default memory type to write-back and since the uncacheable type in + // the PTE overrides any MTRR attribute (though, as per the specs, that is + // not recommended for performance reasons), this reduces the work we + // actually *have* to do with the MTRRs to setting the remaining types + // (usually only write-combining for the frame buffer). + switch (memoryType) { + case B_MTR_UC: + return X86_PTE_CACHING_DISABLED | X86_PTE_WRITE_THROUGH; + + case B_MTR_WC: + // X86_PTE_WRITE_THROUGH would be closer, but the combination with + // MTRR WC is "implementation defined" for Pentium Pro/II. + return 0; + + case B_MTR_WT: + return X86_PTE_WRITE_THROUGH; + + case B_MTR_WP: + case B_MTR_WB: + default: + return 0; + } +} + + #endif // KERNEL_ARCH_X86_PAGING_32_BIT_X86_PAGING_METHOD_32_BIT_H diff --git a/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp b/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp new file mode 100644 index 0000000000..8c0c5ce3e1 --- /dev/null +++ b/src/system/kernel/arch/x86/paging/32bit/X86VMTranslationMap32Bit.cpp @@ -0,0 +1,943 @@ +/* + * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de. + * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. + * Distributed under the terms of the NewOS License. + */ + + +#include "paging/32bit/X86VMTranslationMap32Bit.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "paging/32bit/X86PagingMethod32Bit.h" +#include "paging/32bit/X86PagingStructures32Bit.h" +#include "paging/x86_physical_page_mapper.h" + + +//#define TRACE_X86_VM_TRANSLATION_MAP_32_BIT +#ifdef TRACE_X86_VM_TRANSLATION_MAP_32_BIT +# define TRACE(x...) dprintf(x) +#else +# define TRACE(x...) ; +#endif + + +X86VMTranslationMap32Bit::X86VMTranslationMap32Bit() + : + fPagingStructures(NULL) +{ +} + + +X86VMTranslationMap32Bit::~X86VMTranslationMap32Bit() +{ + if (fPagingStructures == NULL) + return; + + if (fPageMapper != NULL) + fPageMapper->Delete(); + + if (fPagingStructures->pgdir_virt != NULL) { + // cycle through and free all of the user space pgtables + for (uint32 i = VADDR_TO_PDENT(USER_BASE); + i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) { + if ((fPagingStructures->pgdir_virt[i] & X86_PDE_PRESENT) != 0) { + addr_t address = fPagingStructures->pgdir_virt[i] + & X86_PDE_ADDRESS_MASK; + vm_page* page = vm_lookup_page(address / B_PAGE_SIZE); + if (!page) + panic("destroy_tmap: didn't find pgtable page\n"); + DEBUG_PAGE_ACCESS_START(page); + vm_page_set_state(page, PAGE_STATE_FREE); + } + } + } + + fPagingStructures->RemoveReference(); +} + + +status_t +X86VMTranslationMap32Bit::Init(bool kernel) +{ + TRACE("X86VMTranslationMap32Bit::Init()\n"); + + X86VMTranslationMap::Init(kernel); + + fPagingStructures = new(std::nothrow) X86PagingStructures32Bit; + if (fPagingStructures == NULL) + return B_NO_MEMORY; + + X86PagingMethod32Bit* method = X86PagingMethod32Bit::Method(); + + if (!kernel) { + // user + // allocate a physical page mapper + status_t error = method->PhysicalPageMapper() + ->CreateTranslationMapPhysicalPageMapper(&fPageMapper); + if (error != B_OK) + return error; + + // allocate the page directory + page_directory_entry* virtualPageDir = (page_directory_entry*)memalign( + B_PAGE_SIZE, B_PAGE_SIZE); + if (virtualPageDir == NULL) + return B_NO_MEMORY; + + // look up the page directory's physical address + phys_addr_t physicalPageDir; + vm_get_page_mapping(VMAddressSpace::KernelID(), + (addr_t)virtualPageDir, &physicalPageDir); + + fPagingStructures->Init(virtualPageDir, physicalPageDir, + method->KernelVirtualPageDirectory()); + } else { + // kernel + // get the physical page mapper + fPageMapper = method->KernelPhysicalPageMapper(); + + // we already know the kernel pgdir mapping + fPagingStructures->Init(method->KernelVirtualPageDirectory(), + method->KernelPhysicalPageDirectory(), NULL); + } + + return B_OK; +} + + +size_t +X86VMTranslationMap32Bit::MaxPagesNeededToMap(addr_t start, addr_t end) const +{ + // If start == 0, the actual base address is not yet known to the caller and + // we shall assume the worst case. + if (start == 0) { + // offset the range so it has the worst possible alignment + start = 1023 * B_PAGE_SIZE; + end += 1023 * B_PAGE_SIZE; + } + + return VADDR_TO_PDENT(end) + 1 - VADDR_TO_PDENT(start); +} + + +status_t +X86VMTranslationMap32Bit::Map(addr_t va, phys_addr_t pa, uint32 attributes, + uint32 memoryType, vm_page_reservation* reservation) +{ + TRACE("map_tmap: entry pa 0x%lx va 0x%lx\n", pa, va); + +/* + dprintf("pgdir at 0x%x\n", pgdir); + dprintf("index is %d\n", va / B_PAGE_SIZE / 1024); + dprintf("final at 0x%x\n", &pgdir[va / B_PAGE_SIZE / 1024]); + dprintf("value is 0x%x\n", *(int *)&pgdir[va / B_PAGE_SIZE / 1024]); + dprintf("present bit is %d\n", pgdir[va / B_PAGE_SIZE / 1024].present); + dprintf("addr is %d\n", pgdir[va / B_PAGE_SIZE / 1024].addr); +*/ + page_directory_entry* pd = fPagingStructures->pgdir_virt; + + // check to see if a page table exists for this range + uint32 index = VADDR_TO_PDENT(va); + if ((pd[index] & X86_PDE_PRESENT) == 0) { + phys_addr_t pgtable; + vm_page *page; + + // we need to allocate a pgtable + page = vm_page_allocate_page(reservation, + PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR); + + DEBUG_PAGE_ACCESS_END(page); + + pgtable = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE; + + TRACE("map_tmap: asked for free page for pgtable. 0x%lx\n", pgtable); + + // put it in the pgdir + X86PagingMethod32Bit::PutPageTableInPageDir(&pd[index], pgtable, + attributes + | ((attributes & B_USER_PROTECTION) != 0 + ? B_WRITE_AREA : B_KERNEL_WRITE_AREA)); + + // update any other page directories, if it maps kernel space + if (index >= FIRST_KERNEL_PGDIR_ENT + && index < (FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS)) { + X86PagingStructures32Bit::UpdateAllPageDirs(index, pd[index]); + } + + fMapCount++; + } + + // now, fill in the pentry + struct thread* thread = thread_get_current_thread(); + ThreadCPUPinner pinner(thread); + + page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + index = VADDR_TO_PTENT(va); + + ASSERT_PRINT((pt[index] & X86_PTE_PRESENT) == 0, + "virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx32, va, + pt[index]); + + X86PagingMethod32Bit::PutPageTableEntryInTable(&pt[index], pa, attributes, + memoryType, fIsKernelMap); + + pinner.Unlock(); + + // Note: We don't need to invalidate the TLB for this address, as previously + // the entry was not present and the TLB doesn't cache those entries. + + fMapCount++; + + return 0; +} + + +status_t +X86VMTranslationMap32Bit::Unmap(addr_t start, addr_t end) +{ + page_directory_entry *pd = fPagingStructures->pgdir_virt; + + start = ROUNDDOWN(start, B_PAGE_SIZE); + end = ROUNDUP(end, B_PAGE_SIZE); + + TRACE("unmap_tmap: asked to free pages 0x%lx to 0x%lx\n", start, end); + +restart: + if (start >= end) + return B_OK; + + int index = VADDR_TO_PDENT(start); + if ((pd[index] & X86_PDE_PRESENT) == 0) { + // no pagetable here, move the start up to access the next page table + start = ROUNDUP(start + 1, B_PAGE_SIZE * 1024); + if (start == 0) + return B_OK; + goto restart; + } + + struct thread* thread = thread_get_current_thread(); + ThreadCPUPinner pinner(thread); + + page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + + for (index = VADDR_TO_PTENT(start); (index < 1024) && (start < end); + index++, start += B_PAGE_SIZE) { + if ((pt[index] & X86_PTE_PRESENT) == 0) { + // page mapping not valid + continue; + } + + TRACE("unmap_tmap: removing page 0x%lx\n", start); + + page_table_entry oldEntry + = X86PagingMethod32Bit::ClearPageTableEntryFlags(&pt[index], + X86_PTE_PRESENT); + fMapCount--; + + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + // Note, that we only need to invalidate the address, if the + // accessed flags was set, since only then the entry could have been + // in any TLB. + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = start; + + fInvalidPagesCount++; + } + } + + pinner.Unlock(); + + goto restart; +} + + +/*! Caller must have locked the cache of the page to be unmapped. + This object shouldn't be locked. +*/ +status_t +X86VMTranslationMap32Bit::UnmapPage(VMArea* area, addr_t address, + bool updatePageQueue) +{ + ASSERT(address % B_PAGE_SIZE == 0); + + page_directory_entry* pd = fPagingStructures->pgdir_virt; + + TRACE("X86VMTranslationMap32Bit::UnmapPage(%#" B_PRIxADDR ")\n", address); + + RecursiveLocker locker(fLock); + + int index = VADDR_TO_PDENT(address); + if ((pd[index] & X86_PDE_PRESENT) == 0) + return B_ENTRY_NOT_FOUND; + + ThreadCPUPinner pinner(thread_get_current_thread()); + + page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + + index = VADDR_TO_PTENT(address); + page_table_entry oldEntry = X86PagingMethod32Bit::ClearPageTableEntry( + &pt[index]); + + pinner.Unlock(); + + if ((oldEntry & X86_PTE_PRESENT) == 0) { + // page mapping not valid + return B_ENTRY_NOT_FOUND; + } + + fMapCount--; + + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + // Note, that we only need to invalidate the address, if the + // accessed flags was set, since only then the entry could have been + // in any TLB. + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = address; + + fInvalidPagesCount++; + + Flush(); + + // NOTE: Between clearing the page table entry and Flush() other + // processors (actually even this processor with another thread of the + // same team) could still access the page in question via their cached + // entry. We can obviously lose a modified flag in this case, with the + // effect that the page looks unmodified (and might thus be recycled), + // but is actually modified. + // In most cases this is harmless, but for vm_remove_all_page_mappings() + // this is actually a problem. + // Interestingly FreeBSD seems to ignore this problem as well + // (cf. pmap_remove_all()), unless I've missed something. + } + + if (area->cache_type == CACHE_TYPE_DEVICE) + return B_OK; + + // get the page + vm_page* page = vm_lookup_page( + (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); + ASSERT(page != NULL); + + // transfer the accessed/dirty flags to the page + if ((oldEntry & X86_PTE_ACCESSED) != 0) + page->accessed = true; + if ((oldEntry & X86_PTE_DIRTY) != 0) + page->modified = true; + + // remove the mapping object/decrement the wired_count of the page + vm_page_mapping* mapping = NULL; + if (area->wiring == B_NO_LOCK) { + vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); + while ((mapping = iterator.Next()) != NULL) { + if (mapping->area == area) { + area->mappings.Remove(mapping); + page->mappings.Remove(mapping); + break; + } + } + + ASSERT(mapping != NULL); + } else + page->wired_count--; + + locker.Unlock(); + + if (page->wired_count == 0 && page->mappings.IsEmpty()) { + atomic_add(&gMappedPagesCount, -1); + + if (updatePageQueue) { + if (page->Cache()->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + } + } + + if (mapping != NULL) { + bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); + object_cache_free(gPageMappingsObjectCache, mapping, + CACHE_DONT_WAIT_FOR_MEMORY + | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0)); + } + + return B_OK; +} + + +void +X86VMTranslationMap32Bit::UnmapPages(VMArea* area, addr_t base, size_t size, + bool updatePageQueue) +{ + page_directory_entry* pd = fPagingStructures->pgdir_virt; + + addr_t start = base; + addr_t end = base + size; + + TRACE("X86VMTranslationMap32Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#" + B_PRIxADDR ")\n", area, start, end); + + VMAreaMappings queue; + + RecursiveLocker locker(fLock); + + while (start < end) { + int index = VADDR_TO_PDENT(start); + if ((pd[index] & X86_PDE_PRESENT) == 0) { + // no page table here, move the start up to access the next page + // table + start = ROUNDUP(start + 1, B_PAGE_SIZE * 1024); + if (start == 0) + break; + continue; + } + + struct thread* thread = thread_get_current_thread(); + ThreadCPUPinner pinner(thread); + + page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + + for (index = VADDR_TO_PTENT(start); (index < 1024) && (start < end); + index++, start += B_PAGE_SIZE) { + page_table_entry oldEntry + = X86PagingMethod32Bit::ClearPageTableEntry(&pt[index]); + if ((oldEntry & X86_PTE_PRESENT) == 0) + continue; + + fMapCount--; + + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + // Note, that we only need to invalidate the address, if the + // accessed flags was set, since only then the entry could have + // been in any TLB. + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = start; + + fInvalidPagesCount++; + } + + if (area->cache_type != CACHE_TYPE_DEVICE) { + // get the page + vm_page* page = vm_lookup_page( + (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); + ASSERT(page != NULL); + + DEBUG_PAGE_ACCESS_START(page); + + // transfer the accessed/dirty flags to the page + if ((oldEntry & X86_PTE_ACCESSED) != 0) + page->accessed = true; + if ((oldEntry & X86_PTE_DIRTY) != 0) + page->modified = true; + + // remove the mapping object/decrement the wired_count of the + // page + if (area->wiring == B_NO_LOCK) { + vm_page_mapping* mapping = NULL; + vm_page_mappings::Iterator iterator + = page->mappings.GetIterator(); + while ((mapping = iterator.Next()) != NULL) { + if (mapping->area == area) + break; + } + + ASSERT(mapping != NULL); + + area->mappings.Remove(mapping); + page->mappings.Remove(mapping); + queue.Add(mapping); + } else + page->wired_count--; + + if (page->wired_count == 0 && page->mappings.IsEmpty()) { + atomic_add(&gMappedPagesCount, -1); + + if (updatePageQueue) { + if (page->Cache()->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + } + } + + DEBUG_PAGE_ACCESS_END(page); + } + } + + Flush(); + // flush explicitly, since we directly use the lock + + pinner.Unlock(); + } + + // TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not + // really critical here, as in all cases this method is used, the unmapped + // area range is unmapped for good (resized/cut) and the pages will likely + // be freed. + + locker.Unlock(); + + // free removed mappings + bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); + uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY + | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); + while (vm_page_mapping* mapping = queue.RemoveHead()) + object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); +} + + +void +X86VMTranslationMap32Bit::UnmapArea(VMArea* area, bool deletingAddressSpace, + bool ignoreTopCachePageFlags) +{ + if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) { + X86VMTranslationMap32Bit::UnmapPages(area, area->Base(), area->Size(), + true); + return; + } + + bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags; + + page_directory_entry* pd = fPagingStructures->pgdir_virt; + + RecursiveLocker locker(fLock); + + VMAreaMappings mappings; + mappings.MoveFrom(&area->mappings); + + for (VMAreaMappings::Iterator it = mappings.GetIterator(); + vm_page_mapping* mapping = it.Next();) { + vm_page* page = mapping->page; + page->mappings.Remove(mapping); + + VMCache* cache = page->Cache(); + + bool pageFullyUnmapped = false; + if (page->wired_count == 0 && page->mappings.IsEmpty()) { + atomic_add(&gMappedPagesCount, -1); + pageFullyUnmapped = true; + } + + if (unmapPages || cache != area->cache) { + addr_t address = area->Base() + + ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset); + + int index = VADDR_TO_PDENT(address); + if ((pd[index] & X86_PDE_PRESENT) == 0) { + panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " + "has no page dir entry", page, area, address); + continue; + } + + ThreadCPUPinner pinner(thread_get_current_thread()); + + page_table_entry* pt + = (page_table_entry*)fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + page_table_entry oldEntry + = X86PagingMethod32Bit::ClearPageTableEntry( + &pt[VADDR_TO_PTENT(address)]); + + pinner.Unlock(); + + if ((oldEntry & X86_PTE_PRESENT) == 0) { + panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but " + "has no page table entry", page, area, address); + continue; + } + + // transfer the accessed/dirty flags to the page and invalidate + // the mapping, if necessary + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + page->accessed = true; + + if (!deletingAddressSpace) { + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = address; + + fInvalidPagesCount++; + } + } + + if ((oldEntry & X86_PTE_DIRTY) != 0) + page->modified = true; + + if (pageFullyUnmapped) { + DEBUG_PAGE_ACCESS_START(page); + + if (cache->temporary) + vm_page_set_state(page, PAGE_STATE_INACTIVE); + else if (page->modified) + vm_page_set_state(page, PAGE_STATE_MODIFIED); + else + vm_page_set_state(page, PAGE_STATE_CACHED); + + DEBUG_PAGE_ACCESS_END(page); + } + } + + fMapCount--; + } + + Flush(); + // flush explicitely, since we directly use the lock + + locker.Unlock(); + + bool isKernelSpace = area->address_space == VMAddressSpace::Kernel(); + uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY + | (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0); + while (vm_page_mapping* mapping = mappings.RemoveHead()) + object_cache_free(gPageMappingsObjectCache, mapping, freeFlags); +} + + +status_t +X86VMTranslationMap32Bit::Query(addr_t va, phys_addr_t *_physical, + uint32 *_flags) +{ + // default the flags to not present + *_flags = 0; + *_physical = 0; + + int index = VADDR_TO_PDENT(va); + page_directory_entry *pd = fPagingStructures->pgdir_virt; + if ((pd[index] & X86_PDE_PRESENT) == 0) { + // no pagetable here + return B_OK; + } + + struct thread* thread = thread_get_current_thread(); + ThreadCPUPinner pinner(thread); + + page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + page_table_entry entry = pt[VADDR_TO_PTENT(va)]; + + *_physical = entry & X86_PDE_ADDRESS_MASK; + + // read in the page state flags + if ((entry & X86_PTE_USER) != 0) { + *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0) + | B_READ_AREA; + } + + *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0) + | B_KERNEL_READ_AREA + | ((entry & X86_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0) + | ((entry & X86_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0) + | ((entry & X86_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0); + + pinner.Unlock(); + + TRACE("query_tmap: returning pa 0x%lx for va 0x%lx\n", *_physical, va); + + return B_OK; +} + + +status_t +X86VMTranslationMap32Bit::QueryInterrupt(addr_t va, phys_addr_t *_physical, + uint32 *_flags) +{ + *_flags = 0; + *_physical = 0; + + int index = VADDR_TO_PDENT(va); + page_directory_entry* pd = fPagingStructures->pgdir_virt; + if ((pd[index] & X86_PDE_PRESENT) == 0) { + // no pagetable here + return B_OK; + } + + // map page table entry + page_table_entry* pt = (page_table_entry*)X86PagingMethod32Bit::Method() + ->PhysicalPageMapper()->InterruptGetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + page_table_entry entry = pt[VADDR_TO_PTENT(va)]; + + *_physical = entry & X86_PDE_ADDRESS_MASK; + + // read in the page state flags + if ((entry & X86_PTE_USER) != 0) { + *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0) + | B_READ_AREA; + } + + *_flags |= ((entry & X86_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0) + | B_KERNEL_READ_AREA + | ((entry & X86_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0) + | ((entry & X86_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0) + | ((entry & X86_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0); + + return B_OK; +} + + +status_t +X86VMTranslationMap32Bit::Protect(addr_t start, addr_t end, uint32 attributes, + uint32 memoryType) +{ + page_directory_entry *pd = fPagingStructures->pgdir_virt; + + start = ROUNDDOWN(start, B_PAGE_SIZE); + + TRACE("protect_tmap: pages 0x%lx to 0x%lx, attributes %lx\n", start, end, + attributes); + + // compute protection flags + uint32 newProtectionFlags = 0; + if ((attributes & B_USER_PROTECTION) != 0) { + newProtectionFlags = X86_PTE_USER; + if ((attributes & B_WRITE_AREA) != 0) + newProtectionFlags |= X86_PTE_WRITABLE; + } else if ((attributes & B_KERNEL_WRITE_AREA) != 0) + newProtectionFlags = X86_PTE_WRITABLE; + +restart: + if (start >= end) + return B_OK; + + int index = VADDR_TO_PDENT(start); + if ((pd[index] & X86_PDE_PRESENT) == 0) { + // no pagetable here, move the start up to access the next page table + start = ROUNDUP(start + 1, B_PAGE_SIZE * 1024); + if (start == 0) + return B_OK; + goto restart; + } + + struct thread* thread = thread_get_current_thread(); + ThreadCPUPinner pinner(thread); + + page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + + for (index = VADDR_TO_PTENT(start); index < 1024 && start < end; + index++, start += B_PAGE_SIZE) { + page_table_entry entry = pt[index]; + if ((entry & X86_PTE_PRESENT) == 0) { + // page mapping not valid + continue; + } + + TRACE("protect_tmap: protect page 0x%lx\n", start); + + // set the new protection flags -- we want to do that atomically, + // without changing the accessed or dirty flag + page_table_entry oldEntry; + while (true) { + oldEntry = X86PagingMethod32Bit::TestAndSetPageTableEntry( + &pt[index], + (entry & ~(X86_PTE_PROTECTION_MASK | X86_PTE_MEMORY_TYPE_MASK)) + | newProtectionFlags + | X86PagingMethod32Bit::MemoryTypeToPageTableEntryFlags( + memoryType), + entry); + if (oldEntry == entry) + break; + entry = oldEntry; + } + + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + // Note, that we only need to invalidate the address, if the + // accessed flag was set, since only then the entry could have been + // in any TLB. + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = start; + + fInvalidPagesCount++; + } + } + + pinner.Unlock(); + + goto restart; +} + + +status_t +X86VMTranslationMap32Bit::ClearFlags(addr_t va, uint32 flags) +{ + int index = VADDR_TO_PDENT(va); + page_directory_entry* pd = fPagingStructures->pgdir_virt; + if ((pd[index] & X86_PDE_PRESENT) == 0) { + // no pagetable here + return B_OK; + } + + uint32 flagsToClear = ((flags & PAGE_MODIFIED) ? X86_PTE_DIRTY : 0) + | ((flags & PAGE_ACCESSED) ? X86_PTE_ACCESSED : 0); + + struct thread* thread = thread_get_current_thread(); + ThreadCPUPinner pinner(thread); + + page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + index = VADDR_TO_PTENT(va); + + // clear out the flags we've been requested to clear + page_table_entry oldEntry + = X86PagingMethod32Bit::ClearPageTableEntryFlags(&pt[index], + flagsToClear); + + pinner.Unlock(); + + if ((oldEntry & flagsToClear) != 0) { + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = va; + + fInvalidPagesCount++; + } + + return B_OK; +} + + +bool +X86VMTranslationMap32Bit::ClearAccessedAndModified(VMArea* area, addr_t address, + bool unmapIfUnaccessed, bool& _modified) +{ + ASSERT(address % B_PAGE_SIZE == 0); + + page_directory_entry* pd = fPagingStructures->pgdir_virt; + + TRACE("X86VMTranslationMap32Bit::ClearAccessedAndModified(%#" B_PRIxADDR + ")\n", address); + + RecursiveLocker locker(fLock); + + int index = VADDR_TO_PDENT(address); + if ((pd[index] & X86_PDE_PRESENT) == 0) + return false; + + ThreadCPUPinner pinner(thread_get_current_thread()); + + page_table_entry* pt = (page_table_entry*)fPageMapper->GetPageTableAt( + pd[index] & X86_PDE_ADDRESS_MASK); + + index = VADDR_TO_PTENT(address); + + // perform the deed + page_table_entry oldEntry; + + if (unmapIfUnaccessed) { + while (true) { + oldEntry = pt[index]; + if ((oldEntry & X86_PTE_PRESENT) == 0) { + // page mapping not valid + return false; + } + + if (oldEntry & X86_PTE_ACCESSED) { + // page was accessed -- just clear the flags + oldEntry = X86PagingMethod32Bit::ClearPageTableEntryFlags( + &pt[index], X86_PTE_ACCESSED | X86_PTE_DIRTY); + break; + } + + // page hasn't been accessed -- unmap it + if (X86PagingMethod32Bit::TestAndSetPageTableEntry(&pt[index], 0, + oldEntry) == oldEntry) { + break; + } + + // something changed -- check again + } + } else { + oldEntry = X86PagingMethod32Bit::ClearPageTableEntryFlags(&pt[index], + X86_PTE_ACCESSED | X86_PTE_DIRTY); + } + + pinner.Unlock(); + + _modified = (oldEntry & X86_PTE_DIRTY) != 0; + + if ((oldEntry & X86_PTE_ACCESSED) != 0) { + // Note, that we only need to invalidate the address, if the + // accessed flags was set, since only then the entry could have been + // in any TLB. + if (fInvalidPagesCount < PAGE_INVALIDATE_CACHE_SIZE) + fInvalidPages[fInvalidPagesCount] = address; + + fInvalidPagesCount++; + + Flush(); + + return true; + } + + if (!unmapIfUnaccessed) + return false; + + // We have unmapped the address. Do the "high level" stuff. + + fMapCount--; + + if (area->cache_type == CACHE_TYPE_DEVICE) + return false; + + // get the page + vm_page* page = vm_lookup_page( + (oldEntry & X86_PTE_ADDRESS_MASK) / B_PAGE_SIZE); + ASSERT(page != NULL); + + // remove the mapping object/decrement the wired_count of the page + vm_page_mapping* mapping = NULL; + if (area->wiring == B_NO_LOCK) { + vm_page_mappings::Iterator iterator = page->mappings.GetIterator(); + while ((mapping = iterator.Next()) != NULL) { + if (mapping->area == area) { + area->mappings.Remove(mapping); + page->mappings.Remove(mapping); + break; + } + } + + ASSERT(mapping != NULL); + } else + page->wired_count--; + + locker.Unlock(); + + if (page->wired_count == 0 && page->mappings.IsEmpty()) + atomic_add(&gMappedPagesCount, -1); + + if (mapping != NULL) { + object_cache_free(gPageMappingsObjectCache, mapping, + CACHE_DONT_WAIT_FOR_MEMORY | CACHE_DONT_LOCK_KERNEL_SPACE); + // Since this is called by the page daemon, we never want to lock + // the kernel address space. + } + + return false; +} + + +X86PagingStructures* +X86VMTranslationMap32Bit::PagingStructures() const +{ + return fPagingStructures; +} diff --git a/src/system/kernel/arch/x86/paging/32bit/paging.h b/src/system/kernel/arch/x86/paging/32bit/paging.h index 0991e61c21..431d5ae67e 100644 --- a/src/system/kernel/arch/x86/paging/32bit/paging.h +++ b/src/system/kernel/arch/x86/paging/32bit/paging.h @@ -64,46 +64,4 @@ typedef uint32 page_table_entry; typedef uint32 page_directory_entry; -void x86_early_prepare_page_tables(page_table_entry* pageTables, addr_t address, - size_t size); -void x86_put_pgtable_in_pgdir(page_directory_entry* entry, - phys_addr_t physicalPageTable, uint32 attributes); - - -static inline page_table_entry -set_page_table_entry(page_table_entry* entry, page_table_entry newEntry) -{ - return atomic_set((int32*)entry, newEntry); -} - - -static inline page_table_entry -test_and_set_page_table_entry(page_table_entry* entry, - page_table_entry newEntry, page_table_entry oldEntry) -{ - return atomic_test_and_set((int32*)entry, newEntry, oldEntry); -} - - -static inline page_table_entry -clear_page_table_entry(page_table_entry* entry) -{ - return set_page_table_entry(entry, 0); -} - - -static inline page_table_entry -clear_page_table_entry_flags(page_table_entry* entry, uint32 flags) -{ - return atomic_and((int32*)entry, ~flags); -} - - -static inline page_table_entry -set_page_table_entry_flags(page_table_entry* entry, uint32 flags) -{ - return atomic_or((int32*)entry, flags); -} - - #endif // _KERNEL_ARCH_X86_PAGING_32_BIT_PAGING_H