Aren/haiku
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Prefixed memset_physical() and memcpy_to_physical() with "vm_",

Browse Source 
added vm_memcpy_from_physical() and vm_memcpy_physical_page(), and
  added respective functions to the vm_translation_map operations. The
  architecture specific implementation can now decide how to implement
  them most efficiently. Added generic implementations that can be used,
  though.
* Changed vm_{get,put}_physical_page(). The former no longer accepts
  flags (the only flag PHYSICAL_PAGE_DONT_WAIT wasn't needed anymore).
  Instead it returns an implementation-specific handle that has to be
  passed to the latter. Added vm_{get,put}_physical_page_current_cpu()
  and *_debug() variants, that work only for the current CPU,
  respectively when in the kernel debugger. Also adjusted the
  vm_translation_map operations accordingly.
* Made consequent use of the physical memory operations in the source
  tree.
* Also adjusted the m68k and ppc implementations with respect to the
  vm_translation_map operation changes, but they are probably broken,
  nevertheless.
* For x86 the generic physical page mapper isn't used anymore. It is
  suboptimal in any case. For systems with small memory it is too much
  overhead, since one can just map the complete physical memory (that's
  not done yet, though). For systems with large memory it counteracts
  the VM strategy to reuse the least recently used pages. Since those
  pages will most likely not be mapped by the page mapper anymore, it
  will keep remapping chunks. This was also the reason why building
  Haiku in Haiku was significantly faster with only 256 MB RAM (since
  that much could be kept mapped all the time).
  Now we're using a different strategy: We have small pools of virtual
  page slots per CPU that are used for the physical page operations
  (memset_physical(), memcpy_*_physical()) with CPU-pinned thread.
  Furthermore we have four slots per translation map, which are used to
  map page tables.

These changes speed up the Haiku image build in Haiku significantly. On
my Core2 Duo 2.2 GHz 2 GB machine about 40% to 20 min 40 s (KDEBUG
disabled, block cache debug disabled). Still more than factor 3 slower
than FreeBSD and Linux, though.


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@28244 a95241bf-73f2-0310-859d-f6bbb57e9c96
This commit is contained in:
Ingo Weinhold 2008-10-20 00:06:09 +00:00
parent ef8a9c3074
commit 47c40a10a1
33 changed files with 1756 additions and 596 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
headers/private/kernel/arch/x86/arch_vm_translation_map.h
View File

@ -5,24 +5,9 @@
#ifndef _KERNEL_ARCH_x86_VM_TRANSLATION_MAP_H
#define _KERNEL_ARCH_x86_VM_TRANSLATION_MAP_H
#include <arch/vm_translation_map.h>
#define PAGE_INVALIDATE_CACHE_SIZE 64
struct page_directory_entry;
typedef struct vm_translation_map_arch_info {
struct page_directory_entry *pgdir_virt;
struct page_directory_entry *pgdir_phys;
vint32 active_on_cpus;
// mask indicating on which CPUs the map is currently used
int num_invalidate_pages;
addr_t pages_to_invalidate[PAGE_INVALIDATE_CACHE_SIZE];
} vm_translation_map_arch_info;
// quick function to return the physical pgdir of a mapping, needed for a context switch
#ifdef __cplusplus
extern "C"

20
headers/private/kernel/vm.h
View File

@ -91,17 +91,27 @@ status_t vm_unmap_pages(struct vm_area *area, addr_t base, size_t length,
bool preserveModified);
status_t vm_map_page(struct vm_area *area, struct vm_page *page, addr_t address,
uint32 protection);
status_t vm_get_physical_page(addr_t paddr, addr_t *vaddr, uint32 flags);
status_t vm_put_physical_page(addr_t vaddr);
status_t vm_get_physical_page(addr_t paddr, addr_t* vaddr, void** _handle);
status_t vm_put_physical_page(addr_t vaddr, void* handle);
status_t vm_get_physical_page_current_cpu(addr_t paddr, addr_t* vaddr,
void** _handle);
status_t vm_put_physical_page_current_cpu(addr_t vaddr, void* handle);
status_t vm_get_physical_page_debug(addr_t paddr, addr_t* vaddr,
void** _handle);
status_t vm_put_physical_page_debug(addr_t vaddr, void* handle);
void vm_get_info(struct system_memory_info *info);
uint32 vm_num_page_faults(void);
off_t vm_available_memory(void);
off_t vm_available_not_needed_memory(void);
status_t memset_physical(addr_t address, int value, size_t length);
status_t memcpy_to_physical(addr_t to, const void* from, size_t length,
bool user);
status_t vm_memset_physical(addr_t address, int value, size_t length);
status_t vm_memcpy_from_physical(void* to, addr_t from, size_t length,
bool user);
status_t vm_memcpy_to_physical(addr_t to, const void* from, size_t length,
bool user);
void vm_memcpy_physical_page(addr_t to, addr_t from);
// user syscalls
area_id _user_create_area(const char *name, void **address, uint32 addressSpec,

27
headers/private/kernel/vm_translation_map.h
View File

@ -43,9 +43,32 @@ typedef struct vm_translation_map_ops {
uint32 attributes);
status_t (*clear_flags)(vm_translation_map *map, addr_t va, uint32 flags);
void (*flush)(vm_translation_map *map);
// get/put virtual address for physical page -- will be usuable on all CPUs
// (usually more expensive than the *_current_cpu() versions)
status_t (*get_physical_page)(addr_t physicalAddress,
addr_t *_virtualAddress, uint32 flags);
status_t (*put_physical_page)(addr_t virtualAddress);
addr_t *_virtualAddress, void **handle);
status_t (*put_physical_page)(addr_t virtualAddress, void *handle);
// get/put virtual address for physical page -- thread must be pinned the
// whole time
status_t (*get_physical_page_current_cpu)(addr_t physicalAddress,
addr_t *_virtualAddress, void **handle);
status_t (*put_physical_page_current_cpu)(addr_t virtualAddress,
void *handle);
// get/put virtual address for physical in KDL
status_t (*get_physical_page_debug)(addr_t physicalAddress,
addr_t *_virtualAddress, void **handle);
status_t (*put_physical_page_debug)(addr_t virtualAddress, void *handle);
// memory operations on pages
status_t (*memset_physical)(addr_t address, int value, size_t length);
status_t (*memcpy_from_physical)(void* to, addr_t from, size_t length,
bool user);
status_t (*memcpy_to_physical)(addr_t to, const void* from, size_t length,
bool user);
void (*memcpy_physical_page)(addr_t to, addr_t from);
} vm_translation_map_ops;
#include <arch/vm_translation_map.h>

5
headers/private/system/vm_defs.h
View File

@ -47,11 +47,6 @@
(B_KERNEL_PROTECTION | B_USER_CLONEABLE_AREA | B_OVERCOMMITTING_AREA \
| B_SHARED_AREA)
// flags for vm_get_physical_page()
enum {
PHYSICAL_PAGE_DONT_WAIT = 0x01
};
// mapping argument for several internal VM functions
enum {
REGION_NO_PRIVATE_MAP = 0,

24
src/add-ons/kernel/bus_managers/acpi/oshaiku.c
View File

@ -1158,16 +1158,10 @@ ACPI_STATUS
AcpiOsReadMemory(ACPI_PHYSICAL_ADDRESS Address, UINT32 *Value, UINT32 Width)
{
#ifdef _KERNEL_MODE
addr_t pageOffset = Address % B_PAGE_SIZE;
addr_t virtualAddress;
status_t error = vm_get_physical_page(Address - pageOffset,
&virtualAddress, 0);
DEBUG_FUNCTION();
if (error != B_OK)
if (vm_memcpy_from_physical(Value, (addr_t)Address, Width / 8, false)
!= B_OK) {
return AE_ERROR;
memcpy(Value, (void*)(virtualAddress + pageOffset), Width / 8);
vm_put_physical_page(virtualAddress);
}
return AE_OK;
#else
return AE_ERROR;
@ -1193,16 +1187,10 @@ ACPI_STATUS
AcpiOsWriteMemory(ACPI_PHYSICAL_ADDRESS Address, UINT32 Value, UINT32 Width)
{
#ifdef _KERNEL_MODE
addr_t pageOffset = Address % B_PAGE_SIZE;
addr_t virtualAddress;
status_t error = vm_get_physical_page(Address - pageOffset,
&virtualAddress, 0);
DEBUG_FUNCTION();
if (error != B_OK)
if (vm_memcpy_to_physical((addr_t)Address, &Value, Width / 8, false)
!= B_OK) {
return AE_ERROR;
memcpy((void*)(virtualAddress + pageOffset), &Value, Width / 8);
vm_put_physical_page(virtualAddress);
}
return AE_OK;
#else
return AE_ERROR;

25
src/add-ons/kernel/bus_managers/ata/emulation.c
View File

@ -33,7 +33,7 @@ copy_sg_data(scsi_ccb *ccb, uint offset, uint allocationLength,
int sgCount = ccb->sg_count;
int requestSize;
SHOW_FLOW(3, "offset=%u, req_size_limit=%d, size=%d, sg_list=%p, sg_cnt=%d, %s buffer",
SHOW_FLOW(3, "offset=%u, req_size_limit=%d, size=%d, sg_list=%p, sg_cnt=%d, %s buffer",
offset, allocationLength, size, sgList, sgCount, toBuffer ? "to" : "from");
// skip unused S/G entries
@ -46,30 +46,27 @@ copy_sg_data(scsi_ccb *ccb, uint offset, uint allocationLength,
if (sgCount == 0)
return 0;
// remaining bytes we are allowed to copy from/to ccb
// remaining bytes we are allowed to copy from/to ccb
requestSize = min(allocationLength, ccb->data_length) - offset;
// copy one S/G entry at a time
for (; size > 0 && requestSize > 0 && sgCount > 0; ++sgList, --sgCount) {
addr_t virtualAddress;
size_t bytes;
bytes = min(size, requestSize);
bytes = min(bytes, sgList->size);
if (vm_get_physical_page((addr_t)sgList->address, &virtualAddress,
0) != B_OK)
return false;
SHOW_FLOW(4, "buffer=%p, virt_addr=%p, bytes=%d, to_buffer=%d",
buffer, (void *)(virtualAddress + offset), (int)bytes, toBuffer);
buffer, (void *)((addr_t)sgList->address + offset), (int)bytes,
toBuffer);
if (toBuffer)
memcpy(buffer, (void *)(virtualAddress + offset), bytes);
else
memcpy((void *)(virtualAddress + offset), buffer, bytes);
vm_put_physical_page(virtualAddress);
if (toBuffer) {
vm_memcpy_from_physical(buffer, (addr_t)sgList->address + offset,
bytes, false);
} else {
vm_memcpy_to_physical((addr_t)sgList->address + offset, buffer,
bytes, false);
}
buffer = (char *)buffer + bytes;
size -= bytes;

42
src/add-ons/kernel/bus_managers/ata/pio.c
View File

@ -9,36 +9,37 @@
PIO data transmission
This file is more difficult then you might expect as the SCSI system
uses physical addresses everywhere which have to be mapped into
uses physical addresses everywhere which have to be mapped into
virtual address space during transmission. Additionally, during ATAPI
commands we may have to transmit more data then exist because the
data len specified by the command doesn't need to be the same as
data len specified by the command doesn't need to be the same as
of the data buffer provided.
The handling of S/G entries of odd size may look superfluous as the
The handling of S/G entries of odd size may look superfluous as the
SCSI bus manager can take care of that. In general, this would be possible
as most controllers need even alignment for DMA as well, but some can
handle _any_ S/G list and it wouldn't be sensitive to enforce stricter
alignement just for some rare PIO transmissions.
Little hint for the meaning of "transferred": this is the number of bytes
sent over the bus. For read-transmissions, this may be one more then copied
into the buffer (the extra byte read is stored in device->odd_byte), for
write-transmissions, this may be one less (the waiting byte is pending in
device->odd_byte).
In terms of error handling: we don't bother checking transmission of every
single byte via read/write_pio(). At least at the end of the request, when
single byte via read/write_pio(). At least at the end of the request, when
the status bits are verified, we will see that something has gone wrong.
TBD: S/G entries may have odd start address. For non-Intel architecture
we either have to copy data to an aligned buffer or have to modify
we either have to copy data to an aligned buffer or have to modify
PIO-handling in controller drivers.
*/
#include "ide_internal.h"
#include "ide_sim.h"
#include <thread.h>
#include <vm.h>
#include <string.h>
@ -74,7 +75,7 @@ transfer_PIO_virtcont(ide_device_info *device, uint8 *virtualAddress,
if (write) {
// if there is a byte left from last chunk, transmit it together
// with the first byte of the current chunk (IDE requires 16 bits
// with the first byte of the current chunk (IDE requires 16 bits
// to be transmitted at once)
if (device->has_odd_byte) {
uint8 buffer[2];
@ -91,7 +92,7 @@ transfer_PIO_virtcont(ide_device_info *device, uint8 *virtualAddress,
controller->write_pio(channel_cookie, (uint16 *)virtualAddress,
length / 2, false);
// take care if chunk size was odd, which means that 1 byte remains
// take care if chunk size was odd, which means that 1 byte remains
virtualAddress += length & ~1;
*transferred += length & ~1;
@ -112,7 +113,7 @@ transfer_PIO_virtcont(ide_device_info *device, uint8 *virtualAddress,
length / 2, false);
// take care of odd chunk size;
// in this case we read 1 byte to few!
// in this case we read 1 byte to few!
virtualAddress += length & ~1;
*transferred += length & ~1;
@ -145,14 +146,18 @@ transfer_PIO_physcont(ide_device_info *device, addr_t physicalAddress,
// one page into address space at once
while (length > 0) {
addr_t virtualAddress;
void* handle;
int page_left, cur_len;
status_t err;
struct thread* thread = thread_get_current_thread();
SHOW_FLOW(4, "Transmitting to/from physical address %lx, %d bytes left",
physicalAddress, length);
if (vm_get_physical_page(physicalAddress, &virtualAddress,
0) != B_OK) {
thread_pin_to_current_cpu(thread);
if (vm_get_physical_page_current_cpu(physicalAddress, &virtualAddress,
&handle) != B_OK) {
thread_unpin_from_current_cpu(thread);
// ouch: this should never ever happen
//xxx fix this set_sense(device, SCSIS_KEY_HARDWARE_ERROR, SCSIS_ASC_INTERNAL_FAILURE);
return B_ERROR;
@ -161,7 +166,7 @@ transfer_PIO_physcont(ide_device_info *device, addr_t physicalAddress,
ASSERT((physicalAddress & 4097) == (virtualAddress & 4097));
// if chunks starts in the middle of a page, we have even less then
// a page left
// a page left
page_left = B_PAGE_SIZE - physicalAddress % B_PAGE_SIZE;
SHOW_FLOW(4, "page_left=%d", page_left);
@ -173,12 +178,13 @@ transfer_PIO_physcont(ide_device_info *device, addr_t physicalAddress,
err = transfer_PIO_virtcont(device, (char *)virtualAddress,
cur_len, write, transferred);
vm_put_physical_page(virtualAddress);
vm_put_physical_page_current_cpu(virtualAddress);
thread_unpin_from_current_cpu(thread);
if (err != B_OK)
return err;
length -= cur_len;
length -= cur_len;
physicalAddress += cur_len;
}
@ -243,7 +249,7 @@ write_discard_PIO(ide_device_info *device, int length)
int cur_len;
// if device asks for odd number of bytes, append an extra byte to
// make length even (this is the "length + 1" term)
// make length even (this is the "length + 1" term)
cur_len = min(length + 1, (int)(sizeof(buffer))) / 2;
bus->controller->write_pio(bus->channel_cookie, (uint16 *)buffer, cur_len, false);

20
src/add-ons/kernel/bus_managers/ide/emulation.c
View File

@ -86,25 +86,21 @@ copy_sg_data(scsi_ccb *request, uint offset, uint allocationLength,
// copy one S/G entry at a time
for (; size > 0 && requestSize > 0 && sgCount > 0; ++sgList, --sgCount) {
addr_t virtualAddress;
size_t bytes;
bytes = min(size, requestSize);
bytes = min(bytes, sgList->size);
if (vm_get_physical_page((addr_t)sgList->address, &virtualAddress, 0)
!= B_OK)
return false;
SHOW_FLOW(4, "buffer=%p, virt_addr=%p, bytes=%d, to_buffer=%d",
buffer, (void *)(virtualAddress + offset), (int)bytes, toBuffer);
buffer, (void *)(sgList->address + offset), (int)bytes, toBuffer);
if (toBuffer)
memcpy(buffer, (void *)(virtualAddress + offset), bytes);
else
memcpy((void *)(virtualAddress + offset), buffer, bytes);
vm_put_physical_page(virtualAddress);
if (toBuffer) {
vm_memcpy_from_physical(buffer, (addr_t)sgList->address + offset,
bytes, false);
} else {
vm_memcpy_to_physical((addr_t)sgList->address + offset, buffer,
bytes, false);
}
buffer = (char *)buffer + bytes;
size -= bytes;

11
src/add-ons/kernel/bus_managers/ide/pio.c
View File

@ -39,6 +39,7 @@
#include "ide_internal.h"
#include "ide_sim.h"
#include <thread.h>
#include <vm.h>
#include <string.h>
@ -144,13 +145,18 @@ transfer_PIO_physcont(ide_device_info *device, addr_t physicalAddress,
// one page into address space at once
while (length > 0) {
addr_t virtualAddress;
void* handle;
int page_left, cur_len;
status_t err;
struct thread* thread = thread_get_current_thread();
SHOW_FLOW(4, "Transmitting to/from physical address %lx, %d bytes left",
physicalAddress, length);
if (vm_get_physical_page(physicalAddress, &virtualAddress, 0) != B_OK) {
thread_pin_to_current_cpu(thread);
if (vm_get_physical_page_current_cpu(physicalAddress, &virtualAddress,
&handle) != B_OK) {
thread_unpin_from_current_cpu(thread);
// ouch: this should never ever happen
set_sense(device, SCSIS_KEY_HARDWARE_ERROR, SCSIS_ASC_INTERNAL_FAILURE);
return B_ERROR;
@ -169,7 +175,8 @@ transfer_PIO_physcont(ide_device_info *device, addr_t physicalAddress,
err = transfer_PIO_virtcont(device, (uint8 *)virtualAddress,
cur_len, write, transferred);
vm_put_physical_page(virtualAddress);
vm_put_physical_page_current_cpu(virtualAddress, handle);
thread_unpin_from_current_cpu(thread);
if (err != B_OK)
return err;

18
src/add-ons/kernel/bus_managers/scsi/dma_buffer.c
View File

@ -114,21 +114,17 @@ scsi_copy_dma_buffer(scsi_ccb *request, uint32 size, bool to_buffer)
// was allocated in kernel and is thus visible even if the thread
// was changed
for (; size > 0 && num_vecs > 0; ++sg_list, --num_vecs) {
addr_t virtualAddress;
size_t bytes;
bytes = min( size, sg_list->size );
if (vm_get_physical_page((addr_t)sg_list->address, &virtualAddress, 0)
!= B_OK)
return false;
if (to_buffer)
memcpy(buffer_data, (void *)virtualAddress, bytes);
else
memcpy((void *)virtualAddress, buffer_data, bytes);
vm_put_physical_page(virtualAddress);
if (to_buffer) {
vm_memcpy_from_physical(buffer_data, (addr_t)sg_list->address,
bytes, false);
} else {
vm_memcpy_to_physical((addr_t)sg_list->address, buffer_data,
bytes, false);
}
buffer_data += bytes;
}

20
src/add-ons/kernel/bus_managers/scsi/emulation.c
View File

@ -459,24 +459,20 @@ copy_sg_data(scsi_ccb *request, uint offset, uint allocation_length,
// copy one S/G entry at a time
for (; size > 0 && req_size > 0 && sg_count > 0; ++sg_list, --sg_count) {
size_t bytes;
addr_t virtualAddress;
bytes = min(size, req_size);
bytes = min(bytes, sg_list->size);
if (vm_get_physical_page((addr_t)sg_list->address,
(void*)&virtualAddress, 0) != B_OK)
return false;
SHOW_FLOW(0, "buffer = %p, virt_addr = %#lx, bytes = %lu, to_buffer = %d",
buffer, virtualAddress + offset, bytes, to_buffer);
buffer, (addr_t)sg_list->address + offset, bytes, to_buffer);
if (to_buffer)
memcpy(buffer, (void *)(virtualAddress + offset), bytes);
else
memcpy((void *)(virtualAddress + offset), buffer, bytes);
vm_put_physical_page(virtualAddress);
if (to_buffer) {
vm_memcpy_from_physical(buffer, (addr_t)sg_list->address + offset,
bytes, false);
} else {
vm_memcpy_to_physical((addr_t)sg_list->address + offset, buffer,
bytes, false);
}
buffer = (char *)buffer + bytes;
size -= bytes;

10
src/add-ons/kernel/busses/scsi/ahci/util.c
View File

@ -91,16 +91,10 @@ sg_memcpy(const physical_entry *sgTable, int sgCount, const void *data,
int i;
for (i = 0; i < sgCount && dataSize > 0; i++) {
size_t size = min_c(dataSize, sgTable[i].size);
addr_t address;
if (vm_get_physical_page((addr_t)sgTable[i].address, &address, 0)
< B_OK)
return B_ERROR;
TRACE("sg_memcpy phyAddr %p, size %lu\n", sgTable[i].address, size);
TRACE("sg_memcpy phyAddr %p, addr %p, size %lu\n", sgTable[i].address, (void *)address, size);
memcpy((void *)address, data, size);
vm_put_physical_page(address);
vm_memcpy_to_physical((addr_t)sgTable[i].address, data, size, false);
data = (char *)data + size;
dataSize -= size;

8
src/system/kernel/arch/generic/generic_vm_physical_page_mapper.h
View File

@ -1,5 +1,5 @@
/*
* Copyright 2006, Ingo Weinhold <bonefish@cs.tu-berlin.de>.
* Copyright 2006-2008, Ingo Weinhold <bonefish@cs.tu-berlin.de>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
@ -8,6 +8,12 @@
#include <boot/kernel_args.h>
// flags for generic_get_physical_page()
enum {
PHYSICAL_PAGE_DONT_WAIT = 0x01
};
#ifdef __cplusplus
extern "C" {
#endif

151
src/system/kernel/arch/generic/generic_vm_physical_page_ops.cpp Normal file
View File

@ -0,0 +1,151 @@
/*
* Copyright 2008, Ingo Weinhold <ingo_weinhold@gmx.de>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
#include "generic_vm_physical_page_ops.h"
#include <vm.h>
#include <util/AutoLock.h>
status_t
generic_vm_memset_physical(addr_t address, int value, size_t length)
{
ThreadCPUPinner _(thread_get_current_thread());
while (length > 0) {
addr_t pageOffset = address % B_PAGE_SIZE;
addr_t virtualAddress;
void* handle;
status_t error = vm_get_physical_page_current_cpu(address - pageOffset,
&virtualAddress, &handle);
if (error != B_OK)
return error;
size_t toSet = min_c(length, B_PAGE_SIZE - pageOffset);
memset((void*)(virtualAddress + pageOffset), value, toSet);
vm_put_physical_page_current_cpu(virtualAddress, handle);
length -= toSet;
address += toSet;
}
return B_OK;
}
status_t
generic_vm_memcpy_from_physical(void* _to, addr_t from, size_t length,
bool user)
{
uint8* to = (uint8*)_to;
addr_t pageOffset = from % B_PAGE_SIZE;
ThreadCPUPinner _(thread_get_current_thread());
while (length > 0) {
size_t toCopy = min_c(length, B_PAGE_SIZE - pageOffset);
addr_t virtualAddress;
void* handle;
status_t error = vm_get_physical_page_current_cpu(from - pageOffset,
&virtualAddress, &handle);
if (error != B_OK)
return error;
if (user) {
error = user_memcpy(to, (void*)(virtualAddress + pageOffset),
toCopy);
} else
memcpy(to, (void*)(virtualAddress + pageOffset), toCopy);
vm_put_physical_page_current_cpu(virtualAddress, handle);
if (error != B_OK)
return error;
to += toCopy;
from += toCopy;
length -= toCopy;
pageOffset = 0;
}
return B_OK;
}
status_t
generic_vm_memcpy_to_physical(addr_t to, const void* _from, size_t length,
bool user)
{
const uint8* from = (const uint8*)_from;
addr_t pageOffset = to % B_PAGE_SIZE;
ThreadCPUPinner _(thread_get_current_thread());
while (length > 0) {
size_t toCopy = min_c(length, B_PAGE_SIZE - pageOffset);
addr_t virtualAddress;
void* handle;
status_t error = vm_get_physical_page_current_cpu(to - pageOffset,
&virtualAddress, &handle);
if (error != B_OK)
return error;
if (user) {
error = user_memcpy((void*)(virtualAddress + pageOffset), from,
toCopy);
} else
memcpy((void*)(virtualAddress + pageOffset), from, toCopy);
vm_put_physical_page_current_cpu(virtualAddress, handle);
if (error != B_OK)
return error;
to += toCopy;
from += toCopy;
length -= toCopy;
pageOffset = 0;
}
return B_OK;
}
/*! NOTE: If this function is used, vm_get_physical_page_current_cpu() must not
be blocking, since we need to call it twice and could thus deadlock.
*/
void
generic_vm_memcpy_physical_page(addr_t to, addr_t from)
{
ThreadCPUPinner _(thread_get_current_thread());
// map source page
addr_t fromVirtual;
void* fromHandle;
status_t error = vm_get_physical_page_current_cpu(from, &fromVirtual,
&fromHandle);
if (error != B_OK) {
panic("generic_vm_memcpy_physical_page(): Failed to map source page!");
return;
}
// map destination page
addr_t toVirtual;
void* toHandle;
error = vm_get_physical_page_current_cpu(to, &toVirtual, &toHandle);
if (error == B_OK) {
// both pages are mapped -- copy
memcpy((void*)toVirtual, (const void*)fromVirtual, B_PAGE_SIZE);
vm_put_physical_page_current_cpu(toVirtual, toHandle);
} else {
panic("generic_vm_memcpy_physical_page(): Failed to map destination "
"page!");
}
vm_put_physical_page_current_cpu(fromVirtual, fromHandle);
}

25
src/system/kernel/arch/generic/generic_vm_physical_page_ops.h Normal file
View File

@ -0,0 +1,25 @@
/*
* Copyright 2008, Ingo Weinhold <ingo_weinhold@gmx.de>.
* All rights reserved. Distributed under the terms of the MIT License.
*/
#ifndef _KERNEL_GENERIC_VM_PHYSICAL_PAGE_OPS_H
#define _KERNEL_GENERIC_VM_PHYSICAL_PAGE_OPS_H
#include <SupportDefs.h>
#ifdef __cplusplus
extern "C" {
#endif
status_t generic_vm_memset_physical(addr_t address, int value, size_t length);
status_t generic_vm_memcpy_from_physical(void* to, addr_t from, size_t length,
bool user);
status_t generic_vm_memcpy_to_physical(addr_t to, const void* from,
size_t length, bool user);
void generic_vm_memcpy_physical_page(addr_t to, addr_t from);
#ifdef __cplusplus
}
#endif
#endif // _KERNEL_GENERIC_VM_PHYSICAL_PAGE_OPS_H

1
src/system/kernel/arch/m68k/Jamfile
View File

@ -47,6 +47,7 @@ KernelMergeObject kernel_arch_m68k.o :
arch_asm.S
generic_vm_physical_page_mapper.cpp
generic_vm_physical_page_ops.cpp
:
$(TARGET_KERNEL_PIC_CCFLAGS) -Wno-unused

110
src/system/kernel/arch/m68k/arch_vm_translation_map_impl.cpp
View File

@ -55,7 +55,7 @@
#include <stdlib.h>
#include "generic_vm_physical_page_mapper.h"
#include "generic_vm_physical_page_ops.h"
#define TRACE_VM_TMAP
@ -119,8 +119,8 @@ static addr_t sIOSpaceBase;
#define IS_KERNEL_MAP(map) (map->arch_data->rtdir_phys == sKernelPhysicalPageRoot)
static status_t early_query(addr_t va, addr_t *out_physical);
static status_t get_physical_page_tmap(addr_t pa, addr_t *va, uint32 flags);
static status_t put_physical_page_tmap(addr_t va);
static status_t get_physical_page_tmap_internal(addr_t pa, addr_t *va, uint32 flags);
static status_t put_physical_page_tmap_internal(addr_t va);
static void flush_tmap(vm_translation_map *map);
@ -571,7 +571,7 @@ map_tmap(vm_translation_map *map, addr_t va, addr_t pa, uint32 attributes)
}
// now, fill in the pentry
do {
err = get_physical_page_tmap(PRE_TO_PA(pr[rindex]),
err = get_physical_page_tmap_internal(PRE_TO_PA(pr[rindex]),
&pd_pg, PHYSICAL_PAGE_DONT_WAIT);
} while (err < 0);
pd = (page_directory_entry *)pd_pg;
@ -615,7 +615,7 @@ map_tmap(vm_translation_map *map, addr_t va, addr_t pa, uint32 attributes)
}
// now, fill in the pentry
do {
err = get_physical_page_tmap(PDE_TO_PA(pd[dindex]),
err = get_physical_page_tmap_internal(PDE_TO_PA(pd[dindex]),
&pt_pg, PHYSICAL_PAGE_DONT_WAIT);
} while (err < 0);
pt = (page_table_entry *)pt_pg;
@ -627,8 +627,8 @@ map_tmap(vm_translation_map *map, addr_t va, addr_t pa, uint32 attributes)
put_page_table_entry_in_pgtable(&pt[pindex], pa, attributes,
IS_KERNEL_MAP(map));
put_physical_page_tmap(pt_pg);
put_physical_page_tmap(pd_pg);
put_physical_page_tmap_internal(pt_pg);
put_physical_page_tmap_internal(pd_pg);
if (map->arch_data->num_invalidate_pages < PAGE_INVALIDATE_CACHE_SIZE)
map->arch_data->pages_to_invalidate[map->arch_data->num_invalidate_pages] = va;
@ -668,7 +668,7 @@ restart:
}
do {
status = get_physical_page_tmap(PRE_TO_PA(pr[index]),
status = get_physical_page_tmap_internal(PRE_TO_PA(pr[index]),
&pd_pg, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
pd = (page_directory_entry *)pd_pg;
@ -679,12 +679,12 @@ restart:
if (pd[index].type != DT_DIR) {
// no pagetable here, move the start up to access the next page table
start = ROUNDUP(start + 1, B_PAGE_SIZE);
put_physical_page_tmap(pd_pg);
put_physical_page_tmap_internal(pd_pg);
goto restart;
}
do {
status = get_physical_page_tmap(PDE_TO_PA(pd[index]),
status = get_physical_page_tmap_internal(PDE_TO_PA(pd[index]),
&pt_pg, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
pt = (page_table_entry *)pt_pg;
@ -710,8 +710,8 @@ restart:
map->arch_data->num_invalidate_pages++;
}
put_physical_page_tmap(pt_pg);
put_physical_page_tmap(pd_pg);
put_physical_page_tmap_internal(pt_pg);
put_physical_page_tmap_internal(pd_pg);
goto restart;
}
@ -795,7 +795,7 @@ query_tmap(vm_translation_map *map, addr_t va, addr_t *_physical, uint32 *_flags
}
do {
status = get_physical_page_tmap(PRE_TO_PA(pr[index]),
status = get_physical_page_tmap_internal(PRE_TO_PA(pr[index]),
&pd_pg, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
pd = (page_directory_entry *)pd_pg;
@ -806,12 +806,12 @@ query_tmap(vm_translation_map *map, addr_t va, addr_t *_physical, uint32 *_flags
index = VADDR_TO_PDENT(va);
if (pd[index].type != DT_DIR) {
// no pagetable here
put_physical_page_tmap(pd_pg);
put_physical_page_tmap_internal(pd_pg);
return B_NO_ERROR;
}
do {
status = get_physical_page_tmap(PDE_TO_PA(pd[index]),
status = get_physical_page_tmap_internal(PDE_TO_PA(pd[index]),
&pt_pg, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
pt = (page_table_entry *)pt_pg;
@ -825,14 +825,14 @@ query_tmap(vm_translation_map *map, addr_t va, addr_t *_physical, uint32 *_flags
pi = (page_indirect_entry *)pt;
pi_pg = pt_pg;
do {
status = get_physical_page_tmap(PIE_TO_PA(pi[index]),
status = get_physical_page_tmap_internal(PIE_TO_PA(pi[index]),
&pt_pg, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
pt = (page_table_entry *)pt_pg;
// add offset from start of page
pt += PIE_TO_PO(pi[index]) / sizeof(page_table_entry);
// release the indirect table page
put_physical_page_tmap(pi_pg);
put_physical_page_tmap_internal(pi_pg);
}
*_physical = PTE_TO_PA(pt[index]);
@ -847,8 +847,8 @@ query_tmap(vm_translation_map *map, addr_t va, addr_t *_physical, uint32 *_flags
| (pt[index].accessed ? PAGE_ACCESSED : 0)
| ((pt[index].type == DT_PAGE) ? PAGE_PRESENT : 0);
put_physical_page_tmap(pt_pg);
put_physical_page_tmap(pd_pg);
put_physical_page_tmap_internal(pt_pg);
put_physical_page_tmap_internal(pd_pg);
TRACE(("query_tmap: returning pa 0x%lx for va 0x%lx\n", *_physical, va));
@ -890,7 +890,7 @@ restart:
}
do {
status = get_physical_page_tmap(PRE_TO_PA(pr[index]),
status = get_physical_page_tmap_internal(PRE_TO_PA(pr[index]),
&pd_pg, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
pd = (page_directory_entry *)pd_pg;
@ -901,12 +901,12 @@ restart:
if (pd[index].type != DT_DIR) {
// no pagetable here, move the start up to access the next page table
start = ROUNDUP(start + 1, B_PAGE_SIZE);
put_physical_page_tmap(pd_pg);
put_physical_page_tmap_internal(pd_pg);
goto restart;
}
do {
status = get_physical_page_tmap(PDE_TO_PA(pd[index]),
status = get_physical_page_tmap_internal(PDE_TO_PA(pd[index]),
&pt_pg, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
pt = (page_table_entry *)pt_pg;
@ -936,8 +936,8 @@ restart:
map->arch_data->num_invalidate_pages++;
}
put_physical_page_tmap(pt_pg);
put_physical_page_tmap(pd_pg);
put_physical_page_tmap_internal(pt_pg);
put_physical_page_tmap_internal(pd_pg);
goto restart;
}
@ -962,7 +962,7 @@ clear_flags_tmap(vm_translation_map *map, addr_t va, uint32 flags)
}
do {
status = get_physical_page_tmap(PRE_TO_PA(pr[index]),
status = get_physical_page_tmap_internal(PRE_TO_PA(pr[index]),
&pd_pg, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
pd = (page_directory_entry *)pd_pg;
@ -973,12 +973,12 @@ clear_flags_tmap(vm_translation_map *map, addr_t va, uint32 flags)
index = VADDR_TO_PDENT(va);
if (pd[index].type != DT_DIR) {
// no pagetable here
put_physical_page_tmap(pd_pg);
put_physical_page_tmap_internal(pd_pg);
return B_NO_ERROR;
}
do {
status = get_physical_page_tmap(PDE_TO_PA(pd[index]),
status = get_physical_page_tmap_internal(PDE_TO_PA(pd[index]),
&pt_pg, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
pt = (page_table_entry *)pt_pg;
@ -992,14 +992,14 @@ clear_flags_tmap(vm_translation_map *map, addr_t va, uint32 flags)
pi = (page_indirect_entry *)pt;
pi_pg = pt_pg;
do {
status = get_physical_page_tmap(PIE_TO_PA(pi[index]),
status = get_physical_page_tmap_internal(PIE_TO_PA(pi[index]),
&pt_pg, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
pt = (page_table_entry *)pt_pg;
// add offset from start of page
pt += PIE_TO_PO(pi[index]) / sizeof(page_table_entry);
// release the indirect table page
put_physical_page_tmap(pi_pg);
put_physical_page_tmap_internal(pi_pg);
}
// clear out the flags we've been requested to clear
@ -1012,8 +1012,8 @@ clear_flags_tmap(vm_translation_map *map, addr_t va, uint32 flags)
tlb_flush = true;
}
put_physical_page_tmap(pt_pg);
put_physical_page_tmap(pd_pg);
put_physical_page_tmap_internal(pt_pg);
put_physical_page_tmap_internal(pd_pg);
if (tlb_flush) {
if (map->arch_data->num_invalidate_pages < PAGE_INVALIDATE_CACHE_SIZE)
@ -1097,19 +1097,34 @@ map_iospace_chunk(addr_t va, addr_t pa, uint32 flags)
static status_t
get_physical_page_tmap(addr_t pa, addr_t *va, uint32 flags)
get_physical_page_tmap_internal(addr_t pa, addr_t *va, uint32 flags)
{
return generic_get_physical_page(pa, va, flags);
}
static status_t
put_physical_page_tmap(addr_t va)
put_physical_page_tmap_internal(addr_t va)
{
return generic_put_physical_page(va);
}
static status_t
get_physical_page_tmap(addr_t physicalAddress, addr_t *_virtualAddress,
void **handle)
{
return generic_get_physical_page(physicalAddress, _virtualAddress, 0);
}
static status_t
put_physical_page_tmap(addr_t virtualAddress, void *handle)
{
return generic_put_physical_page(virtualAddress);
}
static vm_translation_map_ops tmap_ops = {
destroy_tmap,
lock_tmap,
@ -1124,7 +1139,18 @@ static vm_translation_map_ops tmap_ops = {
clear_flags_tmap,
flush_tmap,
get_physical_page_tmap,
put_physical_page_tmap
put_physical_page_tmap,
get_physical_page_tmap, // *_current_cpu()
put_physical_page_tmap, // *_current_cpu()
get_physical_page_tmap, // *_debug()
put_physical_page_tmap, // *_debug()
// TODO: Replace the *_current_cpu() and *_debug() versions!
generic_vm_memset_physical,
generic_vm_memcpy_from_physical,
generic_vm_memcpy_to_physical,
generic_vm_memcpy_physical_page
// TODO: Verify that this is safe to use!
};
@ -1354,13 +1380,13 @@ m68k_vm_translation_map_init_post_area(kernel_args *args)
index = VADDR_TO_PRENT((addr_t)&sQueryDesc);
physicalPageDir = PRE_TO_PA(sKernelVirtualPageRoot[index]);
get_physical_page_tmap(physicalPageDir,
get_physical_page_tmap_internal(physicalPageDir,
(addr_t *)&pageDirEntry, PHYSICAL_PAGE_DONT_WAIT);
index = VADDR_TO_PDENT((addr_t)&sQueryDesc);
physicalPageTable = PDE_TO_PA(pageDirEntry[index]);
get_physical_page_tmap(physicalPageTable,
get_physical_page_tmap_internal(physicalPageTable,
(addr_t *)&pageTableEntry, PHYSICAL_PAGE_DONT_WAIT);
index = VADDR_TO_PTENT((addr_t)&sQueryDesc);
@ -1370,8 +1396,8 @@ m68k_vm_translation_map_init_post_area(kernel_args *args)
// add offset
physicalIndirectDesc += ((addr_t)&sQueryDesc) % B_PAGE_SIZE;
put_physical_page_tmap((addr_t)pageTableEntry);
put_physical_page_tmap((addr_t)pageDirEntry);
put_physical_page_tmap_internal((addr_t)pageTableEntry);
put_physical_page_tmap_internal((addr_t)pageDirEntry);
// then the va for the page table for the query page.
@ -1380,13 +1406,13 @@ m68k_vm_translation_map_init_post_area(kernel_args *args)
index = VADDR_TO_PRENT(queryPage);
physicalPageDir = PRE_TO_PA(sKernelVirtualPageRoot[index]);
get_physical_page_tmap(physicalPageDir,
get_physical_page_tmap_internal(physicalPageDir,
(addr_t *)&pageDirEntry, PHYSICAL_PAGE_DONT_WAIT);
index = VADDR_TO_PDENT(queryPage);
physicalPageTable = PDE_TO_PA(pageDirEntry[index]);
get_physical_page_tmap(physicalPageTable,
get_physical_page_tmap_internal(physicalPageTable,
(addr_t *)&pageTableEntry, PHYSICAL_PAGE_DONT_WAIT);
index = VADDR_TO_PTENT(queryPage);
@ -1394,8 +1420,8 @@ m68k_vm_translation_map_init_post_area(kernel_args *args)
put_page_indirect_entry_in_pgtable(&pageTableEntry[index], physicalIndirectDesc,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, false);
put_physical_page_tmap((addr_t)pageTableEntry);
put_physical_page_tmap((addr_t)pageDirEntry);
put_physical_page_tmap_internal((addr_t)pageTableEntry);
put_physical_page_tmap_internal((addr_t)pageDirEntry);
//invalidate_TLB(sQueryPageTable);
}
// qmery_tmap_interrupt checks for the NULL, now it can use it

1
src/system/kernel/arch/ppc/Jamfile
View File

@ -28,6 +28,7 @@ KernelMergeObject kernel_arch_ppc.o :
arch_asm.S
generic_vm_physical_page_mapper.cpp
generic_vm_physical_page_ops.cpp
:
$(TARGET_KERNEL_PIC_CCFLAGS) -Wno-unused
;

24
src/system/kernel/arch/ppc/arch_vm_translation_map.cpp
View File

@ -84,6 +84,8 @@
#include <stdlib.h>
#include "generic_vm_physical_page_mapper.h"
#include "generic_vm_physical_page_ops.h"
static struct page_table_entry_group *sPageTable;
static size_t sPageTableSize;
@ -444,16 +446,17 @@ flush_tmap(vm_translation_map *map)
static status_t
get_physical_page_tmap(addr_t pa, addr_t *va, uint32 flags)
get_physical_page_tmap(addr_t physicalAddress, addr_t *_virtualAddress,
void **handle)
{
return generic_get_physical_page(pa, va, flags);
return generic_get_physical_page(physicalAddress, _virtualAddress, 0);
}
static status_t
put_physical_page_tmap(addr_t va)
put_physical_page_tmap(addr_t virtualAddress, void *handle)
{
return generic_put_physical_page(va);
return generic_put_physical_page(virtualAddress);
}
@ -471,7 +474,18 @@ static vm_translation_map_ops tmap_ops = {
clear_flags_tmap,
flush_tmap,
get_physical_page_tmap,
put_physical_page_tmap
put_physical_page_tmap,
get_physical_page_tmap, // *_current_cpu()
put_physical_page_tmap, // *_current_cpu()
get_physical_page_tmap, // *_debug()
put_physical_page_tmap, // *_debug()
// TODO: Replace the *_current_cpu() and *_debug() versions!
generic_vm_memset_physical,
generic_vm_memcpy_from_physical,
generic_vm_memcpy_to_physical,
generic_vm_memcpy_physical_page
// TODO: Verify that this is safe to use!
};

8
src/system/kernel/arch/x86/Jamfile
View File

@ -3,12 +3,10 @@ SubDir HAIKU_TOP src system kernel arch x86 ;
SubDirHdrs [ FDirName $(TARGET_COMMON_DEBUG_OBJECT_DIR) system kernel ] ;
# for syscall_numbers.h
SubDirHdrs $(HAIKU_TOP) src add-ons kernel bus_managers ps2 ;
SubDirHdrs $(SUBDIR) $(DOTDOT) generic ;
SubDirHdrs $(SUBDIR) timers ;
UsePrivateKernelHeaders ;
SEARCH_SOURCE += [ FDirName $(SUBDIR) $(DOTDOT) generic ] ;
UsePrivateHeaders shared ;
SEARCH_SOURCE += [ FDirName $(SUBDIR) timers ] ;
@ -36,12 +34,12 @@ KernelMergeObject kernel_arch_x86.o :
cpuid.S
syscall.S
vm86.cpp
x86_physical_page_mapper.cpp
x86_physical_page_mapper_large_memory.cpp
x86_pit.c
x86_apic.c
x86_hpet.c
generic_vm_physical_page_mapper.cpp
:
$(TARGET_KERNEL_PIC_CCFLAGS)
;

4
src/system/kernel/arch/x86/arch_thread.cpp
View File

@ -8,6 +8,8 @@
#include <arch/thread.h>
#include <string.h>
#include <arch/user_debugger.h>
#include <arch_cpu.h>
#include <debug.h>
@ -20,7 +22,7 @@
#include <vm_address_space.h>
#include <vm_types.h>
#include <string.h>
#include "x86_paging.h"
//#define TRACE_ARCH_THREAD

2
src/system/kernel/arch/x86/arch_vm.cpp
View File

@ -26,6 +26,8 @@
#include <arch/x86/bios.h>
#include "x86_paging.h"
//#define TRACE_ARCH_VM
#ifdef TRACE_ARCH_VM

385
src/system/kernel/arch/x86/arch_vm_translation_map.cpp
View File

@ -1,4 +1,5 @@
/*
* Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de.
* Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
* Distributed under the terms of the MIT License.
*
@ -6,22 +7,27 @@
* Distributed under the terms of the NewOS License.
*/
#include <arch/vm_translation_map.h>
#include <stdlib.h>
#include <string.h>
#include <AutoDeleter.h>
#include <arch_system_info.h>
#include <heap.h>
#include <int.h>
#include <thread.h>
#include <smp.h>
#include <util/AutoLock.h>
#include <util/queue.h>
#include <vm_address_space.h>
#include <vm_page.h>
#include <vm_priv.h>
#include <int.h>
#include <smp.h>
#include <util/queue.h>
#include <heap.h>
#include <arch_system_info.h>
#include <arch/vm_translation_map.h>
#include <thread.h>
#include <string.h>
#include <stdlib.h>
#include "x86_paging.h"
#include "x86_physical_page_mapper.h"
#include "generic_vm_physical_page_mapper.h"
//#define TRACE_VM_TMAP
#ifdef TRACE_VM_TMAP
@ -30,60 +36,16 @@
# define TRACE(x) ;
#endif
// 256 MB of iospace
#define IOSPACE_SIZE (256*1024*1024)
// 4 MB chunks, to optimize for 4 MB pages
#define IOSPACE_CHUNK_SIZE (4*1024*1024)
typedef struct page_table_entry {
uint32 present:1;
uint32 rw:1;
uint32 user:1;
uint32 write_through:1;
uint32 cache_disabled:1;
uint32 accessed:1;
uint32 dirty:1;
uint32 reserved:1;
uint32 global:1;
uint32 avail:3;
uint32 addr:20;
} page_table_entry;
typedef struct page_directory_entry {
uint32 present:1;
uint32 rw:1;
uint32 user:1;
uint32 write_through:1;
uint32 cache_disabled:1;
uint32 accessed:1;
uint32 reserved:1;
uint32 page_size:1;
uint32 global:1;
uint32 avail:3;
uint32 addr:20;
} page_directory_entry;
static page_table_entry *iospace_pgtables = NULL;
static page_table_entry *page_hole = NULL;
static page_directory_entry *page_hole_pgdir = NULL;
static page_directory_entry *sKernelPhysicalPageDirectory = NULL;
static page_directory_entry *sKernelVirtualPageDirectory = NULL;
static addr_t sQueryPages;
static page_table_entry *sQueryPageTable;
static vm_translation_map *tmap_list;
static spinlock tmap_list_lock;
static addr_t sIOSpaceBase;
#define CHATTY_TMAP 0
#define ADDR_SHIFT(x) ((x)>>12)
#define ADDR_REVERSE_SHIFT(x) ((x)<<12)
#define VADDR_TO_PDENT(va) (((va) / B_PAGE_SIZE) / 1024)
#define VADDR_TO_PTENT(va) (((va) / B_PAGE_SIZE) % 1024)
#define FIRST_USER_PGDIR_ENT (VADDR_TO_PDENT(USER_BASE))
#define NUM_USER_PGDIR_ENTS (VADDR_TO_PDENT(ROUNDUP(USER_SIZE, B_PAGE_SIZE * 1024)))
#define FIRST_KERNEL_PGDIR_ENT (VADDR_TO_PDENT(KERNEL_BASE))
@ -91,8 +53,6 @@ static addr_t sIOSpaceBase;
#define IS_KERNEL_MAP(map) (map->arch_data->pgdir_phys == sKernelPhysicalPageDirectory)
static status_t early_query(addr_t va, addr_t *out_physical);
static status_t get_physical_page_tmap(addr_t pa, addr_t *va, uint32 flags);
static status_t put_physical_page_tmap(addr_t va);
static void flush_tmap(vm_translation_map *map);
@ -104,38 +64,8 @@ i386_translation_map_get_pgdir(vm_translation_map *map)
}
static inline void
init_page_directory_entry(page_directory_entry *entry)
{
*(uint32 *)entry = 0;
}
static inline void
update_page_directory_entry(page_directory_entry *entry, page_directory_entry *with)
{
// update page directory entry atomically
*(uint32 *)entry = *(uint32 *)with;
}
static inline void
init_page_table_entry(page_table_entry *entry)
{
*(uint32 *)entry = 0;
}
static inline void
update_page_table_entry(page_table_entry *entry, page_table_entry *with)
{
// update page table entry atomically
*(uint32 *)entry = *(uint32 *)with;
}
static void
_update_all_pgdirs(int index, page_directory_entry e)
void
x86_update_all_pgdirs(int index, page_directory_entry e)
{
vm_translation_map *entry;
unsigned int state = disable_interrupts();
@ -226,6 +156,7 @@ destroy_tmap(vm_translation_map *map)
state = disable_interrupts();
acquire_spinlock(&tmap_list_lock);
// TODO: How about using a doubly linked list?
entry = tmap_list;
while (entry != NULL) {
if (entry == map) {
@ -243,6 +174,9 @@ destroy_tmap(vm_translation_map *map)
release_spinlock(&tmap_list_lock);
restore_interrupts(state);
if (map->arch_data->page_mapper != NULL)
map->arch_data->page_mapper->Delete();
if (map->arch_data->pgdir_virt != NULL) {
// cycle through and free all of the user space pgtables
for (i = VADDR_TO_PDENT(USER_BASE); i <= VADDR_TO_PDENT(USER_BASE + (USER_SIZE - 1)); i++) {
@ -265,8 +199,8 @@ destroy_tmap(vm_translation_map *map)
}
static void
put_pgtable_in_pgdir(page_directory_entry *entry,
void
x86_put_pgtable_in_pgdir(page_directory_entry *entry,
addr_t pgtable_phys, uint32 attributes)
{
page_directory_entry table;
@ -334,7 +268,6 @@ map_tmap(vm_translation_map *map, addr_t va, addr_t pa, uint32 attributes)
page_directory_entry *pd;
page_table_entry *pt;
unsigned int index;
int err;
TRACE(("map_tmap: entry pa 0x%lx va 0x%lx\n", pa, va));
@ -365,28 +298,29 @@ map_tmap(vm_translation_map *map, addr_t va, addr_t pa, uint32 attributes)
TRACE(("map_tmap: asked for free page for pgtable. 0x%lx\n", pgtable));
// put it in the pgdir
put_pgtable_in_pgdir(&pd[index], pgtable, attributes
x86_put_pgtable_in_pgdir(&pd[index], pgtable, attributes
| (attributes & B_USER_PROTECTION ? B_WRITE_AREA : B_KERNEL_WRITE_AREA));
// update any other page directories, if it maps kernel space
if (index >= FIRST_KERNEL_PGDIR_ENT
&& index < (FIRST_KERNEL_PGDIR_ENT + NUM_KERNEL_PGDIR_ENTS))
_update_all_pgdirs(index, pd[index]);
x86_update_all_pgdirs(index, pd[index]);
map->map_count++;
}
// now, fill in the pentry
do {
err = get_physical_page_tmap(ADDR_REVERSE_SHIFT(pd[index].addr),
(addr_t *)&pt, PHYSICAL_PAGE_DONT_WAIT);
} while (err < 0);
struct thread* thread = thread_get_current_thread();
ThreadCPUPinner pinner(thread);
pt = map->arch_data->page_mapper->GetPageTableAt(
ADDR_REVERSE_SHIFT(pd[index].addr));
index = VADDR_TO_PTENT(va);
put_page_table_entry_in_pgtable(&pt[index], pa, attributes,
IS_KERNEL_MAP(map));
put_physical_page_tmap((addr_t)pt);
pinner.Unlock();
if (map->arch_data->num_invalidate_pages < PAGE_INVALIDATE_CACHE_SIZE)
map->arch_data->pages_to_invalidate[map->arch_data->num_invalidate_pages] = va;
@ -404,7 +338,6 @@ unmap_tmap(vm_translation_map *map, addr_t start, addr_t end)
{
page_table_entry *pt;
page_directory_entry *pd = map->arch_data->pgdir_virt;
status_t status;
int index;
start = ROUNDOWN(start, B_PAGE_SIZE);
@ -423,10 +356,11 @@ restart:
goto restart;
}
do {
status = get_physical_page_tmap(ADDR_REVERSE_SHIFT(pd[index].addr),
(addr_t *)&pt, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
struct thread* thread = thread_get_current_thread();
ThreadCPUPinner pinner(thread);
pt = map->arch_data->page_mapper->GetPageTableAt(
ADDR_REVERSE_SHIFT(pd[index].addr));
for (index = VADDR_TO_PTENT(start); (index < 1024) && (start < end);
index++, start += B_PAGE_SIZE) {
@ -446,7 +380,7 @@ restart:
map->arch_data->num_invalidate_pages++;
}
put_physical_page_tmap((addr_t)pt);
pinner.Unlock();
goto restart;
}
@ -459,7 +393,6 @@ query_tmap_interrupt(vm_translation_map *map, addr_t va, addr_t *_physical,
page_directory_entry *pd = map->arch_data->pgdir_virt;
page_table_entry *pt;
addr_t physicalPageTable;
int32 cpu = smp_get_current_cpu();
int32 index;
*_physical = 0;
@ -470,20 +403,9 @@ query_tmap_interrupt(vm_translation_map *map, addr_t va, addr_t *_physical,
return B_ERROR;
}
// map page table entry using our per CPU mapping page
// map page table entry
physicalPageTable = ADDR_REVERSE_SHIFT(pd[index].addr);
pt = (page_table_entry *)(sQueryPages + cpu * B_PAGE_SIZE);
index = VADDR_TO_PDENT((addr_t)pt);
if (pd[index].present == 0) {
// no page table here
return B_ERROR;
}
index = VADDR_TO_PTENT((addr_t)pt);
put_page_table_entry_in_pgtable(&sQueryPageTable[index], physicalPageTable,
B_KERNEL_READ_AREA, false);
invalidate_TLB(pt);
pt = gPhysicalPageMapper->InterruptGetPageTableAt(physicalPageTable);
index = VADDR_TO_PTENT(va);
*_physical = ADDR_REVERSE_SHIFT(pt[index].addr);
@ -502,7 +424,6 @@ query_tmap(vm_translation_map *map, addr_t va, addr_t *_physical, uint32 *_flags
{
page_table_entry *pt;
page_directory_entry *pd = map->arch_data->pgdir_virt;
status_t status;
int32 index;
// default the flags to not present
@ -515,10 +436,11 @@ query_tmap(vm_translation_map *map, addr_t va, addr_t *_physical, uint32 *_flags
return B_NO_ERROR;
}
do {
status = get_physical_page_tmap(ADDR_REVERSE_SHIFT(pd[index].addr),
(addr_t *)&pt, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
struct thread* thread = thread_get_current_thread();
ThreadCPUPinner pinner(thread);
pt = map->arch_data->page_mapper->GetPageTableAt(
ADDR_REVERSE_SHIFT(pd[index].addr));
index = VADDR_TO_PTENT(va);
*_physical = ADDR_REVERSE_SHIFT(pt[index].addr);
@ -532,7 +454,7 @@ query_tmap(vm_translation_map *map, addr_t va, addr_t *_physical, uint32 *_flags
| (pt[index].accessed ? PAGE_ACCESSED : 0)
| (pt[index].present ? PAGE_PRESENT : 0);
put_physical_page_tmap((addr_t)pt);
pinner.Unlock();
TRACE(("query_tmap: returning pa 0x%lx for va 0x%lx\n", *_physical, va));
@ -552,7 +474,6 @@ protect_tmap(vm_translation_map *map, addr_t start, addr_t end, uint32 attribute
{
page_table_entry *pt;
page_directory_entry *pd = map->arch_data->pgdir_virt;
status_t status;
int index;
start = ROUNDOWN(start, B_PAGE_SIZE);
@ -571,10 +492,11 @@ restart:
goto restart;
}
do {
status = get_physical_page_tmap(ADDR_REVERSE_SHIFT(pd[index].addr),
(addr_t *)&pt, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
struct thread* thread = thread_get_current_thread();
ThreadCPUPinner pinner(thread);
pt = map->arch_data->page_mapper->GetPageTableAt(
ADDR_REVERSE_SHIFT(pd[index].addr));
for (index = VADDR_TO_PTENT(start); index < 1024 && start < end; index++, start += B_PAGE_SIZE) {
if (pt[index].present == 0) {
@ -596,7 +518,7 @@ restart:
map->arch_data->num_invalidate_pages++;
}
put_physical_page_tmap((addr_t)pt);
pinner.Unlock();
goto restart;
}
@ -607,7 +529,6 @@ clear_flags_tmap(vm_translation_map *map, addr_t va, uint32 flags)
{
page_table_entry *pt;
page_directory_entry *pd = map->arch_data->pgdir_virt;
status_t status;
int index;
int tlb_flush = false;
@ -617,10 +538,11 @@ clear_flags_tmap(vm_translation_map *map, addr_t va, uint32 flags)
return B_OK;
}
do {
status = get_physical_page_tmap(ADDR_REVERSE_SHIFT(pd[index].addr),
(addr_t *)&pt, PHYSICAL_PAGE_DONT_WAIT);
} while (status < B_OK);
struct thread* thread = thread_get_current_thread();
ThreadCPUPinner pinner(thread);
pt = map->arch_data->page_mapper->GetPageTableAt(
ADDR_REVERSE_SHIFT(pd[index].addr));
index = VADDR_TO_PTENT(va);
// clear out the flags we've been requested to clear
@ -633,7 +555,7 @@ clear_flags_tmap(vm_translation_map *map, addr_t va, uint32 flags)
tlb_flush = true;
}
put_physical_page_tmap((addr_t)pt);
pinner.Unlock();
if (tlb_flush) {
if (map->arch_data->num_invalidate_pages < PAGE_INVALIDATE_CACHE_SIZE)
@ -705,55 +627,6 @@ flush_tmap(vm_translation_map *map)
}
static status_t
map_iospace_chunk(addr_t va, addr_t pa, uint32 flags)
{
int i;
page_table_entry *pt;
addr_t ppn;
pa &= ~(B_PAGE_SIZE - 1); // make sure it's page aligned
va &= ~(B_PAGE_SIZE - 1); // make sure it's page aligned
if (va < sIOSpaceBase || va >= (sIOSpaceBase + IOSPACE_SIZE))
panic("map_iospace_chunk: passed invalid va 0x%lx\n", va);
ppn = ADDR_SHIFT(pa);
pt = &iospace_pgtables[(va - sIOSpaceBase) / B_PAGE_SIZE];
for (i = 0; i < 1024; i++) {
init_page_table_entry(&pt[i]);
pt[i].addr = ppn + i;
pt[i].user = 0;
pt[i].rw = 1;
pt[i].present = 1;
pt[i].global = 1;
}
struct thread* thread = thread_get_current_thread();
thread_pin_to_current_cpu(thread);
arch_cpu_invalidate_TLB_range(va, va + (IOSPACE_CHUNK_SIZE - B_PAGE_SIZE));
smp_send_broadcast_ici(SMP_MSG_INVALIDATE_PAGE_RANGE,
va, va + (IOSPACE_CHUNK_SIZE - B_PAGE_SIZE), 0,
NULL, SMP_MSG_FLAG_SYNC);
thread_unpin_from_current_cpu(thread);
return B_OK;
}
static status_t
get_physical_page_tmap(addr_t pa, addr_t *va, uint32 flags)
{
return generic_get_physical_page(pa, va, flags);
}
static status_t
put_physical_page_tmap(addr_t va)
{
return generic_put_physical_page(va);
}
static vm_translation_map_ops tmap_ops = {
destroy_tmap,
lock_tmap,
@ -766,12 +639,41 @@ static vm_translation_map_ops tmap_ops = {
get_mapped_size_tmap,
protect_tmap,
clear_flags_tmap,
flush_tmap,
get_physical_page_tmap,
put_physical_page_tmap
flush_tmap
// The physical page ops are initialized by the respective physical page
// mapper.
};
// #pragma mark -
void
x86_early_prepare_page_tables(page_table_entry* pageTables, addr_t address,
size_t size)
{
memset(pageTables, 0, B_PAGE_SIZE * (size / (B_PAGE_SIZE * 1024)));
// put the array of pgtables directly into the kernel pagedir
// these will be wired and kept mapped into virtual space to be easy to get
// to
{
addr_t virtualTable = (addr_t)pageTables;
for (size_t i = 0; i < (size / (B_PAGE_SIZE * 1024));
i++, virtualTable += B_PAGE_SIZE) {
addr_t physicalTable;
early_query(virtualTable, &physicalTable);
page_directory_entry* entry = &page_hole_pgdir[
(address / (B_PAGE_SIZE * 1024)) + i];
x86_put_pgtable_in_pgdir(entry, physicalTable,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
}
}
}
// #pragma mark -
// VM API
@ -789,30 +691,41 @@ arch_vm_translation_map_init_map(vm_translation_map *map, bool kernel)
map->map_count = 0;
recursive_lock_init(&map->lock, "translation map");
CObjectDeleter<recursive_lock> lockDeleter(&map->lock,
&recursive_lock_destroy);
map->arch_data = (vm_translation_map_arch_info *)malloc(sizeof(vm_translation_map_arch_info));
if (map->arch_data == NULL) {
recursive_lock_destroy(&map->lock);
if (map->arch_data == NULL)
return B_NO_MEMORY;
}
MemoryDeleter archInfoDeleter(map->arch_data);
map->arch_data->active_on_cpus = 0;
map->arch_data->num_invalidate_pages = 0;
map->arch_data->page_mapper = NULL;
if (!kernel) {
// user
// allocate a physical page mapper
status_t error = gPhysicalPageMapper
->CreateTranslationMapPhysicalPageMapper(
&map->arch_data->page_mapper);
if (error != B_OK)
return error;
// allocate a pgdir
map->arch_data->pgdir_virt = (page_directory_entry *)memalign(
B_PAGE_SIZE, B_PAGE_SIZE);
if (map->arch_data->pgdir_virt == NULL) {
free(map->arch_data);
recursive_lock_destroy(&map->lock);
map->arch_data->page_mapper->Delete();
return B_NO_MEMORY;
}
vm_get_page_mapping(vm_kernel_address_space_id(),
(addr_t)map->arch_data->pgdir_virt, (addr_t *)&map->arch_data->pgdir_phys);
} else {
// kernel
// get the physical page mapper
map->arch_data->page_mapper = gKernelPhysicalPageMapper;
// we already know the kernel pgdir mapping
map->arch_data->pgdir_virt = sKernelVirtualPageDirectory;
map->arch_data->pgdir_phys = sKernelPhysicalPageDirectory;
@ -839,6 +752,9 @@ arch_vm_translation_map_init_map(vm_translation_map *map, bool kernel)
restore_interrupts(state);
}
archInfoDeleter.Detach();
lockDeleter.Detach();
return B_OK;
}
@ -853,8 +769,6 @@ arch_vm_translation_map_init_kernel_map_post_sem(vm_translation_map *map)
status_t
arch_vm_translation_map_init(kernel_args *args)
{
status_t error;
TRACE(("vm_translation_map_init: entry\n"));
// page hole set up in stage2
@ -870,39 +784,8 @@ arch_vm_translation_map_init(kernel_args *args)
B_INITIALIZE_SPINLOCK(&tmap_list_lock);
tmap_list = NULL;
// allocate some space to hold physical page mapping info
iospace_pgtables = (page_table_entry *)vm_allocate_early(args,
B_PAGE_SIZE * (IOSPACE_SIZE / (B_PAGE_SIZE * 1024)), ~0L,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
TRACE(("iospace_pgtables %p\n", iospace_pgtables));
// init physical page mapper
error = generic_vm_physical_page_mapper_init(args, map_iospace_chunk,
&sIOSpaceBase, IOSPACE_SIZE, IOSPACE_CHUNK_SIZE);
if (error != B_OK)
return error;
// initialize our data structures
memset(iospace_pgtables, 0, B_PAGE_SIZE * (IOSPACE_SIZE / (B_PAGE_SIZE * 1024)));
TRACE(("mapping iospace_pgtables\n"));
// put the array of pgtables directly into the kernel pagedir
// these will be wired and kept mapped into virtual space to be easy to get to
{
addr_t phys_pgtable;
addr_t virt_pgtable;
page_directory_entry *e;
int i;
virt_pgtable = (addr_t)iospace_pgtables;
for (i = 0; i < (IOSPACE_SIZE / (B_PAGE_SIZE * 1024)); i++, virt_pgtable += B_PAGE_SIZE) {
early_query(virt_pgtable, &phys_pgtable);
e = &page_hole_pgdir[(sIOSpaceBase / (B_PAGE_SIZE * 1024)) + i];
put_pgtable_in_pgdir(e, phys_pgtable, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
}
}
// TODO: Select the best page mapper!
large_memory_physical_page_ops_init(args, &tmap_ops);
// enable global page feature if available
if (x86_check_feature(IA32_FEATURE_PGE, FEATURE_COMMON)) {
@ -919,7 +802,7 @@ arch_vm_translation_map_init(kernel_args *args)
status_t
arch_vm_translation_map_init_post_sem(kernel_args *args)
{
return generic_vm_physical_page_mapper_init_post_sem(args);
return B_OK;
}
@ -945,50 +828,10 @@ arch_vm_translation_map_init_post_area(kernel_args *args)
if (area < B_OK)
return area;
temp = (void *)iospace_pgtables;
area = create_area("iospace_pgtables", &temp, B_EXACT_ADDRESS,
B_PAGE_SIZE * (IOSPACE_SIZE / (B_PAGE_SIZE * 1024)),
B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
if (area < B_OK)
return area;
error = generic_vm_physical_page_mapper_init_post_area(args);
error = gPhysicalPageMapper->InitPostArea(args);
if (error != B_OK)
return error;
// this area is used for query_tmap_interrupt()
// TODO: Note, this only works as long as all pages belong to the same
// page table, which is not yet enforced (or even tested)!
area = vm_create_null_area(vm_kernel_address_space_id(),
"interrupt query pages", (void **)&sQueryPages, B_ANY_ADDRESS,
B_PAGE_SIZE * (smp_get_num_cpus() + 1));
if (area < B_OK)
return area;
// map the last page of the query pages to the page table entry they're in
{
page_table_entry *pageTableEntry;
addr_t physicalPageTable;
int32 index;
sQueryPageTable = (page_table_entry *)(sQueryPages + smp_get_num_cpus() * B_PAGE_SIZE);
index = VADDR_TO_PDENT((addr_t)sQueryPageTable);
physicalPageTable = ADDR_REVERSE_SHIFT(sKernelVirtualPageDirectory[index].addr);
get_physical_page_tmap(physicalPageTable,
(addr_t *)&pageTableEntry, PHYSICAL_PAGE_DONT_WAIT);
index = VADDR_TO_PTENT((addr_t)sQueryPageTable);
put_page_table_entry_in_pgtable(&pageTableEntry[index], physicalPageTable,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, false);
put_physical_page_tmap((addr_t)pageTableEntry);
//invalidate_TLB(sQueryPageTable);
}
TRACE(("vm_translation_map_init_post_area: done\n"));
return B_OK;
}
@ -1022,7 +865,7 @@ arch_vm_translation_map_early_map(kernel_args *args, addr_t va, addr_t pa,
// put it in the pgdir
e = &page_hole_pgdir[index];
put_pgtable_in_pgdir(e, pgtable, attributes);
x86_put_pgtable_in_pgdir(e, pgtable, attributes);
// zero it out in it's new mapping
memset((unsigned int *)((unsigned int)page_hole + (va / B_PAGE_SIZE / 1024) * B_PAGE_SIZE), 0, B_PAGE_SIZE);

103
src/system/kernel/arch/x86/x86_paging.h Normal file
View File

@ -0,0 +1,103 @@
/*
* Copyright 2005-2006, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
* Distributed under the terms of the MIT License.
*
* Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
* Distributed under the terms of the NewOS License.
*/
#ifndef _KERNEL_ARCH_X86_PAGING_H
#define _KERNEL_ARCH_X86_PAGING_H
#include <SupportDefs.h>
#define PAGE_INVALIDATE_CACHE_SIZE 64
#define ADDR_SHIFT(x) ((x)>>12)
#define ADDR_REVERSE_SHIFT(x) ((x)<<12)
#define VADDR_TO_PDENT(va) (((va) / B_PAGE_SIZE) / 1024)
#define VADDR_TO_PTENT(va) (((va) / B_PAGE_SIZE) % 1024)
class TranslationMapPhysicalPageMapper;
typedef struct page_table_entry {
uint32 present:1;
uint32 rw:1;
uint32 user:1;
uint32 write_through:1;
uint32 cache_disabled:1;
uint32 accessed:1;
uint32 dirty:1;
uint32 reserved:1;
uint32 global:1;
uint32 avail:3;
uint32 addr:20;
} page_table_entry;
typedef struct page_directory_entry {
uint32 present:1;
uint32 rw:1;
uint32 user:1;
uint32 write_through:1;
uint32 cache_disabled:1;
uint32 accessed:1;
uint32 reserved:1;
uint32 page_size:1;
uint32 global:1;
uint32 avail:3;
uint32 addr:20;
} page_directory_entry;
typedef struct vm_translation_map_arch_info {
struct page_directory_entry* pgdir_virt;
struct page_directory_entry* pgdir_phys;
TranslationMapPhysicalPageMapper* page_mapper;
vint32 active_on_cpus;
// mask indicating on which CPUs the map is currently used
int num_invalidate_pages;
addr_t pages_to_invalidate[PAGE_INVALIDATE_CACHE_SIZE];
} vm_translation_map_arch_info;
void x86_early_prepare_page_tables(page_table_entry* pageTables, addr_t address,
size_t size);
void x86_put_pgtable_in_pgdir(page_directory_entry* entry,
addr_t physicalPageTable, uint32 attributes);
void x86_update_all_pgdirs(int index, page_directory_entry entry);
static inline void
init_page_directory_entry(page_directory_entry *entry)
{
*(uint32 *)entry = 0;
}
static inline void
update_page_directory_entry(page_directory_entry *entry, page_directory_entry *with)
{
// update page directory entry atomically
*(uint32 *)entry = *(uint32 *)with;
}
static inline void
init_page_table_entry(page_table_entry *entry)
{
*(uint32 *)entry = 0;
}
static inline void
update_page_table_entry(page_table_entry *entry, page_table_entry *with)
{
// update page table entry atomically
*(uint32 *)entry = *(uint32 *)with;
}
#endif // _KERNEL_ARCH_X86_PAGING_H

20
src/system/kernel/arch/x86/x86_physical_page_mapper.cpp Normal file
View File

@ -0,0 +1,20 @@
/*
* Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de.
* Distributed under the terms of the MIT License.
*/
#include "x86_physical_page_mapper.h"
PhysicalPageMapper* gPhysicalPageMapper;
TranslationMapPhysicalPageMapper* gKernelPhysicalPageMapper;
TranslationMapPhysicalPageMapper::~TranslationMapPhysicalPageMapper()
{
}
PhysicalPageMapper::~PhysicalPageMapper()
{
}

49
src/system/kernel/arch/x86/x86_physical_page_mapper.h Normal file
View File

@ -0,0 +1,49 @@
/*
* Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de.
* Distributed under the terms of the MIT License.
*/
#ifndef _KERNEL_ARCH_X86_PHYSICAL_PAGE_MAPPER_H
#define _KERNEL_ARCH_X86_PHYSICAL_PAGE_MAPPER_H
#include <SupportDefs.h>
struct kernel_args;
struct page_table_entry;
struct vm_translation_map_ops;
class TranslationMapPhysicalPageMapper {
public:
virtual ~TranslationMapPhysicalPageMapper();
virtual void Delete() = 0;
virtual page_table_entry* GetPageTableAt(addr_t physicalAddress) = 0;
// Must be invoked with thread pinned to current CPU.
};
class PhysicalPageMapper {
public:
virtual ~PhysicalPageMapper();
virtual status_t InitPostArea(kernel_args* args) = 0;
virtual status_t CreateTranslationMapPhysicalPageMapper(
TranslationMapPhysicalPageMapper** _mapper)
= 0;
virtual page_table_entry* InterruptGetPageTableAt(
addr_t physicalAddress) = 0;
};
extern PhysicalPageMapper* gPhysicalPageMapper;
extern TranslationMapPhysicalPageMapper* gKernelPhysicalPageMapper;
status_t large_memory_physical_page_ops_init(kernel_args* args,
vm_translation_map_ops* ops);
#endif // _KERNEL_ARCH_X86_PHYSICAL_PAGE_MAPPER_H

981
src/system/kernel/arch/x86/x86_physical_page_mapper_large_memory.cpp Normal file
View File

@ -0,0 +1,981 @@
/*
* Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de.
* Distributed under the terms of the MIT License.
*/
/* Implementation of a physical page mapping strategy (PhysicalPageMapper,
TranslationMapPhysicalPageMapper) suitable for machines with a lot of
memory, i.e. more than we can afford to completely map into the kernel
address space.
We allocate a single page table (one page) that can map 1024 pages and
a corresponding virtual address space region (4 MB). Each of those 1024
slots can map a physical page. We reserve a fixed amount of slot per CPU.
They will be used for physical operations on that CPU (memset()/memcpy()
and {get,put}_physical_page_current_cpu()). A few slots we reserve for each
translation map (TranslationMapPhysicalPageMapper). Those will only be used
with the translation map locked, mapping a page table page. The remaining
slots remain in the global pool and are given out by get_physical_page().
When we run out of slots, we allocate another page table (and virtual
address space region).
*/
#include "x86_physical_page_mapper.h"
#include <new>
#include <AutoDeleter.h>
#include <cpu.h>
#include <lock.h>
#include <smp.h>
#include <util/AutoLock.h>
#include <util/DoublyLinkedList.h>
#include <vm.h>
#include <vm_address_space.h>
#include <vm_translation_map.h>
#include <vm_types.h>
#include "x86_paging.h"
// The number of slots we reserve per translation map from mapping page tables.
// One slot would suffice, since the map is locked while mapping a page table,
// but we re-use several slots on a LRU-basis so that we can keep the mappings
// a little longer, thus avoiding re-mapping.
#define SLOTS_PER_TRANSLATION_MAP 4
#define USER_SLOTS_PER_CPU 16
#define KERNEL_SLOTS_PER_CPU 16
#define TOTAL_SLOTS_PER_CPU (USER_SLOTS_PER_CPU \
+ KERNEL_SLOTS_PER_CPU + 1)
// one slot is for use in interrupts
static const size_t kPageTableAlignment = 1024 * B_PAGE_SIZE;
struct PhysicalPageSlotPool;
struct PhysicalPageSlot {
PhysicalPageSlot* next;
PhysicalPageSlotPool* pool;
addr_t address;
inline void Map(addr_t physicalAddress);
};
struct PhysicalPageSlotPool : DoublyLinkedListLinkImpl<PhysicalPageSlotPool> {
area_id dataArea;
area_id virtualArea;
addr_t virtualBase;
page_table_entry* pageTable;
PhysicalPageSlot* slots;
void Init(area_id dataArea, void* data,
area_id virtualArea, addr_t virtualBase);
inline bool IsEmpty() const;
inline PhysicalPageSlot* GetSlot();
inline void PutSlot(PhysicalPageSlot* slot);
};
class PhysicalPageSlotQueue {
public:
PhysicalPageSlotQueue();
inline PhysicalPageSlot* GetSlot();
inline void GetSlots(PhysicalPageSlot*& slot1,
PhysicalPageSlot*& slot2);
inline void PutSlot(PhysicalPageSlot* slot);
inline void PutSlots(PhysicalPageSlot* slot1,
PhysicalPageSlot* slot2);
private:
PhysicalPageSlot* fSlots;
ConditionVariable fFreeSlotCondition;
ConditionVariable fFreeSlotsCondition;
};
struct PhysicalPageOpsCPUData {
PhysicalPageSlotQueue user;
// Used when copying from/to user memory. This can cause a page fault
// which might need to memcpy()/memset() a page when being handled.
PhysicalPageSlotQueue kernel;
// Used when memset()ing or when memcpy()ing memory non-user memory.
PhysicalPageSlot* interruptSlot;
void Init();
private:
static PhysicalPageSlot* _GetInitialSlot();
};
// #pragma mark -
class LargeMemoryTranslationMapPhysicalPageMapper
: public TranslationMapPhysicalPageMapper {
public:
LargeMemoryTranslationMapPhysicalPageMapper();
virtual ~LargeMemoryTranslationMapPhysicalPageMapper();
status_t Init();
virtual void Delete();
virtual page_table_entry* GetPageTableAt(addr_t physicalAddress);
private:
struct page_slot {
PhysicalPageSlot* slot;
addr_t physicalAddress;
cpu_mask_t valid;
};
page_slot fSlots[SLOTS_PER_TRANSLATION_MAP];
int32 fSlotCount; // must be a power of 2
int32 fNextSlot;
};
class LargeMemoryPhysicalPageMapper : public PhysicalPageMapper {
public:
LargeMemoryPhysicalPageMapper();
status_t Init(kernel_args* args);
virtual status_t InitPostArea(kernel_args* args);
virtual status_t CreateTranslationMapPhysicalPageMapper(
TranslationMapPhysicalPageMapper** _mapper);
virtual page_table_entry* InterruptGetPageTableAt(
addr_t physicalAddress);
inline status_t GetPage(addr_t physicalAddress,
addr_t* virtualAddress, void** handle);
inline status_t PutPage(addr_t virtualAddress, void* handle);
inline status_t GetPageCurrentCPU(addr_t physicalAddress,
addr_t* virtualAddress, void** handle);
inline status_t PutPageCurrentCPU(addr_t virtualAddress,
void* handle);
inline status_t GetPageDebug(addr_t physicalAddress,
addr_t* virtualAddress, void** handle);
inline status_t PutPageDebug(addr_t virtualAddress,
void* handle);
status_t GetSlot(bool canWait,
PhysicalPageSlot*& slot);
void PutSlot(PhysicalPageSlot* slot);
inline PhysicalPageSlotQueue* GetSlotQueue(int32 cpu, bool user);
private:
static status_t _AllocatePool(
PhysicalPageSlotPool*& _pool);
private:
typedef DoublyLinkedList<PhysicalPageSlotPool> PoolList;
mutex fLock;
PoolList fEmptyPools;
PoolList fNonEmptyPools;
PhysicalPageSlot* fDebugSlot;
PhysicalPageSlotPool fInitialPool;
LargeMemoryTranslationMapPhysicalPageMapper fKernelMapper;
PhysicalPageOpsCPUData fPerCPUData[B_MAX_CPU_COUNT];
};
static LargeMemoryPhysicalPageMapper sPhysicalPageMapper;
// #pragma mark - PhysicalPageSlot / PhysicalPageSlotPool
inline void
PhysicalPageSlot::Map(addr_t physicalAddress)
{
page_table_entry& pte = pool->pageTable[
(address - pool->virtualBase) / B_PAGE_SIZE];
init_page_table_entry(&pte);
pte.addr = ADDR_SHIFT(physicalAddress);
pte.user = 0;
pte.rw = 1;
pte.present = 1;
pte.global = 1;
invalidate_TLB(address);
}
void
PhysicalPageSlotPool::Init(area_id dataArea, void* data,
area_id virtualArea, addr_t virtualBase)
{
this->dataArea = dataArea;
this->virtualArea = virtualArea;
this->virtualBase = virtualBase;
pageTable = (page_table_entry*)data;
// init slot list
slots = (PhysicalPageSlot*)(pageTable + 1024);
addr_t slotAddress = virtualBase;
for (int32 i = 0; i < 1024; i++, slotAddress += B_PAGE_SIZE) {
PhysicalPageSlot* slot = &slots[i];
slot->next = slot + 1;
slot->pool = this;
slot->address = slotAddress;
}
slots[1023].next = NULL;
// terminate list
}
inline bool
PhysicalPageSlotPool::IsEmpty() const
{
return slots == NULL;
}
inline PhysicalPageSlot*
PhysicalPageSlotPool::GetSlot()
{
PhysicalPageSlot* slot = slots;
slots = slot->next;
return slot;
}
inline void
PhysicalPageSlotPool::PutSlot(PhysicalPageSlot* slot)
{
slot->next = slots;
slots = slot;
}
// #pragma mark - PhysicalPageSlotQueue
PhysicalPageSlotQueue::PhysicalPageSlotQueue()
:
fSlots(NULL)
{
fFreeSlotCondition.Init(this, "physical page ops slot queue");
fFreeSlotsCondition.Init(this, "physical page ops slots queue");
}
PhysicalPageSlot*
PhysicalPageSlotQueue::GetSlot()
{
InterruptsLocker locker;
// wait for a free slot to turn up
while (fSlots == NULL) {
ConditionVariableEntry entry;
fFreeSlotCondition.Add(&entry);
locker.Unlock();
entry.Wait();
locker.Lock();
}
PhysicalPageSlot* slot = fSlots;
fSlots = slot->next;
return slot;
}
void
PhysicalPageSlotQueue::GetSlots(PhysicalPageSlot*& slot1,
PhysicalPageSlot*& slot2)
{
InterruptsLocker locker;
// wait for two free slot to turn up
while (fSlots == NULL || fSlots->next == NULL) {
ConditionVariableEntry entry;
fFreeSlotsCondition.Add(&entry);
locker.Unlock();
entry.Wait();
locker.Lock();
}
slot1 = fSlots;
slot2 = slot1->next;
fSlots = slot2->next;
}
void
PhysicalPageSlotQueue::PutSlot(PhysicalPageSlot* slot)
{
InterruptsLocker locker;
slot->next = fSlots;
fSlots = slot;
if (slot->next == NULL)
fFreeSlotCondition.NotifyAll();
else if (slot->next->next == NULL)
fFreeSlotCondition.NotifyAll();
}
void
PhysicalPageSlotQueue::PutSlots(PhysicalPageSlot* slot1,
PhysicalPageSlot* slot2)
{
InterruptsLocker locker;
slot1->next = slot2;
slot2->next = fSlots;
fSlots = slot1;
if (slot2->next == NULL)
fFreeSlotCondition.NotifyAll();
else if (slot2->next->next == NULL)
fFreeSlotCondition.NotifyAll();
}
// #pragma mark - PhysicalPageOpsCPUData
void
PhysicalPageOpsCPUData::Init()
{
for (int32 i = 0; i < USER_SLOTS_PER_CPU; i++)
user.PutSlot(_GetInitialSlot());
for (int32 i = 0; i < KERNEL_SLOTS_PER_CPU; i++)
kernel.PutSlot(_GetInitialSlot());
interruptSlot = _GetInitialSlot();
}
/* static */ PhysicalPageSlot*
PhysicalPageOpsCPUData::_GetInitialSlot()
{
PhysicalPageSlot* slot;
status_t error = sPhysicalPageMapper.GetSlot(false, slot);
if (error != B_OK) {
panic("PhysicalPageOpsCPUData::Init(): Failed to get initial "
"physical page slots! Probably too many CPUs.");
return NULL;
}
return slot;
}
// #pragma mark - LargeMemoryTranslationMapPhysicalPageMapper
LargeMemoryTranslationMapPhysicalPageMapper
::LargeMemoryTranslationMapPhysicalPageMapper()
:
fSlotCount(sizeof(fSlots) / sizeof(page_slot)),
fNextSlot(0)
{
memset(fSlots, 0, sizeof(fSlots));
}
LargeMemoryTranslationMapPhysicalPageMapper
::~LargeMemoryTranslationMapPhysicalPageMapper()
{
// put our slots back to the global pool
for (int32 i = 0; i < fSlotCount; i++) {
if (fSlots[i].slot != NULL)
sPhysicalPageMapper.PutSlot(fSlots[i].slot);
}
}
status_t
LargeMemoryTranslationMapPhysicalPageMapper::Init()
{
// get our slots from the global pool
for (int32 i = 0; i < fSlotCount; i++) {
status_t error = sPhysicalPageMapper.GetSlot(true, fSlots[i].slot);
if (error != B_OK)
return error;
// set to invalid physical address, so it won't be used accidentally
fSlots[i].physicalAddress = ~(addr_t)0;
}
return B_OK;
}
void
LargeMemoryTranslationMapPhysicalPageMapper::Delete()
{
delete this;
}
page_table_entry*
LargeMemoryTranslationMapPhysicalPageMapper::GetPageTableAt(
addr_t physicalAddress)
{
ASSERT(physicalAddress % B_PAGE_SIZE == 0);
int32 currentCPU = smp_get_current_cpu();
// maybe the address is already mapped
for (int32 i = 0; i < fSlotCount; i++) {
page_slot& slot = fSlots[i];
if (slot.physicalAddress == physicalAddress) {
fNextSlot = (i + 1) & (fSlotCount - 1);
if ((slot.valid & (1 << currentCPU)) == 0) {
// not valid on this CPU -- invalidate the TLB entry
invalidate_TLB(slot.slot->address);
slot.valid |= 1 << currentCPU;
}
return (page_table_entry*)slot.slot->address;
}
}
// not found -- need to map a fresh one
page_slot& slot = fSlots[fNextSlot];
fNextSlot = (fNextSlot + 1) & (fSlotCount - 1);
slot.physicalAddress = physicalAddress;
slot.slot->Map(physicalAddress);
slot.valid = 1 << currentCPU;
return (page_table_entry*)slot.slot->address;
}
// #pragma mark - LargeMemoryPhysicalPageMapper
LargeMemoryPhysicalPageMapper::LargeMemoryPhysicalPageMapper()
{
mutex_init(&fLock, "large memory physical page mapper");
}
status_t
LargeMemoryPhysicalPageMapper::Init(kernel_args* args)
{
// We reserve more, so that we can guarantee to align the base address
// to page table ranges.
addr_t virtualBase = vm_allocate_early(args,
1024 * B_PAGE_SIZE + kPageTableAlignment - B_PAGE_SIZE, 0, 0);
if (virtualBase == 0) {
panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve "
"physical page pool space in virtual address space!");
return B_ERROR;
}
virtualBase = (virtualBase + kPageTableAlignment - 1)
/ kPageTableAlignment * kPageTableAlignment;
// allocate memory for the page table and data
size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]);
page_table_entry* pageTable = (page_table_entry*)vm_allocate_early(args,
areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
// prepare the page table
x86_early_prepare_page_tables(pageTable, virtualBase,
1024 * B_PAGE_SIZE);
// init the pool structure and add the initial pool
fInitialPool.Init(-1, pageTable, -1, (addr_t)virtualBase);
fNonEmptyPools.Add(&fInitialPool);
// get the debug slot
GetSlot(true, fDebugSlot);
// init the kernel translation map physical page mapper
status_t error = fKernelMapper.Init();
if (error != B_OK) {
panic("LargeMemoryPhysicalPageMapper::Init(): Failed to init "
"kernel translation map physical page mapper!");
return error;
}
gKernelPhysicalPageMapper = &fKernelMapper;
// init the per-CPU data
int32 cpuCount = smp_get_num_cpus();
for (int32 i = 0; i < cpuCount; i++)
fPerCPUData[i].Init();
return B_OK;
}
status_t
LargeMemoryPhysicalPageMapper::InitPostArea(kernel_args* args)
{
// create an area for the (already allocated) data
size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]);
void* temp = fInitialPool.pageTable;
area_id area = create_area("physical page pool", &temp,
B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
if (area < B_OK) {
panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
"create area for physical page pool.");
return area;
}
fInitialPool.dataArea = area;
// create an area for the virtual address space
temp = (void*)fInitialPool.virtualBase;
area = vm_create_null_area(vm_kernel_address_space_id(),
"physical page pool space", &temp, B_EXACT_ADDRESS,
1024 * B_PAGE_SIZE);
if (area < B_OK) {
panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
"create area for physical page pool space.");
return area;
}
fInitialPool.virtualArea = area;
return B_OK;
}
status_t
LargeMemoryPhysicalPageMapper::CreateTranslationMapPhysicalPageMapper(
TranslationMapPhysicalPageMapper** _mapper)
{
LargeMemoryTranslationMapPhysicalPageMapper* mapper
= new(std::nothrow) LargeMemoryTranslationMapPhysicalPageMapper;
if (mapper == NULL)
return B_NO_MEMORY;
status_t error = mapper->Init();
if (error != B_OK) {
delete mapper;
return error;
}
*_mapper = mapper;
return B_OK;
}
page_table_entry*
LargeMemoryPhysicalPageMapper::InterruptGetPageTableAt(addr_t physicalAddress)
{
ASSERT(physicalAddress % B_PAGE_SIZE == 0);
PhysicalPageSlot* slot = fPerCPUData[smp_get_current_cpu()].interruptSlot;
slot->Map(physicalAddress);
return (page_table_entry*)slot->address;
}
status_t
LargeMemoryPhysicalPageMapper::GetPage(addr_t physicalAddress,
addr_t* virtualAddress, void** handle)
{
PhysicalPageSlot* slot;
status_t error = GetSlot(true, slot);
if (error != B_OK)
return error;
slot->Map(physicalAddress);
*handle = slot;
*virtualAddress = slot->address + physicalAddress % B_PAGE_SIZE;
return B_OK;
}
status_t
LargeMemoryPhysicalPageMapper::PutPage(addr_t virtualAddress, void* handle)
{
PutSlot((PhysicalPageSlot*)handle);
return B_OK;
}
status_t
LargeMemoryPhysicalPageMapper::GetPageCurrentCPU(addr_t physicalAddress,
addr_t* virtualAddress, void** handle)
{
// get a slot from the per-cpu user pool
PhysicalPageSlotQueue& slotQueue
= fPerCPUData[smp_get_current_cpu()].user;
PhysicalPageSlot* slot = slotQueue.GetSlot();
slot->Map(physicalAddress);
*virtualAddress = slot->address + physicalAddress % B_PAGE_SIZE;
*handle = slot;
return B_OK;
}
status_t
LargeMemoryPhysicalPageMapper::PutPageCurrentCPU(addr_t virtualAddress,
void* handle)
{
// return the slot to the per-cpu user pool
PhysicalPageSlotQueue& slotQueue
= fPerCPUData[smp_get_current_cpu()].user;
slotQueue.PutSlot((PhysicalPageSlot*)handle);
return B_OK;
}
status_t
LargeMemoryPhysicalPageMapper::GetPageDebug(addr_t physicalAddress,
addr_t* virtualAddress, void** handle)
{
fDebugSlot->Map(physicalAddress);
*handle = fDebugSlot;
*virtualAddress = fDebugSlot->address + physicalAddress % B_PAGE_SIZE;
return B_OK;
}
status_t
LargeMemoryPhysicalPageMapper::PutPageDebug(addr_t virtualAddress, void* handle)
{
return B_OK;
}
status_t
LargeMemoryPhysicalPageMapper::GetSlot(bool canWait,
PhysicalPageSlot*& slot)
{
MutexLocker locker(fLock);
PhysicalPageSlotPool* pool = fNonEmptyPools.Head();
if (pool == NULL) {
if (!canWait)
return B_WOULD_BLOCK;
// allocate new pool
locker.Unlock();
status_t error = _AllocatePool(pool);
if (error != B_OK)
return error;
locker.Lock();
fNonEmptyPools.Add(pool);
pool = fNonEmptyPools.Head();
}
slot = pool->GetSlot();
if (pool->IsEmpty()) {
fNonEmptyPools.Remove(pool);
fEmptyPools.Add(pool);
}
return B_OK;
}
void
LargeMemoryPhysicalPageMapper::PutSlot(PhysicalPageSlot* slot)
{
MutexLocker locker(fLock);
PhysicalPageSlotPool* pool = slot->pool;
if (pool->IsEmpty()) {
fEmptyPools.Remove(pool);
fNonEmptyPools.Add(pool);
}
pool->PutSlot(slot);
}
inline PhysicalPageSlotQueue*
LargeMemoryPhysicalPageMapper::GetSlotQueue(int32 cpu, bool user)
{
return user ? &fPerCPUData[cpu].user : &fPerCPUData[cpu].kernel;
}
/* static */ status_t
LargeMemoryPhysicalPageMapper::_AllocatePool(PhysicalPageSlotPool*& _pool)
{
// create the pool structure
PhysicalPageSlotPool* pool
= new(std::nothrow) PhysicalPageSlotPool;
if (pool == NULL)
return B_NO_MEMORY;
ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool);
// create an area that can contain the page table and the slot
// structures
size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]);
void* data;
area_id dataArea = create_area("physical page pool", &data,
B_ANY_KERNEL_ADDRESS, PAGE_ALIGN(areaSize), B_FULL_LOCK,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
if (dataArea < 0)
return dataArea;
// create the null area for the virtual address space
void* virtualBase;
area_id virtualArea = vm_create_null_area(
vm_kernel_address_space_id(), "physical page pool space",
&virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, 1024 * B_PAGE_SIZE);
if (virtualArea < 0) {
delete_area(dataArea);
return virtualArea;
}
// prepare the page table
memset(data, 0, B_PAGE_SIZE);
// get the page table's physical address
addr_t physicalTable;
vm_translation_map* map = &vm_kernel_address_space()->translation_map;
uint32 dummyFlags;
cpu_status state = disable_interrupts();
map->ops->query_interrupt(map, (addr_t)data, &physicalTable,
&dummyFlags);
restore_interrupts(state);
// put the page table into the page directory
int32 index = (addr_t)virtualBase / (B_PAGE_SIZE * 1024);
page_directory_entry* entry = &map->arch_data->pgdir_virt[index];
x86_put_pgtable_in_pgdir(entry, physicalTable,
B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
x86_update_all_pgdirs(index, *entry);
// init the pool structure
pool->Init(dataArea, data, virtualArea, (addr_t)virtualBase);
poolDeleter.Detach();
_pool = pool;
return B_OK;
}
// #pragma mark - physical page operations
static status_t
large_memory_get_physical_page(addr_t physicalAddress, addr_t *_virtualAddress,
void **_handle)
{
return sPhysicalPageMapper.GetPage(physicalAddress, _virtualAddress,
_handle);
}
static status_t
large_memory_put_physical_page(addr_t virtualAddress, void *handle)
{
return sPhysicalPageMapper.PutPage(virtualAddress, handle);
}
static status_t
large_memory_get_physical_page_current_cpu(addr_t physicalAddress,
addr_t *_virtualAddress, void **_handle)
{
return sPhysicalPageMapper.GetPageCurrentCPU(physicalAddress,
_virtualAddress, _handle);
}
static status_t
large_memory_put_physical_page_current_cpu(addr_t virtualAddress, void *handle)
{
return sPhysicalPageMapper.PutPageCurrentCPU(virtualAddress, handle);
}
static status_t
large_memory_get_physical_page_debug(addr_t physicalAddress,
addr_t *_virtualAddress, void **_handle)
{
return sPhysicalPageMapper.GetPageDebug(physicalAddress,
_virtualAddress, _handle);
}
static status_t
large_memory_put_physical_page_debug(addr_t virtualAddress, void *handle)
{
return sPhysicalPageMapper.PutPageDebug(virtualAddress, handle);
}
static status_t
large_memory_memset_physical(addr_t address, int value,
size_t length)
{
addr_t pageOffset = address % B_PAGE_SIZE;
struct thread* thread = thread_get_current_thread();
ThreadCPUPinner _(thread);
PhysicalPageSlotQueue* slotQueue = sPhysicalPageMapper.GetSlotQueue(
thread->cpu->cpu_num, false);
PhysicalPageSlot* slot = slotQueue->GetSlot();
while (length > 0) {
slot->Map(address - pageOffset);
size_t toSet = min_c(length, B_PAGE_SIZE - pageOffset);
memset((void*)(slot->address + pageOffset), value, toSet);
length -= toSet;
address += toSet;
pageOffset = 0;
}
slotQueue->PutSlot(slot);
return B_OK;
}
static status_t
large_memory_memcpy_from_physical(void* _to, addr_t from, size_t length,
bool user)
{
uint8* to = (uint8*)_to;
addr_t pageOffset = from % B_PAGE_SIZE;
struct thread* thread = thread_get_current_thread();
ThreadCPUPinner _(thread);
PhysicalPageSlotQueue* slotQueue = sPhysicalPageMapper.GetSlotQueue(
thread->cpu->cpu_num, user);
PhysicalPageSlot* slot = slotQueue->GetSlot();
status_t error = B_OK;
while (length > 0) {
size_t toCopy = min_c(length, B_PAGE_SIZE - pageOffset);
slot->Map(from - pageOffset);
if (user) {
error = user_memcpy(to, (void*)(slot->address + pageOffset),
toCopy);
if (error != B_OK)
break;
} else
memcpy(to, (void*)(slot->address + pageOffset), toCopy);
to += toCopy;
from += toCopy;
length -= toCopy;
pageOffset = 0;
}
slotQueue->PutSlot(slot);
return error;
}
static status_t
large_memory_memcpy_to_physical(addr_t to, const void* _from, size_t length,
bool user)
{
const uint8* from = (const uint8*)_from;
addr_t pageOffset = to % B_PAGE_SIZE;
struct thread* thread = thread_get_current_thread();
ThreadCPUPinner _(thread);
PhysicalPageSlotQueue* slotQueue = sPhysicalPageMapper.GetSlotQueue(
thread->cpu->cpu_num, user);
PhysicalPageSlot* slot = slotQueue->GetSlot();
status_t error = B_OK;
while (length > 0) {
size_t toCopy = min_c(length, B_PAGE_SIZE - pageOffset);
slot->Map(to - pageOffset);
if (user) {
error = user_memcpy((void*)(slot->address + pageOffset), from,
toCopy);
if (error != B_OK)
break;
} else
memcpy((void*)(slot->address + pageOffset), from, toCopy);
to += toCopy;
from += toCopy;
length -= toCopy;
pageOffset = 0;
}
slotQueue->PutSlot(slot);
return error;
}
static void
large_memory_memcpy_physical_page(addr_t to, addr_t from)
{
struct thread* thread = thread_get_current_thread();
ThreadCPUPinner _(thread);
PhysicalPageSlotQueue* slotQueue = sPhysicalPageMapper.GetSlotQueue(
thread->cpu->cpu_num, false);
PhysicalPageSlot* fromSlot;
PhysicalPageSlot* toSlot;
slotQueue->GetSlots(fromSlot, toSlot);
fromSlot->Map(from);
toSlot->Map(to);
memcpy((void*)toSlot->address, (void*)fromSlot->address, B_PAGE_SIZE);
slotQueue->PutSlots(fromSlot, toSlot);
}
// #pragma mark - Initialization
status_t
large_memory_physical_page_ops_init(kernel_args* args,
vm_translation_map_ops* ops)
{
gPhysicalPageMapper
= new(&sPhysicalPageMapper) LargeMemoryPhysicalPageMapper;
sPhysicalPageMapper.Init(args);
// init physical ops
ops->get_physical_page = &large_memory_get_physical_page;
ops->put_physical_page = &large_memory_put_physical_page;
ops->get_physical_page_current_cpu
= &large_memory_get_physical_page_current_cpu;
ops->put_physical_page_current_cpu
= &large_memory_put_physical_page_current_cpu;
ops->get_physical_page_debug = &large_memory_get_physical_page_debug;
ops->put_physical_page_debug = &large_memory_put_physical_page_debug;
ops->memset_physical = &large_memory_memset_physical;
ops->memcpy_from_physical = &large_memory_memcpy_from_physical;
ops->memcpy_to_physical = &large_memory_memcpy_to_physical;
ops->memcpy_physical_page = &large_memory_memcpy_physical_page;
return B_OK;
}

44
src/system/kernel/cache/file_cache.cpp vendored
View File

@ -233,22 +233,15 @@ read_into_cache(file_cache_ref *ref, void *cookie, off_t offset,
for (int32 i = 0; i < pageIndex; i++) {
if (useBuffer && bufferSize != 0) {
addr_t virtualAddress;
if (vm_get_physical_page(
pages[i]->physical_page_number * B_PAGE_SIZE,
&virtualAddress, 0) < B_OK) {
panic("could not get physical page");
}
size_t bytes = min_c(bufferSize, (size_t)B_PAGE_SIZE - pageOffset);
user_memcpy((void*)buffer, (void*)(virtualAddress + pageOffset),
bytes);
vm_memcpy_from_physical((void*)buffer,
pages[i]->physical_page_number * B_PAGE_SIZE + pageOffset,
bytes, true);
buffer += bytes;
bufferSize -= bytes;
pageOffset = 0;
vm_put_physical_page(virtualAddress);
}
}
@ -364,7 +357,7 @@ write_to_cache(file_cache_ref *ref, void *cookie, off_t offset,
if (offset + pageOffset + bufferSize == ref->cache->virtual_end) {
// the space in the page after this write action needs to be cleaned
memset_physical(last + lastPageOffset, 0,
vm_memset_physical(last + lastPageOffset, 0,
B_PAGE_SIZE - lastPageOffset);
} else {
// the end of this write does not happen on a page boundary, so we
@ -381,7 +374,8 @@ write_to_cache(file_cache_ref *ref, void *cookie, off_t offset,
if (bytesRead < B_PAGE_SIZE) {
// the space beyond the file size needs to be cleaned
memset_physical(last + bytesRead, 0, B_PAGE_SIZE - bytesRead);
vm_memset_physical(last + bytesRead, 0,
B_PAGE_SIZE - bytesRead);
}
}
}
@ -393,10 +387,11 @@ write_to_cache(file_cache_ref *ref, void *cookie, off_t offset,
if (useBuffer) {
// copy data from user buffer
memcpy_to_physical(base + pageOffset, (void *)buffer, bytes, true);
vm_memcpy_to_physical(base + pageOffset, (void *)buffer, bytes,
true);
} else {
// clear buffer instead
memset_physical(base + pageOffset, 0, bytes);
vm_memset_physical(base + pageOffset, 0, bytes);
}
bufferSize -= bytes;
@ -634,26 +629,21 @@ cache_io(void *_cacheRef, void *cookie, off_t offset, addr_t buffer,
page->state = PAGE_STATE_BUSY;
locker.Unlock();
addr_t virtualAddress;
vm_get_physical_page(page->physical_page_number * B_PAGE_SIZE,
&virtualAddress, 0);
// copy the contents of the page already in memory
addr_t pageAddress = page->physical_page_number * B_PAGE_SIZE
+ pageOffset;
if (doWrite) {
if (useBuffer) {
user_memcpy((void *)(virtualAddress + pageOffset),
(void *)buffer, bytesInPage);
vm_memcpy_to_physical(pageAddress, (void*)buffer,
bytesInPage, true);
} else {
user_memset((void *)(virtualAddress + pageOffset),
0, bytesInPage);
vm_memset_physical(pageAddress, 0, bytesInPage);
}
} else if (useBuffer) {
user_memcpy((void *)buffer,
(void *)(virtualAddress + pageOffset), bytesInPage);
vm_memcpy_from_physical((void*)buffer, pageAddress,
bytesInPage, true);
}
vm_put_physical_page(virtualAddress);
locker.Lock();
page->state = oldPageState;

2
src/system/kernel/cache/vnode_store.cpp vendored
View File

@ -66,7 +66,7 @@ VMVnodeCache::Read(off_t offset, const iovec *vecs, size_t count,
addr_t address = (addr_t)vecs[i].iov_base + vecs[i].iov_len - length;
if ((flags & B_PHYSICAL_IO_REQUEST) != 0)
memset_physical(address, 0, length);
vm_memset_physical(address, 0, length);
else
memset((void*)address, 0, length);

27
src/system/kernel/device_manager/io_requests.cpp
View File

@ -1104,32 +1104,15 @@ IORequest::_CopySimple(void* bounceBuffer, void* external, size_t size,
/* static */ status_t
IORequest::_CopyPhysical(void* _bounceBuffer, void* _external, size_t size,
IORequest::_CopyPhysical(void* bounceBuffer, void* external, size_t size,
team_id team, bool copyIn)
{
uint8* bounceBuffer = (uint8*)_bounceBuffer;
addr_t external = (addr_t)_external;
while (size > 0) {
addr_t pageOffset = external % B_PAGE_SIZE;
addr_t virtualAddress;
status_t error = vm_get_physical_page(external - pageOffset,
&virtualAddress, 0);
if (error != B_OK)
return error;
size_t toCopy = min_c(size, B_PAGE_SIZE - pageOffset);
_CopySimple(bounceBuffer, (void*)(virtualAddress + pageOffset), toCopy,
team, copyIn);
vm_put_physical_page(virtualAddress);
size -= toCopy;
bounceBuffer += toCopy;
external += toCopy;
if (copyIn) {
return vm_memcpy_from_physical(bounceBuffer, (addr_t)external, size,
false);
}
return B_OK;
return vm_memcpy_to_physical((addr_t)external, bounceBuffer, size, false);
}

5
src/system/kernel/fs/vfs_request_io.cpp
View File

@ -92,8 +92,9 @@ public:
while (length > 0) {
addr_t pageOffset = buffer % B_PAGE_SIZE;
addr_t virtualAddress;
void* handle;
status_t error = vm_get_physical_page(buffer - pageOffset,
&virtualAddress, 0);
&virtualAddress, &handle);
if (error != B_OK)
return error;
@ -102,7 +103,7 @@ public:
error = InternalIO(offset, (void*)(virtualAddress + pageOffset),
&transferred);
vm_put_physical_page(virtualAddress);
vm_put_physical_page(virtualAddress, handle);
if (error != B_OK)
return error;

156
src/system/kernel/vm/vm.cpp
View File

@ -3095,6 +3095,8 @@ display_mem(int argc, char **argv)
if (num <= 0)
num = displayWidth;
void* physicalPageHandle = NULL;
if (physical) {
int32 offset = address & (B_PAGE_SIZE - 1);
if (num * itemSize + offset > B_PAGE_SIZE) {
@ -3104,17 +3106,12 @@ display_mem(int argc, char **argv)
address = ROUNDOWN(address, B_PAGE_SIZE);
gKernelStartup = true;
// vm_get_physical_page() needs to lock...
if (vm_get_physical_page(address, &copyAddress, PHYSICAL_PAGE_DONT_WAIT)
!= B_OK) {
if (vm_get_physical_page_debug(address, &copyAddress,
&physicalPageHandle) != B_OK) {
kprintf("getting the hardware page failed.");
gKernelStartup = false;
return 0;
}
gKernelStartup = false;
address += offset;
copyAddress += offset;
} else
@ -3202,9 +3199,7 @@ display_mem(int argc, char **argv)
if (physical) {
copyAddress = ROUNDOWN(copyAddress, B_PAGE_SIZE);
gKernelStartup = true;
vm_put_physical_page(copyAddress);
gKernelStartup = false;
vm_put_physical_page_debug(copyAddress, physicalPageHandle);
}
return 0;
}
@ -4611,7 +4606,6 @@ fault_get_page(vm_translation_map *map, vm_cache *topCache, off_t cacheOffset,
// fault_find_page() didn't find the page, it would return the top cache
// for write faults.
vm_page *sourcePage = page;
void *source, *dest;
// ToDo: if memory is low, it might be a good idea to steal the page
// from our source cache - if possible, that is
@ -4623,26 +4617,9 @@ if (cacheOffset == 0x12000)
sourcePage, page, sourcePage->cache, topCacheRef->cache);
#endif
// try to get a mapping for the src and dest page so we can copy it
for (;;) {
map->ops->get_physical_page(
sourcePage->physical_page_number * B_PAGE_SIZE,
(addr_t *)&source, 0);
if (map->ops->get_physical_page(
page->physical_page_number * B_PAGE_SIZE,
(addr_t *)&dest, PHYSICAL_PAGE_DONT_WAIT) == B_OK)
break;
// it couldn't map the second one, so sleep and retry
// keeps an extremely rare deadlock from occuring
map->ops->put_physical_page((addr_t)source);
snooze(5000);
}
memcpy(dest, source, B_PAGE_SIZE);
map->ops->put_physical_page((addr_t)source);
map->ops->put_physical_page((addr_t)dest);
// copy the page
vm_memcpy_physical_page(page->physical_page_number * B_PAGE_SIZE,
sourcePage->physical_page_number * B_PAGE_SIZE);
if (sourcePage->state != PAGE_STATE_MODIFIED)
vm_page_set_state(sourcePage, PAGE_STATE_ACTIVE);
@ -4860,9 +4837,47 @@ found:
status_t
vm_get_physical_page(addr_t paddr, addr_t *_vaddr, uint32 flags)
vm_get_physical_page(addr_t paddr, addr_t* _vaddr, void** _handle)
{
return (*vm_kernel_address_space()->translation_map.ops->get_physical_page)(paddr, _vaddr, flags);
return vm_kernel_address_space()->translation_map.ops->get_physical_page(
paddr, _vaddr, _handle);
}
status_t
vm_put_physical_page(addr_t vaddr, void* handle)
{
return vm_kernel_address_space()->translation_map.ops->put_physical_page(
vaddr, handle);
}
status_t
vm_get_physical_page_current_cpu(addr_t paddr, addr_t* _vaddr, void** _handle)
{
return vm_kernel_address_space()->translation_map.ops
->get_physical_page_current_cpu(paddr, _vaddr, _handle);
}
status_t
vm_put_physical_page_current_cpu(addr_t vaddr, void* handle)
{
return vm_kernel_address_space()->translation_map.ops
->put_physical_page_current_cpu(vaddr, handle);
}
status_t
vm_get_physical_page_debug(addr_t paddr, addr_t* _vaddr, void** _handle)
{
return vm_kernel_address_space()->translation_map.ops
->get_physical_page_debug(paddr, _vaddr, _handle);
}
status_t
vm_put_physical_page_debug(addr_t vaddr, void* handle)
{
return vm_kernel_address_space()->translation_map.ops
->put_physical_page_debug(vaddr, handle);
}
@ -4903,13 +4918,6 @@ vm_available_not_needed_memory(void)
}
status_t
vm_put_physical_page(addr_t vaddr)
{
return (*vm_kernel_address_space()->translation_map.ops->put_physical_page)(vaddr);
}
void
vm_unreserve_memory(size_t amount)
{
@ -5163,66 +5171,34 @@ vm_resize_area(area_id areaID, size_t newSize, bool kernel)
status_t
memset_physical(addr_t address, int value, size_t length)
vm_memset_physical(addr_t address, int value, size_t length)
{
ThreadCPUPinner _(thread_get_current_thread());
while (length > 0) {
addr_t pageOffset = address % B_PAGE_SIZE;
addr_t virtualAddress;
status_t error = vm_get_physical_page(address - pageOffset,
&virtualAddress, 0);
if (error != B_OK)
return error;
size_t toSet = min_c(length, B_PAGE_SIZE - pageOffset);
memset((void*)(virtualAddress + pageOffset), value, toSet);
vm_put_physical_page(virtualAddress);
length -= toSet;
address += toSet;
}
return B_OK;
return vm_kernel_address_space()->translation_map.ops->memset_physical(
address, value, length);
}
status_t
memcpy_to_physical(addr_t to, const void* _from, size_t length, bool user)
vm_memcpy_from_physical(void* to, addr_t from, size_t length, bool user)
{
const uint8* from = (const uint8*)_from;
addr_t pageOffset = to % B_PAGE_SIZE;
return vm_kernel_address_space()->translation_map.ops->memcpy_from_physical(
to, from, length, user);
}
ThreadCPUPinner _(thread_get_current_thread());
while (length > 0) {
size_t toCopy = min_c(length, B_PAGE_SIZE - pageOffset);
status_t
vm_memcpy_to_physical(addr_t to, const void* _from, size_t length, bool user)
{
return vm_kernel_address_space()->translation_map.ops->memcpy_to_physical(
to, _from, length, user);
}
addr_t virtualAddress;
status_t error = vm_get_physical_page(to - pageOffset, &virtualAddress,
0);
if (error != B_OK)
return error;
if (user) {
error = user_memcpy((void*)(virtualAddress + pageOffset), from,
toCopy);
} else
memcpy((void*)(virtualAddress + pageOffset), from, toCopy);
vm_put_physical_page(virtualAddress);
if (error != B_OK)
return error;
to += toCopy;
from += toCopy;
length -= toCopy;
pageOffset = 0;
}
return B_OK;
void
vm_memcpy_physical_page(addr_t to, addr_t from)
{
return vm_kernel_address_space()->translation_map.ops->memcpy_physical_page(
to, from);
}

9
src/system/kernel/vm/vm_page.cpp
View File

@ -821,13 +821,8 @@ move_page_to_active_or_inactive_queue(vm_page *page, bool dequeued)
static void
clear_page(struct vm_page *page)
{
addr_t virtualAddress;
vm_get_physical_page(page->physical_page_number << PAGE_SHIFT,
&virtualAddress, 0);
memset((void *)virtualAddress, 0, B_PAGE_SIZE);
vm_put_physical_page(virtualAddress);
vm_memset_physical(page->physical_page_number << PAGE_SHIFT, 0,
B_PAGE_SIZE);
}