mirrors/qemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

memory: per-AddressSpace dispatch

Browse Source 
Currently we use a global radix tree to dispatch memory access.  This only
works with a single address space; to support multiple address spaces we
make the radix tree a member of AddressSpace (via an intermediate structure
AddressSpaceDispatch to avoid exposing too many internals).

A side effect is that address_space_io also gains a dispatch table.  When
we remove all the pre-memory-API I/O registrations, we can use that for
dispatching I/O and get rid of the original I/O dispatch.

Signed-off-by: Avi Kivity <avi@redhat.com>
This commit is contained in:
Avi Kivity 2012-10-03 16:22:53 +02:00
parent 0e8a6d47af
commit ac1970fbe8
6 changed files with 194 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cputlb.c
View File

@ -21,6 +21,7 @@
#include "cpu.h"
#include "exec-all.h"
#include "memory.h"
#include "exec-memory.h"
#include "cputlb.h"
@ -251,7 +252,7 @@ void tlb_set_page(CPUArchState *env, target_ulong vaddr,
if (size != TARGET_PAGE_SIZE) {
tlb_add_large_page(env, vaddr, size);
}
section = phys_page_find(paddr >> TARGET_PAGE_BITS);
section = phys_page_find(address_space_memory.dispatch, paddr >> TARGET_PAGE_BITS);
#if defined(DEBUG_TLB)
printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
" prot=%x idx=%d pd=0x%08lx\n",

3
cputlb.h
View File

@ -26,7 +26,8 @@ void tlb_unprotect_code_phys(CPUArchState *env, ram_addr_t ram_addr,
target_ulong vaddr);
void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
uintptr_t length);
MemoryRegionSection *phys_page_find(target_phys_addr_t index);
MemoryRegionSection *phys_page_find(struct AddressSpaceDispatch *d,
target_phys_addr_t index);
void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length);
void tlb_set_dirty(CPUArchState *env, target_ulong vaddr);
extern int tlb_flush_count;

174
exec.c
View File

@ -187,7 +187,6 @@ uintptr_t qemu_host_page_mask;
static void *l1_map[V_L1_SIZE];
#if !defined(CONFIG_USER_ONLY)
typedef struct PhysPageEntry PhysPageEntry;
static MemoryRegionSection *phys_sections;
static unsigned phys_sections_nb, phys_sections_nb_alloc;
@ -196,22 +195,12 @@ static uint16_t phys_section_notdirty;
static uint16_t phys_section_rom;
static uint16_t phys_section_watch;
struct PhysPageEntry {
uint16_t is_leaf : 1;
/* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
uint16_t ptr : 15;
};
/* Simple allocator for PhysPageEntry nodes */
static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
#define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
/* This is a multi-level map on the physical address space.
The bottom level has pointers to MemoryRegionSections. */
static PhysPageEntry phys_map = { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
static void io_mem_init(void);
static void memory_map_init(void);
@ -459,18 +448,19 @@ static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index,
}
}
static void phys_page_set(target_phys_addr_t index, target_phys_addr_t nb,
static void phys_page_set(AddressSpaceDispatch *d,
target_phys_addr_t index, target_phys_addr_t nb,
uint16_t leaf)
{
/* Wildly overreserve - it doesn't matter much. */
phys_map_node_reserve(3 * P_L2_LEVELS);
phys_page_set_level(&phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
}
MemoryRegionSection *phys_page_find(target_phys_addr_t index)
MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, target_phys_addr_t index)
{
PhysPageEntry lp = phys_map;
PhysPageEntry lp = d->phys_map;
PhysPageEntry *p;
int i;
uint16_t s_index = phys_section_unassigned;
@ -1472,7 +1462,7 @@ void tb_invalidate_phys_addr(target_phys_addr_t addr)
ram_addr_t ram_addr;
MemoryRegionSection *section;
section = phys_page_find(addr >> TARGET_PAGE_BITS);
section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
if (!(memory_region_is_ram(section->mr)
|| (section->mr->rom_device && section->mr->readable))) {
return;
@ -2210,9 +2200,9 @@ static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
lp->ptr = PHYS_MAP_NODE_NIL;
}
static void destroy_all_mappings(void)
static void destroy_all_mappings(AddressSpaceDispatch *d)
{
destroy_l2_mapping(&phys_map, P_L2_LEVELS - 1);
destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
phys_map_nodes_reset();
}
@ -2232,12 +2222,12 @@ static void phys_sections_clear(void)
phys_sections_nb = 0;
}
static void register_subpage(MemoryRegionSection *section)
static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
{
subpage_t *subpage;
target_phys_addr_t base = section->offset_within_address_space
& TARGET_PAGE_MASK;
MemoryRegionSection *existing = phys_page_find(base >> TARGET_PAGE_BITS);
MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
MemoryRegionSection subsection = {
.offset_within_address_space = base,
.size = TARGET_PAGE_SIZE,
@ -2249,7 +2239,7 @@ static void register_subpage(MemoryRegionSection *section)
if (!(existing->mr->subpage)) {
subpage = subpage_init(base);
subsection.mr = &subpage->iomem;
phys_page_set(base >> TARGET_PAGE_BITS, 1,
phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
phys_section_add(&subsection));
} else {
subpage = container_of(existing->mr, subpage_t, iomem);
@ -2260,7 +2250,7 @@ static void register_subpage(MemoryRegionSection *section)
}
static void register_multipage(MemoryRegionSection *section)
static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *section)
{
target_phys_addr_t start_addr = section->offset_within_address_space;
ram_addr_t size = section->size;
@ -2270,13 +2260,13 @@ static void register_multipage(MemoryRegionSection *section)
assert(size);
addr = start_addr;
phys_page_set(addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
phys_page_set(d, addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
section_index);
}
void cpu_register_physical_memory_log(MemoryRegionSection *section,
bool readonly)
static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
{
AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
MemoryRegionSection now = *section, remain = *section;
if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
@ -2284,7 +2274,7 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section,
now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
- now.offset_within_address_space,
now.size);
register_subpage(&now);
register_subpage(d, &now);
remain.size -= now.size;
remain.offset_within_address_space += now.size;
remain.offset_within_region += now.size;
@ -2293,10 +2283,10 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section,
now = remain;
if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
now.size = TARGET_PAGE_SIZE;
register_subpage(&now);
register_subpage(d, &now);
} else {
now.size &= TARGET_PAGE_MASK;
register_multipage(&now);
register_multipage(d, &now);
}
remain.size -= now.size;
remain.offset_within_address_space += now.size;
@ -2304,7 +2294,7 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section,
}
now = remain;
if (now.size) {
register_subpage(&now);
register_subpage(d, &now);
}
}
@ -3155,11 +3145,17 @@ static void io_mem_init(void)
"watch", UINT64_MAX);
}
static void mem_begin(MemoryListener *listener)
{
AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
destroy_all_mappings(d);
d->phys_map.ptr = PHYS_MAP_NODE_NIL;
}
static void core_begin(MemoryListener *listener)
{
destroy_all_mappings();
phys_sections_clear();
phys_map.ptr = PHYS_MAP_NODE_NIL;
phys_section_unassigned = dummy_section(&io_mem_unassigned);
phys_section_notdirty = dummy_section(&io_mem_notdirty);
phys_section_rom = dummy_section(&io_mem_rom);
@ -3178,18 +3174,6 @@ static void tcg_commit(MemoryListener *listener)
}
}
static void core_region_add(MemoryListener *listener,
MemoryRegionSection *section)
{
cpu_register_physical_memory_log(section, section->readonly);
}
static void core_region_nop(MemoryListener *listener,
MemoryRegionSection *section)
{
cpu_register_physical_memory_log(section, section->readonly);
}
static void core_log_global_start(MemoryListener *listener)
{
cpu_physical_memory_set_dirty_tracking(1);
@ -3220,11 +3204,9 @@ static void io_region_del(MemoryListener *listener,
static MemoryListener core_memory_listener = {
.begin = core_begin,
.region_add = core_region_add,
.region_nop = core_region_nop,
.log_global_start = core_log_global_start,
.log_global_stop = core_log_global_stop,
.priority = 0,
.priority = 1,
};
static MemoryListener io_memory_listener = {
@ -3237,6 +3219,21 @@ static MemoryListener tcg_memory_listener = {
.commit = tcg_commit,
};
void address_space_init_dispatch(AddressSpace *as)
{
AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
d->listener = (MemoryListener) {
.begin = mem_begin,
.region_add = mem_add,
.region_nop = mem_add,
.priority = 0,
};
as->dispatch = d;
memory_listener_register(&d->listener, as);
}
static void memory_map_init(void)
{
system_memory = g_malloc(sizeof(*system_memory));
@ -3321,9 +3318,10 @@ static void invalidate_and_set_dirty(target_phys_addr_t addr,
xen_modified_memory(addr, length);
}
void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
int len, int is_write)
void address_space_rw(AddressSpace *as, target_phys_addr_t addr, uint8_t *buf,
int len, bool is_write)
{
AddressSpaceDispatch *d = as->dispatch;
int l;
uint8_t *ptr;
uint32_t val;
@ -3335,7 +3333,7 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
l = (page + TARGET_PAGE_SIZE) - addr;
if (l > len)
l = len;
section = phys_page_find(page >> TARGET_PAGE_BITS);
section = phys_page_find(d, page >> TARGET_PAGE_BITS);
if (is_write) {
if (!memory_region_is_ram(section->mr)) {
@ -3406,10 +3404,36 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
}
}
void address_space_write(AddressSpace *as, target_phys_addr_t addr,
const uint8_t *buf, int len)
{
address_space_rw(as, addr, (uint8_t *)buf, len, true);
}
/**
* address_space_read: read from an address space.
*
* @as: #AddressSpace to be accessed
* @addr: address within that address space
* @buf: buffer with the data transferred
*/
void address_space_read(AddressSpace *as, target_phys_addr_t addr, uint8_t *buf, int len)
{
address_space_rw(as, addr, buf, len, false);
}
void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
int len, int is_write)
{
return address_space_rw(&address_space_memory, addr, buf, len, is_write);
}
/* used for ROM loading : can write in RAM and ROM */
void cpu_physical_memory_write_rom(target_phys_addr_t addr,
const uint8_t *buf, int len)
{
AddressSpaceDispatch *d = address_space_memory.dispatch;
int l;
uint8_t *ptr;
target_phys_addr_t page;
@ -3420,7 +3444,7 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr,
l = (page + TARGET_PAGE_SIZE) - addr;
if (l > len)
l = len;
section = phys_page_find(page >> TARGET_PAGE_BITS);
section = phys_page_find(d, page >> TARGET_PAGE_BITS);
if (!(memory_region_is_ram(section->mr) ||
memory_region_is_romd(section->mr))) {
@ -3494,10 +3518,12 @@ static void cpu_notify_map_clients(void)
* Use cpu_register_map_client() to know when retrying the map operation is
* likely to succeed.
*/
void *cpu_physical_memory_map(target_phys_addr_t addr,
target_phys_addr_t *plen,
int is_write)
void *address_space_map(AddressSpace *as,
target_phys_addr_t addr,
target_phys_addr_t *plen,
bool is_write)
{
AddressSpaceDispatch *d = as->dispatch;
target_phys_addr_t len = *plen;
target_phys_addr_t todo = 0;
int l;
@ -3512,7 +3538,7 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
l = (page + TARGET_PAGE_SIZE) - addr;
if (l > len)
l = len;
section = phys_page_find(page >> TARGET_PAGE_BITS);
section = phys_page_find(d, page >> TARGET_PAGE_BITS);
if (!(memory_region_is_ram(section->mr) && !section->readonly)) {
if (todo || bounce.buffer) {
@ -3522,7 +3548,7 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
bounce.addr = addr;
bounce.len = l;
if (!is_write) {
cpu_physical_memory_read(addr, bounce.buffer, l);
address_space_read(as, addr, bounce.buffer, l);
}
*plen = l;
@ -3543,12 +3569,12 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
return ret;
}
/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
/* Unmaps a memory region previously mapped by address_space_map().
* Will also mark the memory as dirty if is_write == 1. access_len gives
* the amount of memory that was actually read or written by the caller.
*/
void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
int is_write, target_phys_addr_t access_len)
void address_space_unmap(AddressSpace *as, void *buffer, target_phys_addr_t len,
int is_write, target_phys_addr_t access_len)
{
if (buffer != bounce.buffer) {
if (is_write) {
@ -3569,13 +3595,26 @@ void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
return;
}
if (is_write) {
cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
address_space_write(as, bounce.addr, bounce.buffer, access_len);
}
qemu_vfree(bounce.buffer);
bounce.buffer = NULL;
cpu_notify_map_clients();
}
void *cpu_physical_memory_map(target_phys_addr_t addr,
target_phys_addr_t *plen,
int is_write)
{
return address_space_map(&address_space_memory, addr, plen, is_write);
}
void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
int is_write, target_phys_addr_t access_len)
{
return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
}
/* warning: addr must be aligned */
static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
enum device_endian endian)
@ -3584,7 +3623,7 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
uint32_t val;
MemoryRegionSection *section;
section = phys_page_find(addr >> TARGET_PAGE_BITS);
section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
if (!(memory_region_is_ram(section->mr) ||
memory_region_is_romd(section->mr))) {
@ -3643,7 +3682,7 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,
uint64_t val;
MemoryRegionSection *section;
section = phys_page_find(addr >> TARGET_PAGE_BITS);
section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
if (!(memory_region_is_ram(section->mr) ||
memory_region_is_romd(section->mr))) {
@ -3710,7 +3749,7 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,
uint64_t val;
MemoryRegionSection *section;
section = phys_page_find(addr >> TARGET_PAGE_BITS);
section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
if (!(memory_region_is_ram(section->mr) ||
memory_region_is_romd(section->mr))) {
@ -3769,7 +3808,7 @@ void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
uint8_t *ptr;
MemoryRegionSection *section;
section = phys_page_find(addr >> TARGET_PAGE_BITS);
section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
if (!memory_region_is_ram(section->mr) || section->readonly) {
addr = memory_region_section_addr(section, addr);
@ -3801,7 +3840,7 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
uint8_t *ptr;
MemoryRegionSection *section;
section = phys_page_find(addr >> TARGET_PAGE_BITS);
section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
if (!memory_region_is_ram(section->mr) || section->readonly) {
addr = memory_region_section_addr(section, addr);
@ -3830,7 +3869,7 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,
uint8_t *ptr;
MemoryRegionSection *section;
section = phys_page_find(addr >> TARGET_PAGE_BITS);
section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
if (!memory_region_is_ram(section->mr) || section->readonly) {
addr = memory_region_section_addr(section, addr);
@ -3897,7 +3936,7 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,
uint8_t *ptr;
MemoryRegionSection *section;
section = phys_page_find(addr >> TARGET_PAGE_BITS);
section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
if (!memory_region_is_ram(section->mr) || section->readonly) {
addr = memory_region_section_addr(section, addr);
@ -4133,7 +4172,8 @@ bool cpu_physical_memory_is_io(target_phys_addr_t phys_addr)
{
MemoryRegionSection *section;
section = phys_page_find(phys_addr >> TARGET_PAGE_BITS);
section = phys_page_find(address_space_memory.dispatch,
phys_addr >> TARGET_PAGE_BITS);
return !(memory_region_is_ram(section->mr) ||
memory_region_is_romd(section->mr));

22
memory-internal.h
View File

@ -22,6 +22,26 @@
#ifndef CONFIG_USER_ONLY
#include "hw/xen.h"
typedef struct PhysPageEntry PhysPageEntry;
struct PhysPageEntry {
uint16_t is_leaf : 1;
/* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
uint16_t ptr : 15;
};
typedef struct AddressSpaceDispatch AddressSpaceDispatch;
struct AddressSpaceDispatch {
/* This is a multi-level map on the physical address space.
* The bottom level has pointers to MemoryRegionSections.
*/
PhysPageEntry phys_map;
MemoryListener listener;
};
void address_space_init_dispatch(AddressSpace *as);
ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
MemoryRegion *mr);
ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr);
@ -30,8 +50,6 @@ void qemu_ram_free_from_ptr(ram_addr_t addr);
struct MemoryRegion;
struct MemoryRegionSection;
void cpu_register_physical_memory_log(struct MemoryRegionSection *section,
bool readonly);
void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);

1
memory.c
View File

@ -1539,6 +1539,7 @@ void address_space_init(AddressSpace *as, MemoryRegion *root)
QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link);
as->name = NULL;
memory_region_transaction_commit();
address_space_init_dispatch(as);
}
uint64_t io_mem_read(MemoryRegion *mr, target_phys_addr_t addr, unsigned size)

62
memory.h
View File

@ -169,6 +169,7 @@ struct AddressSpace {
struct FlatView *current_map;
int ioeventfd_nb;
struct MemoryRegionIoeventfd *ioeventfds;
struct AddressSpaceDispatch *dispatch;
QTAILQ_ENTRY(AddressSpace) address_spaces_link;
};
@ -803,6 +804,67 @@ void mtree_info(fprintf_function mon_printf, void *f);
*/
void address_space_init(AddressSpace *as, MemoryRegion *root);
/**
* address_space_rw: read from or write to an address space.
*
* @as: #AddressSpace to be accessed
* @addr: address within that address space
* @buf: buffer with the data transferred
* @is_write: indicates the transfer direction
*/
void address_space_rw(AddressSpace *as, target_phys_addr_t addr, uint8_t *buf,
int len, bool is_write);
/**
* address_space_write: write to address space.
*
* @as: #AddressSpace to be accessed
* @addr: address within that address space
* @buf: buffer with the data transferred
*/
void address_space_write(AddressSpace *as, target_phys_addr_t addr,
const uint8_t *buf, int len);
/**
* address_space_read: read from an address space.
*
* @as: #AddressSpace to be accessed
* @addr: address within that address space
* @buf: buffer with the data transferred
*/
void address_space_read(AddressSpace *as, target_phys_addr_t addr, uint8_t *buf, int len);
/* address_space_map: map a physical memory region into a host virtual address
*
* May map a subset of the requested range, given by and returned in @plen.
* May return %NULL if resources needed to perform the mapping are exhausted.
* Use only for reads OR writes - not for read-modify-write operations.
* Use cpu_register_map_client() to know when retrying the map operation is
* likely to succeed.
*
* @as: #AddressSpace to be accessed
* @addr: address within that address space
* @plen: pointer to length of buffer; updated on return
* @is_write: indicates the transfer direction
*/
void *address_space_map(AddressSpace *as, target_phys_addr_t addr,
target_phys_addr_t *plen, bool is_write);
/* address_space_unmap: Unmaps a memory region previously mapped by address_space_map()
*
* Will also mark the memory as dirty if @is_write == %true. @access_len gives
* the amount of memory that was actually read or written by the caller.
*
* @as: #AddressSpace used
* @addr: address within that address space
* @len: buffer length as returned by address_space_map()
* @access_len: amount of data actually transferred
* @is_write: indicates the transfer direction
*/
void address_space_unmap(AddressSpace *as, void *buffer, target_phys_addr_t len,
int is_write, target_phys_addr_t access_len);
#endif
#endif