accel/tcg: Split out tb-maint.c
Move all of the TranslationBlock flushing and page linking code from translate-all.c to tb-maint.c. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
This commit is contained in:
Richard Henderson
parent
55098769bf
commit
8a14b62c3f
@ -35,6 +35,27 @@ typedef struct PageDesc {
|
|||||||
#endif
|
#endif
|
||||||
} PageDesc;
|
} PageDesc;
|
||||||
|
|
||||||
|
/* Size of the L2 (and L3, etc) page tables. */
|
||||||
|
#define V_L2_BITS 10
|
||||||
|
#define V_L2_SIZE (1 << V_L2_BITS)
|
||||||
|
|
||||||
|
/*
|
||||||
|
* L1 Mapping properties
|
||||||
|
*/
|
||||||
|
extern int v_l1_size;
|
||||||
|
extern int v_l1_shift;
|
||||||
|
extern int v_l2_levels;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The bottom level has pointers to PageDesc, and is indexed by
|
||||||
|
* anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
|
||||||
|
*/
|
||||||
|
#define V_L1_MIN_BITS 4
|
||||||
|
#define V_L1_MAX_BITS (V_L2_BITS + 3)
|
||||||
|
#define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
|
||||||
|
|
||||||
|
extern void *l1_map[V_L1_MAX_SIZE];
|
||||||
|
|
||||||
PageDesc *page_find_alloc(tb_page_addr_t index, bool alloc);
|
PageDesc *page_find_alloc(tb_page_addr_t index, bool alloc);
|
||||||
|
|
||||||
static inline PageDesc *page_find(tb_page_addr_t index)
|
static inline PageDesc *page_find(tb_page_addr_t index)
|
||||||
@ -42,12 +63,46 @@ static inline PageDesc *page_find(tb_page_addr_t index)
|
|||||||
return page_find_alloc(index, false);
|
return page_find_alloc(index, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* list iterators for lists of tagged pointers in TranslationBlock */
|
||||||
|
#define TB_FOR_EACH_TAGGED(head, tb, n, field) \
|
||||||
|
for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \
|
||||||
|
tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \
|
||||||
|
tb = (TranslationBlock *)((uintptr_t)tb & ~1))
|
||||||
|
|
||||||
|
#define PAGE_FOR_EACH_TB(pagedesc, tb, n) \
|
||||||
|
TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next)
|
||||||
|
|
||||||
|
#define TB_FOR_EACH_JMP(head_tb, tb, n) \
|
||||||
|
TB_FOR_EACH_TAGGED((head_tb)->jmp_list_head, tb, n, jmp_list_next)
|
||||||
|
|
||||||
|
/* In user-mode page locks aren't used; mmap_lock is enough */
|
||||||
|
#ifdef CONFIG_USER_ONLY
|
||||||
|
#define assert_page_locked(pd) tcg_debug_assert(have_mmap_lock())
|
||||||
|
static inline void page_lock(PageDesc *pd) { }
|
||||||
|
static inline void page_unlock(PageDesc *pd) { }
|
||||||
|
#else
|
||||||
|
#ifdef CONFIG_DEBUG_TCG
|
||||||
|
void do_assert_page_locked(const PageDesc *pd, const char *file, int line);
|
||||||
|
#define assert_page_locked(pd) do_assert_page_locked(pd, __FILE__, __LINE__)
|
||||||
|
#else
|
||||||
|
#define assert_page_locked(pd)
|
||||||
|
#endif
|
||||||
|
void page_lock(PageDesc *pd);
|
||||||
|
void page_unlock(PageDesc *pd);
|
||||||
|
#endif
|
||||||
|
|
||||||
TranslationBlock *tb_gen_code(CPUState *cpu, target_ulong pc,
|
TranslationBlock *tb_gen_code(CPUState *cpu, target_ulong pc,
|
||||||
target_ulong cs_base, uint32_t flags,
|
target_ulong cs_base, uint32_t flags,
|
||||||
int cflags);
|
int cflags);
|
||||||
G_NORETURN void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr);
|
G_NORETURN void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr);
|
||||||
void page_init(void);
|
void page_init(void);
|
||||||
void tb_htable_init(void);
|
void tb_htable_init(void);
|
||||||
|
void tb_reset_jump(TranslationBlock *tb, int n);
|
||||||
|
TranslationBlock *tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
|
||||||
|
tb_page_addr_t phys_page2);
|
||||||
|
bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc);
|
||||||
|
int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
|
||||||
|
uintptr_t searched_pc, bool reset_icount);
|
||||||
|
|
||||||
/* Return the current PC from CPU, which may be cached in TB. */
|
/* Return the current PC from CPU, which may be cached in TB. */
|
||||||
static inline target_ulong log_pc(CPUState *cpu, const TranslationBlock *tb)
|
static inline target_ulong log_pc(CPUState *cpu, const TranslationBlock *tb)
|
||||||
|
|||||||
@ -3,6 +3,7 @@ tcg_ss.add(files(
|
|||||||
'tcg-all.c',
|
'tcg-all.c',
|
||||||
'cpu-exec-common.c',
|
'cpu-exec-common.c',
|
||||||
'cpu-exec.c',
|
'cpu-exec.c',
|
||||||
|
'tb-maint.c',
|
||||||
'tcg-runtime-gvec.c',
|
'tcg-runtime-gvec.c',
|
||||||
'tcg-runtime.c',
|
'tcg-runtime.c',
|
||||||
'translate-all.c',
|
'translate-all.c',
|
||||||
|
|||||||
735
accel/tcg/tb-maint.c
Normal file
735
accel/tcg/tb-maint.c
Normal file
@ -0,0 +1,735 @@
|
|||||||
|
/*
|
||||||
|
* Translation Block Maintaince
|
||||||
|
*
|
||||||
|
* Copyright (c) 2003 Fabrice Bellard
|
||||||
|
*
|
||||||
|
* This library is free software; you can redistribute it and/or
|
||||||
|
* modify it under the terms of the GNU Lesser General Public
|
||||||
|
* License as published by the Free Software Foundation; either
|
||||||
|
* version 2.1 of the License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This library is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
* Lesser General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Lesser General Public
|
||||||
|
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "qemu/osdep.h"
|
||||||
|
#include "exec/cputlb.h"
|
||||||
|
#include "exec/log.h"
|
||||||
|
#include "exec/translate-all.h"
|
||||||
|
#include "sysemu/tcg.h"
|
||||||
|
#include "tcg/tcg.h"
|
||||||
|
#include "tb-hash.h"
|
||||||
|
#include "tb-context.h"
|
||||||
|
#include "internal.h"
|
||||||
|
|
||||||
|
/* FIXME: tb_invalidate_phys_range is declared in different places. */
|
||||||
|
#ifdef CONFIG_USER_ONLY
|
||||||
|
#include "exec/exec-all.h"
|
||||||
|
#else
|
||||||
|
#include "exec/ram_addr.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
static bool tb_cmp(const void *ap, const void *bp)
|
||||||
|
{
|
||||||
|
const TranslationBlock *a = ap;
|
||||||
|
const TranslationBlock *b = bp;
|
||||||
|
|
||||||
|
return ((TARGET_TB_PCREL || tb_pc(a) == tb_pc(b)) &&
|
||||||
|
a->cs_base == b->cs_base &&
|
||||||
|
a->flags == b->flags &&
|
||||||
|
(tb_cflags(a) & ~CF_INVALID) == (tb_cflags(b) & ~CF_INVALID) &&
|
||||||
|
a->trace_vcpu_dstate == b->trace_vcpu_dstate &&
|
||||||
|
a->page_addr[0] == b->page_addr[0] &&
|
||||||
|
a->page_addr[1] == b->page_addr[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
void tb_htable_init(void)
|
||||||
|
{
|
||||||
|
unsigned int mode = QHT_MODE_AUTO_RESIZE;
|
||||||
|
|
||||||
|
qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Set to NULL all the 'first_tb' fields in all PageDescs. */
|
||||||
|
static void page_flush_tb_1(int level, void **lp)
|
||||||
|
{
|
||||||
|
int i;
|
||||||
|
|
||||||
|
if (*lp == NULL) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (level == 0) {
|
||||||
|
PageDesc *pd = *lp;
|
||||||
|
|
||||||
|
for (i = 0; i < V_L2_SIZE; ++i) {
|
||||||
|
page_lock(&pd[i]);
|
||||||
|
pd[i].first_tb = (uintptr_t)NULL;
|
||||||
|
page_unlock(&pd[i]);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
void **pp = *lp;
|
||||||
|
|
||||||
|
for (i = 0; i < V_L2_SIZE; ++i) {
|
||||||
|
page_flush_tb_1(level - 1, pp + i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void page_flush_tb(void)
|
||||||
|
{
|
||||||
|
int i, l1_sz = v_l1_size;
|
||||||
|
|
||||||
|
for (i = 0; i < l1_sz; i++) {
|
||||||
|
page_flush_tb_1(v_l2_levels, l1_map + i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* flush all the translation blocks */
|
||||||
|
static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
|
||||||
|
{
|
||||||
|
bool did_flush = false;
|
||||||
|
|
||||||
|
mmap_lock();
|
||||||
|
/* If it is already been done on request of another CPU, just retry. */
|
||||||
|
if (tb_ctx.tb_flush_count != tb_flush_count.host_int) {
|
||||||
|
goto done;
|
||||||
|
}
|
||||||
|
did_flush = true;
|
||||||
|
|
||||||
|
CPU_FOREACH(cpu) {
|
||||||
|
tcg_flush_jmp_cache(cpu);
|
||||||
|
}
|
||||||
|
|
||||||
|
qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE);
|
||||||
|
page_flush_tb();
|
||||||
|
|
||||||
|
tcg_region_reset_all();
|
||||||
|
/* XXX: flush processor icache at this point if cache flush is expensive */
|
||||||
|
qatomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1);
|
||||||
|
|
||||||
|
done:
|
||||||
|
mmap_unlock();
|
||||||
|
if (did_flush) {
|
||||||
|
qemu_plugin_flush_cb();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void tb_flush(CPUState *cpu)
|
||||||
|
{
|
||||||
|
if (tcg_enabled()) {
|
||||||
|
unsigned tb_flush_count = qatomic_mb_read(&tb_ctx.tb_flush_count);
|
||||||
|
|
||||||
|
if (cpu_in_exclusive_context(cpu)) {
|
||||||
|
do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count));
|
||||||
|
} else {
|
||||||
|
async_safe_run_on_cpu(cpu, do_tb_flush,
|
||||||
|
RUN_ON_CPU_HOST_INT(tb_flush_count));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* user-mode: call with mmap_lock held
|
||||||
|
* !user-mode: call with @pd->lock held
|
||||||
|
*/
|
||||||
|
static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb)
|
||||||
|
{
|
||||||
|
TranslationBlock *tb1;
|
||||||
|
uintptr_t *pprev;
|
||||||
|
unsigned int n1;
|
||||||
|
|
||||||
|
assert_page_locked(pd);
|
||||||
|
pprev = &pd->first_tb;
|
||||||
|
PAGE_FOR_EACH_TB(pd, tb1, n1) {
|
||||||
|
if (tb1 == tb) {
|
||||||
|
*pprev = tb1->page_next[n1];
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pprev = &tb1->page_next[n1];
|
||||||
|
}
|
||||||
|
g_assert_not_reached();
|
||||||
|
}
|
||||||
|
|
||||||
|
/* remove @orig from its @n_orig-th jump list */
|
||||||
|
static inline void tb_remove_from_jmp_list(TranslationBlock *orig, int n_orig)
|
||||||
|
{
|
||||||
|
uintptr_t ptr, ptr_locked;
|
||||||
|
TranslationBlock *dest;
|
||||||
|
TranslationBlock *tb;
|
||||||
|
uintptr_t *pprev;
|
||||||
|
int n;
|
||||||
|
|
||||||
|
/* mark the LSB of jmp_dest[] so that no further jumps can be inserted */
|
||||||
|
ptr = qatomic_or_fetch(&orig->jmp_dest[n_orig], 1);
|
||||||
|
dest = (TranslationBlock *)(ptr & ~1);
|
||||||
|
if (dest == NULL) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
qemu_spin_lock(&dest->jmp_lock);
|
||||||
|
/*
|
||||||
|
* While acquiring the lock, the jump might have been removed if the
|
||||||
|
* destination TB was invalidated; check again.
|
||||||
|
*/
|
||||||
|
ptr_locked = qatomic_read(&orig->jmp_dest[n_orig]);
|
||||||
|
if (ptr_locked != ptr) {
|
||||||
|
qemu_spin_unlock(&dest->jmp_lock);
|
||||||
|
/*
|
||||||
|
* The only possibility is that the jump was unlinked via
|
||||||
|
* tb_jump_unlink(dest). Seeing here another destination would be a bug,
|
||||||
|
* because we set the LSB above.
|
||||||
|
*/
|
||||||
|
g_assert(ptr_locked == 1 && dest->cflags & CF_INVALID);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
/*
|
||||||
|
* We first acquired the lock, and since the destination pointer matches,
|
||||||
|
* we know for sure that @orig is in the jmp list.
|
||||||
|
*/
|
||||||
|
pprev = &dest->jmp_list_head;
|
||||||
|
TB_FOR_EACH_JMP(dest, tb, n) {
|
||||||
|
if (tb == orig && n == n_orig) {
|
||||||
|
*pprev = tb->jmp_list_next[n];
|
||||||
|
/* no need to set orig->jmp_dest[n]; setting the LSB was enough */
|
||||||
|
qemu_spin_unlock(&dest->jmp_lock);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pprev = &tb->jmp_list_next[n];
|
||||||
|
}
|
||||||
|
g_assert_not_reached();
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Reset the jump entry 'n' of a TB so that it is not chained to another TB.
|
||||||
|
*/
|
||||||
|
void tb_reset_jump(TranslationBlock *tb, int n)
|
||||||
|
{
|
||||||
|
uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]);
|
||||||
|
tb_set_jmp_target(tb, n, addr);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* remove any jumps to the TB */
|
||||||
|
static inline void tb_jmp_unlink(TranslationBlock *dest)
|
||||||
|
{
|
||||||
|
TranslationBlock *tb;
|
||||||
|
int n;
|
||||||
|
|
||||||
|
qemu_spin_lock(&dest->jmp_lock);
|
||||||
|
|
||||||
|
TB_FOR_EACH_JMP(dest, tb, n) {
|
||||||
|
tb_reset_jump(tb, n);
|
||||||
|
qatomic_and(&tb->jmp_dest[n], (uintptr_t)NULL | 1);
|
||||||
|
/* No need to clear the list entry; setting the dest ptr is enough */
|
||||||
|
}
|
||||||
|
dest->jmp_list_head = (uintptr_t)NULL;
|
||||||
|
|
||||||
|
qemu_spin_unlock(&dest->jmp_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void tb_jmp_cache_inval_tb(TranslationBlock *tb)
|
||||||
|
{
|
||||||
|
CPUState *cpu;
|
||||||
|
|
||||||
|
if (TARGET_TB_PCREL) {
|
||||||
|
/* A TB may be at any virtual address */
|
||||||
|
CPU_FOREACH(cpu) {
|
||||||
|
tcg_flush_jmp_cache(cpu);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
uint32_t h = tb_jmp_cache_hash_func(tb_pc(tb));
|
||||||
|
|
||||||
|
CPU_FOREACH(cpu) {
|
||||||
|
CPUJumpCache *jc = cpu->tb_jmp_cache;
|
||||||
|
|
||||||
|
if (qatomic_read(&jc->array[h].tb) == tb) {
|
||||||
|
qatomic_set(&jc->array[h].tb, NULL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* In user-mode, call with mmap_lock held.
|
||||||
|
* In !user-mode, if @rm_from_page_list is set, call with the TB's pages'
|
||||||
|
* locks held.
|
||||||
|
*/
|
||||||
|
static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list)
|
||||||
|
{
|
||||||
|
PageDesc *p;
|
||||||
|
uint32_t h;
|
||||||
|
tb_page_addr_t phys_pc;
|
||||||
|
uint32_t orig_cflags = tb_cflags(tb);
|
||||||
|
|
||||||
|
assert_memory_lock();
|
||||||
|
|
||||||
|
/* make sure no further incoming jumps will be chained to this TB */
|
||||||
|
qemu_spin_lock(&tb->jmp_lock);
|
||||||
|
qatomic_set(&tb->cflags, tb->cflags | CF_INVALID);
|
||||||
|
qemu_spin_unlock(&tb->jmp_lock);
|
||||||
|
|
||||||
|
/* remove the TB from the hash list */
|
||||||
|
phys_pc = tb->page_addr[0];
|
||||||
|
h = tb_hash_func(phys_pc, (TARGET_TB_PCREL ? 0 : tb_pc(tb)),
|
||||||
|
tb->flags, orig_cflags, tb->trace_vcpu_dstate);
|
||||||
|
if (!qht_remove(&tb_ctx.htable, tb, h)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* remove the TB from the page list */
|
||||||
|
if (rm_from_page_list) {
|
||||||
|
p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
|
||||||
|
tb_page_remove(p, tb);
|
||||||
|
if (tb->page_addr[1] != -1) {
|
||||||
|
p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
|
||||||
|
tb_page_remove(p, tb);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* remove the TB from the hash list */
|
||||||
|
tb_jmp_cache_inval_tb(tb);
|
||||||
|
|
||||||
|
/* suppress this TB from the two jump lists */
|
||||||
|
tb_remove_from_jmp_list(tb, 0);
|
||||||
|
tb_remove_from_jmp_list(tb, 1);
|
||||||
|
|
||||||
|
/* suppress any remaining jumps to this TB */
|
||||||
|
tb_jmp_unlink(tb);
|
||||||
|
|
||||||
|
qatomic_set(&tb_ctx.tb_phys_invalidate_count,
|
||||||
|
tb_ctx.tb_phys_invalidate_count + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void tb_phys_invalidate__locked(TranslationBlock *tb)
|
||||||
|
{
|
||||||
|
qemu_thread_jit_write();
|
||||||
|
do_tb_phys_invalidate(tb, true);
|
||||||
|
qemu_thread_jit_execute();
|
||||||
|
}
|
||||||
|
|
||||||
|
static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
|
||||||
|
PageDesc **ret_p2, tb_page_addr_t phys2, bool alloc)
|
||||||
|
{
|
||||||
|
PageDesc *p1, *p2;
|
||||||
|
tb_page_addr_t page1;
|
||||||
|
tb_page_addr_t page2;
|
||||||
|
|
||||||
|
assert_memory_lock();
|
||||||
|
g_assert(phys1 != -1);
|
||||||
|
|
||||||
|
page1 = phys1 >> TARGET_PAGE_BITS;
|
||||||
|
page2 = phys2 >> TARGET_PAGE_BITS;
|
||||||
|
|
||||||
|
p1 = page_find_alloc(page1, alloc);
|
||||||
|
if (ret_p1) {
|
||||||
|
*ret_p1 = p1;
|
||||||
|
}
|
||||||
|
if (likely(phys2 == -1)) {
|
||||||
|
page_lock(p1);
|
||||||
|
return;
|
||||||
|
} else if (page1 == page2) {
|
||||||
|
page_lock(p1);
|
||||||
|
if (ret_p2) {
|
||||||
|
*ret_p2 = p1;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
p2 = page_find_alloc(page2, alloc);
|
||||||
|
if (ret_p2) {
|
||||||
|
*ret_p2 = p2;
|
||||||
|
}
|
||||||
|
if (page1 < page2) {
|
||||||
|
page_lock(p1);
|
||||||
|
page_lock(p2);
|
||||||
|
} else {
|
||||||
|
page_lock(p2);
|
||||||
|
page_lock(p1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef CONFIG_USER_ONLY
|
||||||
|
static inline void page_lock_tb(const TranslationBlock *tb) { }
|
||||||
|
static inline void page_unlock_tb(const TranslationBlock *tb) { }
|
||||||
|
#else
|
||||||
|
/* lock the page(s) of a TB in the correct acquisition order */
|
||||||
|
static void page_lock_tb(const TranslationBlock *tb)
|
||||||
|
{
|
||||||
|
page_lock_pair(NULL, tb->page_addr[0], NULL, tb->page_addr[1], false);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void page_unlock_tb(const TranslationBlock *tb)
|
||||||
|
{
|
||||||
|
PageDesc *p1 = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
|
||||||
|
|
||||||
|
page_unlock(p1);
|
||||||
|
if (unlikely(tb->page_addr[1] != -1)) {
|
||||||
|
PageDesc *p2 = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
|
||||||
|
|
||||||
|
if (p2 != p1) {
|
||||||
|
page_unlock(p2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Invalidate one TB.
|
||||||
|
* Called with mmap_lock held in user-mode.
|
||||||
|
*/
|
||||||
|
void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
|
||||||
|
{
|
||||||
|
if (page_addr == -1 && tb->page_addr[0] != -1) {
|
||||||
|
page_lock_tb(tb);
|
||||||
|
do_tb_phys_invalidate(tb, true);
|
||||||
|
page_unlock_tb(tb);
|
||||||
|
} else {
|
||||||
|
do_tb_phys_invalidate(tb, false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Add the tb in the target page and protect it if necessary.
|
||||||
|
* Called with mmap_lock held for user-mode emulation.
|
||||||
|
* Called with @p->lock held in !user-mode.
|
||||||
|
*/
|
||||||
|
static inline void tb_page_add(PageDesc *p, TranslationBlock *tb,
|
||||||
|
unsigned int n, tb_page_addr_t page_addr)
|
||||||
|
{
|
||||||
|
#ifndef CONFIG_USER_ONLY
|
||||||
|
bool page_already_protected;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
assert_page_locked(p);
|
||||||
|
|
||||||
|
tb->page_addr[n] = page_addr;
|
||||||
|
tb->page_next[n] = p->first_tb;
|
||||||
|
#ifndef CONFIG_USER_ONLY
|
||||||
|
page_already_protected = p->first_tb != (uintptr_t)NULL;
|
||||||
|
#endif
|
||||||
|
p->first_tb = (uintptr_t)tb | n;
|
||||||
|
|
||||||
|
#if defined(CONFIG_USER_ONLY)
|
||||||
|
/* translator_loop() must have made all TB pages non-writable */
|
||||||
|
assert(!(p->flags & PAGE_WRITE));
|
||||||
|
#else
|
||||||
|
/*
|
||||||
|
* If some code is already present, then the pages are already
|
||||||
|
* protected. So we handle the case where only the first TB is
|
||||||
|
* allocated in a physical page.
|
||||||
|
*/
|
||||||
|
if (!page_already_protected) {
|
||||||
|
tlb_protect_code(page_addr);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Add a new TB and link it to the physical page tables. phys_page2 is
|
||||||
|
* (-1) to indicate that only one page contains the TB.
|
||||||
|
*
|
||||||
|
* Called with mmap_lock held for user-mode emulation.
|
||||||
|
*
|
||||||
|
* Returns a pointer @tb, or a pointer to an existing TB that matches @tb.
|
||||||
|
* Note that in !user-mode, another thread might have already added a TB
|
||||||
|
* for the same block of guest code that @tb corresponds to. In that case,
|
||||||
|
* the caller should discard the original @tb, and use instead the returned TB.
|
||||||
|
*/
|
||||||
|
TranslationBlock *tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
|
||||||
|
tb_page_addr_t phys_page2)
|
||||||
|
{
|
||||||
|
PageDesc *p;
|
||||||
|
PageDesc *p2 = NULL;
|
||||||
|
void *existing_tb = NULL;
|
||||||
|
uint32_t h;
|
||||||
|
|
||||||
|
assert_memory_lock();
|
||||||
|
tcg_debug_assert(!(tb->cflags & CF_INVALID));
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Add the TB to the page list, acquiring first the pages's locks.
|
||||||
|
* We keep the locks held until after inserting the TB in the hash table,
|
||||||
|
* so that if the insertion fails we know for sure that the TBs are still
|
||||||
|
* in the page descriptors.
|
||||||
|
* Note that inserting into the hash table first isn't an option, since
|
||||||
|
* we can only insert TBs that are fully initialized.
|
||||||
|
*/
|
||||||
|
page_lock_pair(&p, phys_pc, &p2, phys_page2, true);
|
||||||
|
tb_page_add(p, tb, 0, phys_pc);
|
||||||
|
if (p2) {
|
||||||
|
tb_page_add(p2, tb, 1, phys_page2);
|
||||||
|
} else {
|
||||||
|
tb->page_addr[1] = -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* add in the hash table */
|
||||||
|
h = tb_hash_func(phys_pc, (TARGET_TB_PCREL ? 0 : tb_pc(tb)),
|
||||||
|
tb->flags, tb->cflags, tb->trace_vcpu_dstate);
|
||||||
|
qht_insert(&tb_ctx.htable, tb, h, &existing_tb);
|
||||||
|
|
||||||
|
/* remove TB from the page(s) if we couldn't insert it */
|
||||||
|
if (unlikely(existing_tb)) {
|
||||||
|
tb_page_remove(p, tb);
|
||||||
|
if (p2) {
|
||||||
|
tb_page_remove(p2, tb);
|
||||||
|
}
|
||||||
|
tb = existing_tb;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (p2 && p2 != p) {
|
||||||
|
page_unlock(p2);
|
||||||
|
}
|
||||||
|
page_unlock(p);
|
||||||
|
return tb;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* @p must be non-NULL.
|
||||||
|
* user-mode: call with mmap_lock held.
|
||||||
|
* !user-mode: call with all @pages locked.
|
||||||
|
*/
|
||||||
|
static void
|
||||||
|
tb_invalidate_phys_page_range__locked(struct page_collection *pages,
|
||||||
|
PageDesc *p, tb_page_addr_t start,
|
||||||
|
tb_page_addr_t end,
|
||||||
|
uintptr_t retaddr)
|
||||||
|
{
|
||||||
|
TranslationBlock *tb;
|
||||||
|
tb_page_addr_t tb_start, tb_end;
|
||||||
|
int n;
|
||||||
|
#ifdef TARGET_HAS_PRECISE_SMC
|
||||||
|
CPUState *cpu = current_cpu;
|
||||||
|
CPUArchState *env = NULL;
|
||||||
|
bool current_tb_not_found = retaddr != 0;
|
||||||
|
bool current_tb_modified = false;
|
||||||
|
TranslationBlock *current_tb = NULL;
|
||||||
|
target_ulong current_pc = 0;
|
||||||
|
target_ulong current_cs_base = 0;
|
||||||
|
uint32_t current_flags = 0;
|
||||||
|
#endif /* TARGET_HAS_PRECISE_SMC */
|
||||||
|
|
||||||
|
assert_page_locked(p);
|
||||||
|
|
||||||
|
#if defined(TARGET_HAS_PRECISE_SMC)
|
||||||
|
if (cpu != NULL) {
|
||||||
|
env = cpu->env_ptr;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/*
|
||||||
|
* We remove all the TBs in the range [start, end[.
|
||||||
|
* XXX: see if in some cases it could be faster to invalidate all the code
|
||||||
|
*/
|
||||||
|
PAGE_FOR_EACH_TB(p, tb, n) {
|
||||||
|
assert_page_locked(p);
|
||||||
|
/* NOTE: this is subtle as a TB may span two physical pages */
|
||||||
|
if (n == 0) {
|
||||||
|
/* NOTE: tb_end may be after the end of the page, but
|
||||||
|
it is not a problem */
|
||||||
|
tb_start = tb->page_addr[0];
|
||||||
|
tb_end = tb_start + tb->size;
|
||||||
|
} else {
|
||||||
|
tb_start = tb->page_addr[1];
|
||||||
|
tb_end = tb_start + ((tb->page_addr[0] + tb->size)
|
||||||
|
& ~TARGET_PAGE_MASK);
|
||||||
|
}
|
||||||
|
if (!(tb_end <= start || tb_start >= end)) {
|
||||||
|
#ifdef TARGET_HAS_PRECISE_SMC
|
||||||
|
if (current_tb_not_found) {
|
||||||
|
current_tb_not_found = false;
|
||||||
|
/* now we have a real cpu fault */
|
||||||
|
current_tb = tcg_tb_lookup(retaddr);
|
||||||
|
}
|
||||||
|
if (current_tb == tb &&
|
||||||
|
(tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
|
||||||
|
/*
|
||||||
|
* If we are modifying the current TB, we must stop
|
||||||
|
* its execution. We could be more precise by checking
|
||||||
|
* that the modification is after the current PC, but it
|
||||||
|
* would require a specialized function to partially
|
||||||
|
* restore the CPU state.
|
||||||
|
*/
|
||||||
|
current_tb_modified = true;
|
||||||
|
cpu_restore_state_from_tb(cpu, current_tb, retaddr, true);
|
||||||
|
cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base,
|
||||||
|
¤t_flags);
|
||||||
|
}
|
||||||
|
#endif /* TARGET_HAS_PRECISE_SMC */
|
||||||
|
tb_phys_invalidate__locked(tb);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#if !defined(CONFIG_USER_ONLY)
|
||||||
|
/* if no code remaining, no need to continue to use slow writes */
|
||||||
|
if (!p->first_tb) {
|
||||||
|
tlb_unprotect_code(start);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
#ifdef TARGET_HAS_PRECISE_SMC
|
||||||
|
if (current_tb_modified) {
|
||||||
|
page_collection_unlock(pages);
|
||||||
|
/* Force execution of one insn next time. */
|
||||||
|
cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu);
|
||||||
|
mmap_unlock();
|
||||||
|
cpu_loop_exit_noexc(cpu);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Invalidate all TBs which intersect with the target physical address range
|
||||||
|
* [start;end[. NOTE: start and end must refer to the *same* physical page.
|
||||||
|
* 'is_cpu_write_access' should be true if called from a real cpu write
|
||||||
|
* access: the virtual CPU will exit the current TB if code is modified inside
|
||||||
|
* this TB.
|
||||||
|
*
|
||||||
|
* Called with mmap_lock held for user-mode emulation
|
||||||
|
*/
|
||||||
|
void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end)
|
||||||
|
{
|
||||||
|
struct page_collection *pages;
|
||||||
|
PageDesc *p;
|
||||||
|
|
||||||
|
assert_memory_lock();
|
||||||
|
|
||||||
|
p = page_find(start >> TARGET_PAGE_BITS);
|
||||||
|
if (p == NULL) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
pages = page_collection_lock(start, end);
|
||||||
|
tb_invalidate_phys_page_range__locked(pages, p, start, end, 0);
|
||||||
|
page_collection_unlock(pages);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Invalidate all TBs which intersect with the target physical address range
|
||||||
|
* [start;end[. NOTE: start and end may refer to *different* physical pages.
|
||||||
|
* 'is_cpu_write_access' should be true if called from a real cpu write
|
||||||
|
* access: the virtual CPU will exit the current TB if code is modified inside
|
||||||
|
* this TB.
|
||||||
|
*
|
||||||
|
* Called with mmap_lock held for user-mode emulation.
|
||||||
|
*/
|
||||||
|
#ifdef CONFIG_SOFTMMU
|
||||||
|
void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end)
|
||||||
|
#else
|
||||||
|
void tb_invalidate_phys_range(target_ulong start, target_ulong end)
|
||||||
|
#endif
|
||||||
|
{
|
||||||
|
struct page_collection *pages;
|
||||||
|
tb_page_addr_t next;
|
||||||
|
|
||||||
|
assert_memory_lock();
|
||||||
|
|
||||||
|
pages = page_collection_lock(start, end);
|
||||||
|
for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
|
||||||
|
start < end;
|
||||||
|
start = next, next += TARGET_PAGE_SIZE) {
|
||||||
|
PageDesc *pd = page_find(start >> TARGET_PAGE_BITS);
|
||||||
|
tb_page_addr_t bound = MIN(next, end);
|
||||||
|
|
||||||
|
if (pd == NULL) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0);
|
||||||
|
}
|
||||||
|
page_collection_unlock(pages);
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef CONFIG_SOFTMMU
|
||||||
|
/*
|
||||||
|
* len must be <= 8 and start must be a multiple of len.
|
||||||
|
* Called via softmmu_template.h when code areas are written to with
|
||||||
|
* iothread mutex not held.
|
||||||
|
*
|
||||||
|
* Call with all @pages in the range [@start, @start + len[ locked.
|
||||||
|
*/
|
||||||
|
void tb_invalidate_phys_page_fast(struct page_collection *pages,
|
||||||
|
tb_page_addr_t start, int len,
|
||||||
|
uintptr_t retaddr)
|
||||||
|
{
|
||||||
|
PageDesc *p;
|
||||||
|
|
||||||
|
assert_memory_lock();
|
||||||
|
|
||||||
|
p = page_find(start >> TARGET_PAGE_BITS);
|
||||||
|
if (!p) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_page_locked(p);
|
||||||
|
tb_invalidate_phys_page_range__locked(pages, p, start, start + len,
|
||||||
|
retaddr);
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
/*
|
||||||
|
* Called with mmap_lock held. If pc is not 0 then it indicates the
|
||||||
|
* host PC of the faulting store instruction that caused this invalidate.
|
||||||
|
* Returns true if the caller needs to abort execution of the current
|
||||||
|
* TB (because it was modified by this store and the guest CPU has
|
||||||
|
* precise-SMC semantics).
|
||||||
|
*/
|
||||||
|
bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)
|
||||||
|
{
|
||||||
|
TranslationBlock *tb;
|
||||||
|
PageDesc *p;
|
||||||
|
int n;
|
||||||
|
#ifdef TARGET_HAS_PRECISE_SMC
|
||||||
|
TranslationBlock *current_tb = NULL;
|
||||||
|
CPUState *cpu = current_cpu;
|
||||||
|
CPUArchState *env = NULL;
|
||||||
|
int current_tb_modified = 0;
|
||||||
|
target_ulong current_pc = 0;
|
||||||
|
target_ulong current_cs_base = 0;
|
||||||
|
uint32_t current_flags = 0;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
assert_memory_lock();
|
||||||
|
|
||||||
|
addr &= TARGET_PAGE_MASK;
|
||||||
|
p = page_find(addr >> TARGET_PAGE_BITS);
|
||||||
|
if (!p) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef TARGET_HAS_PRECISE_SMC
|
||||||
|
if (p->first_tb && pc != 0) {
|
||||||
|
current_tb = tcg_tb_lookup(pc);
|
||||||
|
}
|
||||||
|
if (cpu != NULL) {
|
||||||
|
env = cpu->env_ptr;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
assert_page_locked(p);
|
||||||
|
PAGE_FOR_EACH_TB(p, tb, n) {
|
||||||
|
#ifdef TARGET_HAS_PRECISE_SMC
|
||||||
|
if (current_tb == tb &&
|
||||||
|
(tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
|
||||||
|
/*
|
||||||
|
* If we are modifying the current TB, we must stop its execution.
|
||||||
|
* We could be more precise by checking that the modification is
|
||||||
|
* after the current PC, but it would require a specialized
|
||||||
|
* function to partially restore the CPU state.
|
||||||
|
*/
|
||||||
|
current_tb_modified = 1;
|
||||||
|
cpu_restore_state_from_tb(cpu, current_tb, pc, true);
|
||||||
|
cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base,
|
||||||
|
¤t_flags);
|
||||||
|
}
|
||||||
|
#endif /* TARGET_HAS_PRECISE_SMC */
|
||||||
|
tb_phys_invalidate(tb, addr);
|
||||||
|
}
|
||||||
|
p->first_tb = (uintptr_t)NULL;
|
||||||
|
#ifdef TARGET_HAS_PRECISE_SMC
|
||||||
|
if (current_tb_modified) {
|
||||||
|
/* Force execution of one insn next time. */
|
||||||
|
cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
@ -109,18 +109,6 @@ struct page_collection {
|
|||||||
struct page_entry *max;
|
struct page_entry *max;
|
||||||
};
|
};
|
||||||
|
|
||||||
/* list iterators for lists of tagged pointers in TranslationBlock */
|
|
||||||
#define TB_FOR_EACH_TAGGED(head, tb, n, field) \
|
|
||||||
for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \
|
|
||||||
tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \
|
|
||||||
tb = (TranslationBlock *)((uintptr_t)tb & ~1))
|
|
||||||
|
|
||||||
#define PAGE_FOR_EACH_TB(pagedesc, tb, n) \
|
|
||||||
TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next)
|
|
||||||
|
|
||||||
#define TB_FOR_EACH_JMP(head_tb, tb, n) \
|
|
||||||
TB_FOR_EACH_TAGGED((head_tb)->jmp_list_head, tb, n, jmp_list_next)
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* In system mode we want L1_MAP to be based on ram offsets,
|
* In system mode we want L1_MAP to be based on ram offsets,
|
||||||
* while in user mode we want it to be based on virtual addresses.
|
* while in user mode we want it to be based on virtual addresses.
|
||||||
@ -138,10 +126,6 @@ struct page_collection {
|
|||||||
# define L1_MAP_ADDR_SPACE_BITS MIN(HOST_LONG_BITS, TARGET_ABI_BITS)
|
# define L1_MAP_ADDR_SPACE_BITS MIN(HOST_LONG_BITS, TARGET_ABI_BITS)
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
/* Size of the L2 (and L3, etc) page tables. */
|
|
||||||
#define V_L2_BITS 10
|
|
||||||
#define V_L2_SIZE (1 << V_L2_BITS)
|
|
||||||
|
|
||||||
/* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */
|
/* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */
|
||||||
QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS >
|
QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS >
|
||||||
sizeof_field(TranslationBlock, trace_vcpu_dstate)
|
sizeof_field(TranslationBlock, trace_vcpu_dstate)
|
||||||
@ -150,18 +134,11 @@ QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS >
|
|||||||
/*
|
/*
|
||||||
* L1 Mapping properties
|
* L1 Mapping properties
|
||||||
*/
|
*/
|
||||||
static int v_l1_size;
|
int v_l1_size;
|
||||||
static int v_l1_shift;
|
int v_l1_shift;
|
||||||
static int v_l2_levels;
|
int v_l2_levels;
|
||||||
|
|
||||||
/* The bottom level has pointers to PageDesc, and is indexed by
|
void *l1_map[V_L1_MAX_SIZE];
|
||||||
* anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
|
|
||||||
*/
|
|
||||||
#define V_L1_MIN_BITS 4
|
|
||||||
#define V_L1_MAX_BITS (V_L2_BITS + 3)
|
|
||||||
#define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
|
|
||||||
|
|
||||||
static void *l1_map[V_L1_MAX_SIZE];
|
|
||||||
|
|
||||||
TBContext tb_ctx;
|
TBContext tb_ctx;
|
||||||
|
|
||||||
@ -274,8 +251,8 @@ static int encode_search(TranslationBlock *tb, uint8_t *block)
|
|||||||
* When reset_icount is true, current TB will be interrupted and
|
* When reset_icount is true, current TB will be interrupted and
|
||||||
* icount should be recalculated.
|
* icount should be recalculated.
|
||||||
*/
|
*/
|
||||||
static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
|
int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
|
||||||
uintptr_t searched_pc, bool reset_icount)
|
uintptr_t searched_pc, bool reset_icount)
|
||||||
{
|
{
|
||||||
target_ulong data[TARGET_INSN_START_WORDS];
|
target_ulong data[TARGET_INSN_START_WORDS];
|
||||||
uintptr_t host_pc = (uintptr_t)tb->tc.ptr;
|
uintptr_t host_pc = (uintptr_t)tb->tc.ptr;
|
||||||
@ -487,26 +464,8 @@ PageDesc *page_find_alloc(tb_page_addr_t index, bool alloc)
|
|||||||
return pd + (index & (V_L2_SIZE - 1));
|
return pd + (index & (V_L2_SIZE - 1));
|
||||||
}
|
}
|
||||||
|
|
||||||
static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
|
|
||||||
PageDesc **ret_p2, tb_page_addr_t phys2, bool alloc);
|
|
||||||
|
|
||||||
/* In user-mode page locks aren't used; mmap_lock is enough */
|
/* In user-mode page locks aren't used; mmap_lock is enough */
|
||||||
#ifdef CONFIG_USER_ONLY
|
#ifdef CONFIG_USER_ONLY
|
||||||
|
|
||||||
#define assert_page_locked(pd) tcg_debug_assert(have_mmap_lock())
|
|
||||||
|
|
||||||
static inline void page_lock(PageDesc *pd)
|
|
||||||
{ }
|
|
||||||
|
|
||||||
static inline void page_unlock(PageDesc *pd)
|
|
||||||
{ }
|
|
||||||
|
|
||||||
static inline void page_lock_tb(const TranslationBlock *tb)
|
|
||||||
{ }
|
|
||||||
|
|
||||||
static inline void page_unlock_tb(const TranslationBlock *tb)
|
|
||||||
{ }
|
|
||||||
|
|
||||||
struct page_collection *
|
struct page_collection *
|
||||||
page_collection_lock(tb_page_addr_t start, tb_page_addr_t end)
|
page_collection_lock(tb_page_addr_t start, tb_page_addr_t end)
|
||||||
{
|
{
|
||||||
@ -555,8 +514,7 @@ static void page_unlock__debug(const PageDesc *pd)
|
|||||||
g_assert(removed);
|
g_assert(removed);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
void do_assert_page_locked(const PageDesc *pd, const char *file, int line)
|
||||||
do_assert_page_locked(const PageDesc *pd, const char *file, int line)
|
|
||||||
{
|
{
|
||||||
if (unlikely(!page_is_locked(pd))) {
|
if (unlikely(!page_is_locked(pd))) {
|
||||||
error_report("assert_page_lock: PageDesc %p not locked @ %s:%d",
|
error_report("assert_page_lock: PageDesc %p not locked @ %s:%d",
|
||||||
@ -565,8 +523,6 @@ do_assert_page_locked(const PageDesc *pd, const char *file, int line)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#define assert_page_locked(pd) do_assert_page_locked(pd, __FILE__, __LINE__)
|
|
||||||
|
|
||||||
void assert_no_pages_locked(void)
|
void assert_no_pages_locked(void)
|
||||||
{
|
{
|
||||||
ht_pages_locked_debug_init();
|
ht_pages_locked_debug_init();
|
||||||
@ -575,50 +531,23 @@ void assert_no_pages_locked(void)
|
|||||||
|
|
||||||
#else /* !CONFIG_DEBUG_TCG */
|
#else /* !CONFIG_DEBUG_TCG */
|
||||||
|
|
||||||
#define assert_page_locked(pd)
|
static inline void page_lock__debug(const PageDesc *pd) { }
|
||||||
|
static inline void page_unlock__debug(const PageDesc *pd) { }
|
||||||
static inline void page_lock__debug(const PageDesc *pd)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline void page_unlock__debug(const PageDesc *pd)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
#endif /* CONFIG_DEBUG_TCG */
|
#endif /* CONFIG_DEBUG_TCG */
|
||||||
|
|
||||||
static inline void page_lock(PageDesc *pd)
|
void page_lock(PageDesc *pd)
|
||||||
{
|
{
|
||||||
page_lock__debug(pd);
|
page_lock__debug(pd);
|
||||||
qemu_spin_lock(&pd->lock);
|
qemu_spin_lock(&pd->lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void page_unlock(PageDesc *pd)
|
void page_unlock(PageDesc *pd)
|
||||||
{
|
{
|
||||||
qemu_spin_unlock(&pd->lock);
|
qemu_spin_unlock(&pd->lock);
|
||||||
page_unlock__debug(pd);
|
page_unlock__debug(pd);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* lock the page(s) of a TB in the correct acquisition order */
|
|
||||||
static inline void page_lock_tb(const TranslationBlock *tb)
|
|
||||||
{
|
|
||||||
page_lock_pair(NULL, tb->page_addr[0], NULL, tb->page_addr[1], false);
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline void page_unlock_tb(const TranslationBlock *tb)
|
|
||||||
{
|
|
||||||
PageDesc *p1 = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
|
|
||||||
|
|
||||||
page_unlock(p1);
|
|
||||||
if (unlikely(tb->page_addr[1] != -1)) {
|
|
||||||
PageDesc *p2 = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
|
|
||||||
|
|
||||||
if (p2 != p1) {
|
|
||||||
page_unlock(p2);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline struct page_entry *
|
static inline struct page_entry *
|
||||||
page_entry_new(PageDesc *pd, tb_page_addr_t index)
|
page_entry_new(PageDesc *pd, tb_page_addr_t index)
|
||||||
{
|
{
|
||||||
@ -790,434 +719,6 @@ void page_collection_unlock(struct page_collection *set)
|
|||||||
|
|
||||||
#endif /* !CONFIG_USER_ONLY */
|
#endif /* !CONFIG_USER_ONLY */
|
||||||
|
|
||||||
static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
|
|
||||||
PageDesc **ret_p2, tb_page_addr_t phys2, bool alloc)
|
|
||||||
{
|
|
||||||
PageDesc *p1, *p2;
|
|
||||||
tb_page_addr_t page1;
|
|
||||||
tb_page_addr_t page2;
|
|
||||||
|
|
||||||
assert_memory_lock();
|
|
||||||
g_assert(phys1 != -1);
|
|
||||||
|
|
||||||
page1 = phys1 >> TARGET_PAGE_BITS;
|
|
||||||
page2 = phys2 >> TARGET_PAGE_BITS;
|
|
||||||
|
|
||||||
p1 = page_find_alloc(page1, alloc);
|
|
||||||
if (ret_p1) {
|
|
||||||
*ret_p1 = p1;
|
|
||||||
}
|
|
||||||
if (likely(phys2 == -1)) {
|
|
||||||
page_lock(p1);
|
|
||||||
return;
|
|
||||||
} else if (page1 == page2) {
|
|
||||||
page_lock(p1);
|
|
||||||
if (ret_p2) {
|
|
||||||
*ret_p2 = p1;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
p2 = page_find_alloc(page2, alloc);
|
|
||||||
if (ret_p2) {
|
|
||||||
*ret_p2 = p2;
|
|
||||||
}
|
|
||||||
if (page1 < page2) {
|
|
||||||
page_lock(p1);
|
|
||||||
page_lock(p2);
|
|
||||||
} else {
|
|
||||||
page_lock(p2);
|
|
||||||
page_lock(p1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool tb_cmp(const void *ap, const void *bp)
|
|
||||||
{
|
|
||||||
const TranslationBlock *a = ap;
|
|
||||||
const TranslationBlock *b = bp;
|
|
||||||
|
|
||||||
return ((TARGET_TB_PCREL || tb_pc(a) == tb_pc(b)) &&
|
|
||||||
a->cs_base == b->cs_base &&
|
|
||||||
a->flags == b->flags &&
|
|
||||||
(tb_cflags(a) & ~CF_INVALID) == (tb_cflags(b) & ~CF_INVALID) &&
|
|
||||||
a->trace_vcpu_dstate == b->trace_vcpu_dstate &&
|
|
||||||
a->page_addr[0] == b->page_addr[0] &&
|
|
||||||
a->page_addr[1] == b->page_addr[1]);
|
|
||||||
}
|
|
||||||
|
|
||||||
void tb_htable_init(void)
|
|
||||||
{
|
|
||||||
unsigned int mode = QHT_MODE_AUTO_RESIZE;
|
|
||||||
|
|
||||||
qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Set to NULL all the 'first_tb' fields in all PageDescs. */
|
|
||||||
static void page_flush_tb_1(int level, void **lp)
|
|
||||||
{
|
|
||||||
int i;
|
|
||||||
|
|
||||||
if (*lp == NULL) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (level == 0) {
|
|
||||||
PageDesc *pd = *lp;
|
|
||||||
|
|
||||||
for (i = 0; i < V_L2_SIZE; ++i) {
|
|
||||||
page_lock(&pd[i]);
|
|
||||||
pd[i].first_tb = (uintptr_t)NULL;
|
|
||||||
page_unlock(&pd[i]);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
void **pp = *lp;
|
|
||||||
|
|
||||||
for (i = 0; i < V_L2_SIZE; ++i) {
|
|
||||||
page_flush_tb_1(level - 1, pp + i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void page_flush_tb(void)
|
|
||||||
{
|
|
||||||
int i, l1_sz = v_l1_size;
|
|
||||||
|
|
||||||
for (i = 0; i < l1_sz; i++) {
|
|
||||||
page_flush_tb_1(v_l2_levels, l1_map + i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* flush all the translation blocks */
|
|
||||||
static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
|
|
||||||
{
|
|
||||||
bool did_flush = false;
|
|
||||||
|
|
||||||
mmap_lock();
|
|
||||||
/* If it is already been done on request of another CPU,
|
|
||||||
* just retry.
|
|
||||||
*/
|
|
||||||
if (tb_ctx.tb_flush_count != tb_flush_count.host_int) {
|
|
||||||
goto done;
|
|
||||||
}
|
|
||||||
did_flush = true;
|
|
||||||
|
|
||||||
CPU_FOREACH(cpu) {
|
|
||||||
tcg_flush_jmp_cache(cpu);
|
|
||||||
}
|
|
||||||
|
|
||||||
qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE);
|
|
||||||
page_flush_tb();
|
|
||||||
|
|
||||||
tcg_region_reset_all();
|
|
||||||
/* XXX: flush processor icache at this point if cache flush is
|
|
||||||
expensive */
|
|
||||||
qatomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1);
|
|
||||||
|
|
||||||
done:
|
|
||||||
mmap_unlock();
|
|
||||||
if (did_flush) {
|
|
||||||
qemu_plugin_flush_cb();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void tb_flush(CPUState *cpu)
|
|
||||||
{
|
|
||||||
if (tcg_enabled()) {
|
|
||||||
unsigned tb_flush_count = qatomic_mb_read(&tb_ctx.tb_flush_count);
|
|
||||||
|
|
||||||
if (cpu_in_exclusive_context(cpu)) {
|
|
||||||
do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count));
|
|
||||||
} else {
|
|
||||||
async_safe_run_on_cpu(cpu, do_tb_flush,
|
|
||||||
RUN_ON_CPU_HOST_INT(tb_flush_count));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* user-mode: call with mmap_lock held
|
|
||||||
* !user-mode: call with @pd->lock held
|
|
||||||
*/
|
|
||||||
static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb)
|
|
||||||
{
|
|
||||||
TranslationBlock *tb1;
|
|
||||||
uintptr_t *pprev;
|
|
||||||
unsigned int n1;
|
|
||||||
|
|
||||||
assert_page_locked(pd);
|
|
||||||
pprev = &pd->first_tb;
|
|
||||||
PAGE_FOR_EACH_TB(pd, tb1, n1) {
|
|
||||||
if (tb1 == tb) {
|
|
||||||
*pprev = tb1->page_next[n1];
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
pprev = &tb1->page_next[n1];
|
|
||||||
}
|
|
||||||
g_assert_not_reached();
|
|
||||||
}
|
|
||||||
|
|
||||||
/* remove @orig from its @n_orig-th jump list */
|
|
||||||
static inline void tb_remove_from_jmp_list(TranslationBlock *orig, int n_orig)
|
|
||||||
{
|
|
||||||
uintptr_t ptr, ptr_locked;
|
|
||||||
TranslationBlock *dest;
|
|
||||||
TranslationBlock *tb;
|
|
||||||
uintptr_t *pprev;
|
|
||||||
int n;
|
|
||||||
|
|
||||||
/* mark the LSB of jmp_dest[] so that no further jumps can be inserted */
|
|
||||||
ptr = qatomic_or_fetch(&orig->jmp_dest[n_orig], 1);
|
|
||||||
dest = (TranslationBlock *)(ptr & ~1);
|
|
||||||
if (dest == NULL) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
qemu_spin_lock(&dest->jmp_lock);
|
|
||||||
/*
|
|
||||||
* While acquiring the lock, the jump might have been removed if the
|
|
||||||
* destination TB was invalidated; check again.
|
|
||||||
*/
|
|
||||||
ptr_locked = qatomic_read(&orig->jmp_dest[n_orig]);
|
|
||||||
if (ptr_locked != ptr) {
|
|
||||||
qemu_spin_unlock(&dest->jmp_lock);
|
|
||||||
/*
|
|
||||||
* The only possibility is that the jump was unlinked via
|
|
||||||
* tb_jump_unlink(dest). Seeing here another destination would be a bug,
|
|
||||||
* because we set the LSB above.
|
|
||||||
*/
|
|
||||||
g_assert(ptr_locked == 1 && dest->cflags & CF_INVALID);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
/*
|
|
||||||
* We first acquired the lock, and since the destination pointer matches,
|
|
||||||
* we know for sure that @orig is in the jmp list.
|
|
||||||
*/
|
|
||||||
pprev = &dest->jmp_list_head;
|
|
||||||
TB_FOR_EACH_JMP(dest, tb, n) {
|
|
||||||
if (tb == orig && n == n_orig) {
|
|
||||||
*pprev = tb->jmp_list_next[n];
|
|
||||||
/* no need to set orig->jmp_dest[n]; setting the LSB was enough */
|
|
||||||
qemu_spin_unlock(&dest->jmp_lock);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
pprev = &tb->jmp_list_next[n];
|
|
||||||
}
|
|
||||||
g_assert_not_reached();
|
|
||||||
}
|
|
||||||
|
|
||||||
/* reset the jump entry 'n' of a TB so that it is not chained to
|
|
||||||
another TB */
|
|
||||||
static inline void tb_reset_jump(TranslationBlock *tb, int n)
|
|
||||||
{
|
|
||||||
uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]);
|
|
||||||
tb_set_jmp_target(tb, n, addr);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* remove any jumps to the TB */
|
|
||||||
static inline void tb_jmp_unlink(TranslationBlock *dest)
|
|
||||||
{
|
|
||||||
TranslationBlock *tb;
|
|
||||||
int n;
|
|
||||||
|
|
||||||
qemu_spin_lock(&dest->jmp_lock);
|
|
||||||
|
|
||||||
TB_FOR_EACH_JMP(dest, tb, n) {
|
|
||||||
tb_reset_jump(tb, n);
|
|
||||||
qatomic_and(&tb->jmp_dest[n], (uintptr_t)NULL | 1);
|
|
||||||
/* No need to clear the list entry; setting the dest ptr is enough */
|
|
||||||
}
|
|
||||||
dest->jmp_list_head = (uintptr_t)NULL;
|
|
||||||
|
|
||||||
qemu_spin_unlock(&dest->jmp_lock);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void tb_jmp_cache_inval_tb(TranslationBlock *tb)
|
|
||||||
{
|
|
||||||
CPUState *cpu;
|
|
||||||
|
|
||||||
if (TARGET_TB_PCREL) {
|
|
||||||
/* A TB may be at any virtual address */
|
|
||||||
CPU_FOREACH(cpu) {
|
|
||||||
tcg_flush_jmp_cache(cpu);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
uint32_t h = tb_jmp_cache_hash_func(tb_pc(tb));
|
|
||||||
|
|
||||||
CPU_FOREACH(cpu) {
|
|
||||||
CPUJumpCache *jc = cpu->tb_jmp_cache;
|
|
||||||
|
|
||||||
if (qatomic_read(&jc->array[h].tb) == tb) {
|
|
||||||
qatomic_set(&jc->array[h].tb, NULL);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* In user-mode, call with mmap_lock held.
|
|
||||||
* In !user-mode, if @rm_from_page_list is set, call with the TB's pages'
|
|
||||||
* locks held.
|
|
||||||
*/
|
|
||||||
static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list)
|
|
||||||
{
|
|
||||||
PageDesc *p;
|
|
||||||
uint32_t h;
|
|
||||||
tb_page_addr_t phys_pc;
|
|
||||||
uint32_t orig_cflags = tb_cflags(tb);
|
|
||||||
|
|
||||||
assert_memory_lock();
|
|
||||||
|
|
||||||
/* make sure no further incoming jumps will be chained to this TB */
|
|
||||||
qemu_spin_lock(&tb->jmp_lock);
|
|
||||||
qatomic_set(&tb->cflags, tb->cflags | CF_INVALID);
|
|
||||||
qemu_spin_unlock(&tb->jmp_lock);
|
|
||||||
|
|
||||||
/* remove the TB from the hash list */
|
|
||||||
phys_pc = tb->page_addr[0];
|
|
||||||
h = tb_hash_func(phys_pc, (TARGET_TB_PCREL ? 0 : tb_pc(tb)),
|
|
||||||
tb->flags, orig_cflags, tb->trace_vcpu_dstate);
|
|
||||||
if (!qht_remove(&tb_ctx.htable, tb, h)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* remove the TB from the page list */
|
|
||||||
if (rm_from_page_list) {
|
|
||||||
p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
|
|
||||||
tb_page_remove(p, tb);
|
|
||||||
if (tb->page_addr[1] != -1) {
|
|
||||||
p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
|
|
||||||
tb_page_remove(p, tb);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* remove the TB from the hash list */
|
|
||||||
tb_jmp_cache_inval_tb(tb);
|
|
||||||
|
|
||||||
/* suppress this TB from the two jump lists */
|
|
||||||
tb_remove_from_jmp_list(tb, 0);
|
|
||||||
tb_remove_from_jmp_list(tb, 1);
|
|
||||||
|
|
||||||
/* suppress any remaining jumps to this TB */
|
|
||||||
tb_jmp_unlink(tb);
|
|
||||||
|
|
||||||
qatomic_set(&tb_ctx.tb_phys_invalidate_count,
|
|
||||||
tb_ctx.tb_phys_invalidate_count + 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void tb_phys_invalidate__locked(TranslationBlock *tb)
|
|
||||||
{
|
|
||||||
qemu_thread_jit_write();
|
|
||||||
do_tb_phys_invalidate(tb, true);
|
|
||||||
qemu_thread_jit_execute();
|
|
||||||
}
|
|
||||||
|
|
||||||
/* invalidate one TB
|
|
||||||
*
|
|
||||||
* Called with mmap_lock held in user-mode.
|
|
||||||
*/
|
|
||||||
void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
|
|
||||||
{
|
|
||||||
if (page_addr == -1 && tb->page_addr[0] != -1) {
|
|
||||||
page_lock_tb(tb);
|
|
||||||
do_tb_phys_invalidate(tb, true);
|
|
||||||
page_unlock_tb(tb);
|
|
||||||
} else {
|
|
||||||
do_tb_phys_invalidate(tb, false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* add the tb in the target page and protect it if necessary
|
|
||||||
*
|
|
||||||
* Called with mmap_lock held for user-mode emulation.
|
|
||||||
* Called with @p->lock held in !user-mode.
|
|
||||||
*/
|
|
||||||
static inline void tb_page_add(PageDesc *p, TranslationBlock *tb,
|
|
||||||
unsigned int n, tb_page_addr_t page_addr)
|
|
||||||
{
|
|
||||||
#ifndef CONFIG_USER_ONLY
|
|
||||||
bool page_already_protected;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
assert_page_locked(p);
|
|
||||||
|
|
||||||
tb->page_addr[n] = page_addr;
|
|
||||||
tb->page_next[n] = p->first_tb;
|
|
||||||
#ifndef CONFIG_USER_ONLY
|
|
||||||
page_already_protected = p->first_tb != (uintptr_t)NULL;
|
|
||||||
#endif
|
|
||||||
p->first_tb = (uintptr_t)tb | n;
|
|
||||||
|
|
||||||
#if defined(CONFIG_USER_ONLY)
|
|
||||||
/* translator_loop() must have made all TB pages non-writable */
|
|
||||||
assert(!(p->flags & PAGE_WRITE));
|
|
||||||
#else
|
|
||||||
/* if some code is already present, then the pages are already
|
|
||||||
protected. So we handle the case where only the first TB is
|
|
||||||
allocated in a physical page */
|
|
||||||
if (!page_already_protected) {
|
|
||||||
tlb_protect_code(page_addr);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Add a new TB and link it to the physical page tables. phys_page2 is
|
|
||||||
* (-1) to indicate that only one page contains the TB.
|
|
||||||
*
|
|
||||||
* Called with mmap_lock held for user-mode emulation.
|
|
||||||
*
|
|
||||||
* Returns a pointer @tb, or a pointer to an existing TB that matches @tb.
|
|
||||||
* Note that in !user-mode, another thread might have already added a TB
|
|
||||||
* for the same block of guest code that @tb corresponds to. In that case,
|
|
||||||
* the caller should discard the original @tb, and use instead the returned TB.
|
|
||||||
*/
|
|
||||||
static TranslationBlock *
|
|
||||||
tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
|
|
||||||
tb_page_addr_t phys_page2)
|
|
||||||
{
|
|
||||||
PageDesc *p;
|
|
||||||
PageDesc *p2 = NULL;
|
|
||||||
void *existing_tb = NULL;
|
|
||||||
uint32_t h;
|
|
||||||
|
|
||||||
assert_memory_lock();
|
|
||||||
tcg_debug_assert(!(tb->cflags & CF_INVALID));
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Add the TB to the page list, acquiring first the pages's locks.
|
|
||||||
* We keep the locks held until after inserting the TB in the hash table,
|
|
||||||
* so that if the insertion fails we know for sure that the TBs are still
|
|
||||||
* in the page descriptors.
|
|
||||||
* Note that inserting into the hash table first isn't an option, since
|
|
||||||
* we can only insert TBs that are fully initialized.
|
|
||||||
*/
|
|
||||||
page_lock_pair(&p, phys_pc, &p2, phys_page2, true);
|
|
||||||
tb_page_add(p, tb, 0, phys_pc);
|
|
||||||
if (p2) {
|
|
||||||
tb_page_add(p2, tb, 1, phys_page2);
|
|
||||||
} else {
|
|
||||||
tb->page_addr[1] = -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* add in the hash table */
|
|
||||||
h = tb_hash_func(phys_pc, (TARGET_TB_PCREL ? 0 : tb_pc(tb)),
|
|
||||||
tb->flags, tb->cflags, tb->trace_vcpu_dstate);
|
|
||||||
qht_insert(&tb_ctx.htable, tb, h, &existing_tb);
|
|
||||||
|
|
||||||
/* remove TB from the page(s) if we couldn't insert it */
|
|
||||||
if (unlikely(existing_tb)) {
|
|
||||||
tb_page_remove(p, tb);
|
|
||||||
if (p2) {
|
|
||||||
tb_page_remove(p2, tb);
|
|
||||||
}
|
|
||||||
tb = existing_tb;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (p2 && p2 != p) {
|
|
||||||
page_unlock(p2);
|
|
||||||
}
|
|
||||||
page_unlock(p);
|
|
||||||
return tb;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Called with mmap_lock held for user mode emulation. */
|
/* Called with mmap_lock held for user mode emulation. */
|
||||||
TranslationBlock *tb_gen_code(CPUState *cpu,
|
TranslationBlock *tb_gen_code(CPUState *cpu,
|
||||||
target_ulong pc, target_ulong cs_base,
|
target_ulong pc, target_ulong cs_base,
|
||||||
@ -1497,251 +998,6 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
|
|||||||
return tb;
|
return tb;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* @p must be non-NULL.
|
|
||||||
* user-mode: call with mmap_lock held.
|
|
||||||
* !user-mode: call with all @pages locked.
|
|
||||||
*/
|
|
||||||
static void
|
|
||||||
tb_invalidate_phys_page_range__locked(struct page_collection *pages,
|
|
||||||
PageDesc *p, tb_page_addr_t start,
|
|
||||||
tb_page_addr_t end,
|
|
||||||
uintptr_t retaddr)
|
|
||||||
{
|
|
||||||
TranslationBlock *tb;
|
|
||||||
tb_page_addr_t tb_start, tb_end;
|
|
||||||
int n;
|
|
||||||
#ifdef TARGET_HAS_PRECISE_SMC
|
|
||||||
CPUState *cpu = current_cpu;
|
|
||||||
CPUArchState *env = NULL;
|
|
||||||
bool current_tb_not_found = retaddr != 0;
|
|
||||||
bool current_tb_modified = false;
|
|
||||||
TranslationBlock *current_tb = NULL;
|
|
||||||
target_ulong current_pc = 0;
|
|
||||||
target_ulong current_cs_base = 0;
|
|
||||||
uint32_t current_flags = 0;
|
|
||||||
#endif /* TARGET_HAS_PRECISE_SMC */
|
|
||||||
|
|
||||||
assert_page_locked(p);
|
|
||||||
|
|
||||||
#if defined(TARGET_HAS_PRECISE_SMC)
|
|
||||||
if (cpu != NULL) {
|
|
||||||
env = cpu->env_ptr;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/* we remove all the TBs in the range [start, end[ */
|
|
||||||
/* XXX: see if in some cases it could be faster to invalidate all
|
|
||||||
the code */
|
|
||||||
PAGE_FOR_EACH_TB(p, tb, n) {
|
|
||||||
assert_page_locked(p);
|
|
||||||
/* NOTE: this is subtle as a TB may span two physical pages */
|
|
||||||
if (n == 0) {
|
|
||||||
/* NOTE: tb_end may be after the end of the page, but
|
|
||||||
it is not a problem */
|
|
||||||
tb_start = tb->page_addr[0];
|
|
||||||
tb_end = tb_start + tb->size;
|
|
||||||
} else {
|
|
||||||
tb_start = tb->page_addr[1];
|
|
||||||
tb_end = tb_start + ((tb->page_addr[0] + tb->size)
|
|
||||||
& ~TARGET_PAGE_MASK);
|
|
||||||
}
|
|
||||||
if (!(tb_end <= start || tb_start >= end)) {
|
|
||||||
#ifdef TARGET_HAS_PRECISE_SMC
|
|
||||||
if (current_tb_not_found) {
|
|
||||||
current_tb_not_found = false;
|
|
||||||
/* now we have a real cpu fault */
|
|
||||||
current_tb = tcg_tb_lookup(retaddr);
|
|
||||||
}
|
|
||||||
if (current_tb == tb &&
|
|
||||||
(tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
|
|
||||||
/*
|
|
||||||
* If we are modifying the current TB, we must stop
|
|
||||||
* its execution. We could be more precise by checking
|
|
||||||
* that the modification is after the current PC, but it
|
|
||||||
* would require a specialized function to partially
|
|
||||||
* restore the CPU state.
|
|
||||||
*/
|
|
||||||
current_tb_modified = true;
|
|
||||||
cpu_restore_state_from_tb(cpu, current_tb, retaddr, true);
|
|
||||||
cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base,
|
|
||||||
¤t_flags);
|
|
||||||
}
|
|
||||||
#endif /* TARGET_HAS_PRECISE_SMC */
|
|
||||||
tb_phys_invalidate__locked(tb);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#if !defined(CONFIG_USER_ONLY)
|
|
||||||
/* if no code remaining, no need to continue to use slow writes */
|
|
||||||
if (!p->first_tb) {
|
|
||||||
tlb_unprotect_code(start);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
#ifdef TARGET_HAS_PRECISE_SMC
|
|
||||||
if (current_tb_modified) {
|
|
||||||
page_collection_unlock(pages);
|
|
||||||
/* Force execution of one insn next time. */
|
|
||||||
cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu);
|
|
||||||
mmap_unlock();
|
|
||||||
cpu_loop_exit_noexc(cpu);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Invalidate all TBs which intersect with the target physical address range
|
|
||||||
* [start;end[. NOTE: start and end must refer to the *same* physical page.
|
|
||||||
* 'is_cpu_write_access' should be true if called from a real cpu write
|
|
||||||
* access: the virtual CPU will exit the current TB if code is modified inside
|
|
||||||
* this TB.
|
|
||||||
*
|
|
||||||
* Called with mmap_lock held for user-mode emulation
|
|
||||||
*/
|
|
||||||
void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end)
|
|
||||||
{
|
|
||||||
struct page_collection *pages;
|
|
||||||
PageDesc *p;
|
|
||||||
|
|
||||||
assert_memory_lock();
|
|
||||||
|
|
||||||
p = page_find(start >> TARGET_PAGE_BITS);
|
|
||||||
if (p == NULL) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
pages = page_collection_lock(start, end);
|
|
||||||
tb_invalidate_phys_page_range__locked(pages, p, start, end, 0);
|
|
||||||
page_collection_unlock(pages);
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Invalidate all TBs which intersect with the target physical address range
|
|
||||||
* [start;end[. NOTE: start and end may refer to *different* physical pages.
|
|
||||||
* 'is_cpu_write_access' should be true if called from a real cpu write
|
|
||||||
* access: the virtual CPU will exit the current TB if code is modified inside
|
|
||||||
* this TB.
|
|
||||||
*
|
|
||||||
* Called with mmap_lock held for user-mode emulation.
|
|
||||||
*/
|
|
||||||
#ifdef CONFIG_SOFTMMU
|
|
||||||
void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end)
|
|
||||||
#else
|
|
||||||
void tb_invalidate_phys_range(target_ulong start, target_ulong end)
|
|
||||||
#endif
|
|
||||||
{
|
|
||||||
struct page_collection *pages;
|
|
||||||
tb_page_addr_t next;
|
|
||||||
|
|
||||||
assert_memory_lock();
|
|
||||||
|
|
||||||
pages = page_collection_lock(start, end);
|
|
||||||
for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
|
|
||||||
start < end;
|
|
||||||
start = next, next += TARGET_PAGE_SIZE) {
|
|
||||||
PageDesc *pd = page_find(start >> TARGET_PAGE_BITS);
|
|
||||||
tb_page_addr_t bound = MIN(next, end);
|
|
||||||
|
|
||||||
if (pd == NULL) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0);
|
|
||||||
}
|
|
||||||
page_collection_unlock(pages);
|
|
||||||
}
|
|
||||||
|
|
||||||
#ifdef CONFIG_SOFTMMU
|
|
||||||
/* len must be <= 8 and start must be a multiple of len.
|
|
||||||
* Called via softmmu_template.h when code areas are written to with
|
|
||||||
* iothread mutex not held.
|
|
||||||
*
|
|
||||||
* Call with all @pages in the range [@start, @start + len[ locked.
|
|
||||||
*/
|
|
||||||
void tb_invalidate_phys_page_fast(struct page_collection *pages,
|
|
||||||
tb_page_addr_t start, int len,
|
|
||||||
uintptr_t retaddr)
|
|
||||||
{
|
|
||||||
PageDesc *p;
|
|
||||||
|
|
||||||
assert_memory_lock();
|
|
||||||
|
|
||||||
p = page_find(start >> TARGET_PAGE_BITS);
|
|
||||||
if (!p) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
assert_page_locked(p);
|
|
||||||
tb_invalidate_phys_page_range__locked(pages, p, start, start + len,
|
|
||||||
retaddr);
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
/* Called with mmap_lock held. If pc is not 0 then it indicates the
|
|
||||||
* host PC of the faulting store instruction that caused this invalidate.
|
|
||||||
* Returns true if the caller needs to abort execution of the current
|
|
||||||
* TB (because it was modified by this store and the guest CPU has
|
|
||||||
* precise-SMC semantics).
|
|
||||||
*/
|
|
||||||
static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)
|
|
||||||
{
|
|
||||||
TranslationBlock *tb;
|
|
||||||
PageDesc *p;
|
|
||||||
int n;
|
|
||||||
#ifdef TARGET_HAS_PRECISE_SMC
|
|
||||||
TranslationBlock *current_tb = NULL;
|
|
||||||
CPUState *cpu = current_cpu;
|
|
||||||
CPUArchState *env = NULL;
|
|
||||||
int current_tb_modified = 0;
|
|
||||||
target_ulong current_pc = 0;
|
|
||||||
target_ulong current_cs_base = 0;
|
|
||||||
uint32_t current_flags = 0;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
assert_memory_lock();
|
|
||||||
|
|
||||||
addr &= TARGET_PAGE_MASK;
|
|
||||||
p = page_find(addr >> TARGET_PAGE_BITS);
|
|
||||||
if (!p) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ifdef TARGET_HAS_PRECISE_SMC
|
|
||||||
if (p->first_tb && pc != 0) {
|
|
||||||
current_tb = tcg_tb_lookup(pc);
|
|
||||||
}
|
|
||||||
if (cpu != NULL) {
|
|
||||||
env = cpu->env_ptr;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
assert_page_locked(p);
|
|
||||||
PAGE_FOR_EACH_TB(p, tb, n) {
|
|
||||||
#ifdef TARGET_HAS_PRECISE_SMC
|
|
||||||
if (current_tb == tb &&
|
|
||||||
(tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
|
|
||||||
/* If we are modifying the current TB, we must stop
|
|
||||||
its execution. We could be more precise by checking
|
|
||||||
that the modification is after the current PC, but it
|
|
||||||
would require a specialized function to partially
|
|
||||||
restore the CPU state */
|
|
||||||
|
|
||||||
current_tb_modified = 1;
|
|
||||||
cpu_restore_state_from_tb(cpu, current_tb, pc, true);
|
|
||||||
cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base,
|
|
||||||
¤t_flags);
|
|
||||||
}
|
|
||||||
#endif /* TARGET_HAS_PRECISE_SMC */
|
|
||||||
tb_phys_invalidate(tb, addr);
|
|
||||||
}
|
|
||||||
p->first_tb = (uintptr_t)NULL;
|
|
||||||
#ifdef TARGET_HAS_PRECISE_SMC
|
|
||||||
if (current_tb_modified) {
|
|
||||||
/* Force execution of one insn next time. */
|
|
||||||
cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/* user-mode: call with mmap_lock held */
|
/* user-mode: call with mmap_lock held */
|
||||||
void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr)
|
void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr)
|
||||||
{
|
{
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user