/* * Translation Block Maintaince * * Copyright (c) 2003 Fabrice Bellard * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, see . */ #include "qemu/osdep.h" #include "qemu/interval-tree.h" #include "exec/cputlb.h" #include "exec/log.h" #include "exec/exec-all.h" #include "exec/translate-all.h" #include "sysemu/tcg.h" #include "tcg/tcg.h" #include "tb-hash.h" #include "tb-context.h" #include "internal.h" /* List iterators for lists of tagged pointers in TranslationBlock. */ #define TB_FOR_EACH_TAGGED(head, tb, n, field) \ for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \ tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \ tb = (TranslationBlock *)((uintptr_t)tb & ~1)) #define TB_FOR_EACH_JMP(head_tb, tb, n) \ TB_FOR_EACH_TAGGED((head_tb)->jmp_list_head, tb, n, jmp_list_next) static bool tb_cmp(const void *ap, const void *bp) { const TranslationBlock *a = ap; const TranslationBlock *b = bp; return ((tb_cflags(a) & CF_PCREL || a->pc == b->pc) && a->cs_base == b->cs_base && a->flags == b->flags && (tb_cflags(a) & ~CF_INVALID) == (tb_cflags(b) & ~CF_INVALID) && a->trace_vcpu_dstate == b->trace_vcpu_dstate && tb_page_addr0(a) == tb_page_addr0(b) && tb_page_addr1(a) == tb_page_addr1(b)); } void tb_htable_init(void) { unsigned int mode = QHT_MODE_AUTO_RESIZE; qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode); } typedef struct PageDesc PageDesc; #ifdef CONFIG_USER_ONLY /* * In user-mode page locks aren't used; mmap_lock is enough. */ #define assert_page_locked(pd) tcg_debug_assert(have_mmap_lock()) static inline void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1, PageDesc **ret_p2, tb_page_addr_t phys2, bool alloc) { *ret_p1 = NULL; *ret_p2 = NULL; } static inline void page_unlock(PageDesc *pd) { } static inline void page_lock_tb(const TranslationBlock *tb) { } static inline void page_unlock_tb(const TranslationBlock *tb) { } /* * For user-only, since we are protecting all of memory with a single lock, * and because the two pages of a TranslationBlock are always contiguous, * use a single data structure to record all TranslationBlocks. */ static IntervalTreeRoot tb_root; static void tb_remove_all(void) { assert_memory_lock(); memset(&tb_root, 0, sizeof(tb_root)); } /* Call with mmap_lock held. */ static void tb_record(TranslationBlock *tb, PageDesc *p1, PageDesc *p2) { target_ulong addr; int flags; assert_memory_lock(); tb->itree.last = tb->itree.start + tb->size - 1; /* translator_loop() must have made all TB pages non-writable */ addr = tb_page_addr0(tb); flags = page_get_flags(addr); assert(!(flags & PAGE_WRITE)); addr = tb_page_addr1(tb); if (addr != -1) { flags = page_get_flags(addr); assert(!(flags & PAGE_WRITE)); } interval_tree_insert(&tb->itree, &tb_root); } /* Call with mmap_lock held. */ static void tb_remove(TranslationBlock *tb) { assert_memory_lock(); interval_tree_remove(&tb->itree, &tb_root); } /* TODO: For now, still shared with translate-all.c for system mode. */ #define PAGE_FOR_EACH_TB(start, end, pagedesc, T, N) \ for (T = foreach_tb_first(start, end), \ N = foreach_tb_next(T, start, end); \ T != NULL; \ T = N, N = foreach_tb_next(N, start, end)) typedef TranslationBlock *PageForEachNext; static PageForEachNext foreach_tb_first(tb_page_addr_t start, tb_page_addr_t end) { IntervalTreeNode *n = interval_tree_iter_first(&tb_root, start, end - 1); return n ? container_of(n, TranslationBlock, itree) : NULL; } static PageForEachNext foreach_tb_next(PageForEachNext tb, tb_page_addr_t start, tb_page_addr_t end) { IntervalTreeNode *n; if (tb) { n = interval_tree_iter_next(&tb->itree, start, end - 1); if (n) { return container_of(n, TranslationBlock, itree); } } return NULL; } #else /* * In system mode we want L1_MAP to be based on ram offsets. */ #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS #else # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS #endif /* Size of the L2 (and L3, etc) page tables. */ #define V_L2_BITS 10 #define V_L2_SIZE (1 << V_L2_BITS) /* * L1 Mapping properties */ static int v_l1_size; static int v_l1_shift; static int v_l2_levels; /* * The bottom level has pointers to PageDesc, and is indexed by * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size. */ #define V_L1_MIN_BITS 4 #define V_L1_MAX_BITS (V_L2_BITS + 3) #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS) static void *l1_map[V_L1_MAX_SIZE]; struct PageDesc { QemuSpin lock; /* list of TBs intersecting this ram page */ uintptr_t first_tb; }; void page_table_config_init(void) { uint32_t v_l1_bits; assert(TARGET_PAGE_BITS); /* The bits remaining after N lower levels of page tables. */ v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS; if (v_l1_bits < V_L1_MIN_BITS) { v_l1_bits += V_L2_BITS; } v_l1_size = 1 << v_l1_bits; v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits; v_l2_levels = v_l1_shift / V_L2_BITS - 1; assert(v_l1_bits <= V_L1_MAX_BITS); assert(v_l1_shift % V_L2_BITS == 0); assert(v_l2_levels >= 0); } static PageDesc *page_find_alloc(tb_page_addr_t index, bool alloc) { PageDesc *pd; void **lp; int i; /* Level 1. Always allocated. */ lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1)); /* Level 2..N-1. */ for (i = v_l2_levels; i > 0; i--) { void **p = qatomic_rcu_read(lp); if (p == NULL) { void *existing; if (!alloc) { return NULL; } p = g_new0(void *, V_L2_SIZE); existing = qatomic_cmpxchg(lp, NULL, p); if (unlikely(existing)) { g_free(p); p = existing; } } lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1)); } pd = qatomic_rcu_read(lp); if (pd == NULL) { void *existing; if (!alloc) { return NULL; } pd = g_new0(PageDesc, V_L2_SIZE); for (int i = 0; i < V_L2_SIZE; i++) { qemu_spin_init(&pd[i].lock); } existing = qatomic_cmpxchg(lp, NULL, pd); if (unlikely(existing)) { for (int i = 0; i < V_L2_SIZE; i++) { qemu_spin_destroy(&pd[i].lock); } g_free(pd); pd = existing; } } return pd + (index & (V_L2_SIZE - 1)); } static inline PageDesc *page_find(tb_page_addr_t index) { return page_find_alloc(index, false); } /** * struct page_entry - page descriptor entry * @pd: pointer to the &struct PageDesc of the page this entry represents * @index: page index of the page * @locked: whether the page is locked * * This struct helps us keep track of the locked state of a page, without * bloating &struct PageDesc. * * A page lock protects accesses to all fields of &struct PageDesc. * * See also: &struct page_collection. */ struct page_entry { PageDesc *pd; tb_page_addr_t index; bool locked; }; /** * struct page_collection - tracks a set of pages (i.e. &struct page_entry's) * @tree: Binary search tree (BST) of the pages, with key == page index * @max: Pointer to the page in @tree with the highest page index * * To avoid deadlock we lock pages in ascending order of page index. * When operating on a set of pages, we need to keep track of them so that * we can lock them in order and also unlock them later. For this we collect * pages (i.e. &struct page_entry's) in a binary search @tree. Given that the * @tree implementation we use does not provide an O(1) operation to obtain the * highest-ranked element, we use @max to keep track of the inserted page * with the highest index. This is valuable because if a page is not in * the tree and its index is higher than @max's, then we can lock it * without breaking the locking order rule. * * Note on naming: 'struct page_set' would be shorter, but we already have a few * page_set_*() helpers, so page_collection is used instead to avoid confusion. * * See also: page_collection_lock(). */ struct page_collection { GTree *tree; struct page_entry *max; }; typedef int PageForEachNext; #define PAGE_FOR_EACH_TB(start, end, pagedesc, tb, n) \ TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next) #ifdef CONFIG_DEBUG_TCG static __thread GHashTable *ht_pages_locked_debug; static void ht_pages_locked_debug_init(void) { if (ht_pages_locked_debug) { return; } ht_pages_locked_debug = g_hash_table_new(NULL, NULL); } static bool page_is_locked(const PageDesc *pd) { PageDesc *found; ht_pages_locked_debug_init(); found = g_hash_table_lookup(ht_pages_locked_debug, pd); return !!found; } static void page_lock__debug(PageDesc *pd) { ht_pages_locked_debug_init(); g_assert(!page_is_locked(pd)); g_hash_table_insert(ht_pages_locked_debug, pd, pd); } static void page_unlock__debug(const PageDesc *pd) { bool removed; ht_pages_locked_debug_init(); g_assert(page_is_locked(pd)); removed = g_hash_table_remove(ht_pages_locked_debug, pd); g_assert(removed); } static void do_assert_page_locked(const PageDesc *pd, const char *file, int line) { if (unlikely(!page_is_locked(pd))) { error_report("assert_page_lock: PageDesc %p not locked @ %s:%d", pd, file, line); abort(); } } #define assert_page_locked(pd) do_assert_page_locked(pd, __FILE__, __LINE__) void assert_no_pages_locked(void) { ht_pages_locked_debug_init(); g_assert(g_hash_table_size(ht_pages_locked_debug) == 0); } #else /* !CONFIG_DEBUG_TCG */ static inline void page_lock__debug(const PageDesc *pd) { } static inline void page_unlock__debug(const PageDesc *pd) { } static inline void assert_page_locked(const PageDesc *pd) { } #endif /* CONFIG_DEBUG_TCG */ static void page_lock(PageDesc *pd) { page_lock__debug(pd); qemu_spin_lock(&pd->lock); } static void page_unlock(PageDesc *pd) { qemu_spin_unlock(&pd->lock); page_unlock__debug(pd); } static inline struct page_entry * page_entry_new(PageDesc *pd, tb_page_addr_t index) { struct page_entry *pe = g_malloc(sizeof(*pe)); pe->index = index; pe->pd = pd; pe->locked = false; return pe; } static void page_entry_destroy(gpointer p) { struct page_entry *pe = p; g_assert(pe->locked); page_unlock(pe->pd); g_free(pe); } /* returns false on success */ static bool page_entry_trylock(struct page_entry *pe) { bool busy; busy = qemu_spin_trylock(&pe->pd->lock); if (!busy) { g_assert(!pe->locked); pe->locked = true; page_lock__debug(pe->pd); } return busy; } static void do_page_entry_lock(struct page_entry *pe) { page_lock(pe->pd); g_assert(!pe->locked); pe->locked = true; } static gboolean page_entry_lock(gpointer key, gpointer value, gpointer data) { struct page_entry *pe = value; do_page_entry_lock(pe); return FALSE; } static gboolean page_entry_unlock(gpointer key, gpointer value, gpointer data) { struct page_entry *pe = value; if (pe->locked) { pe->locked = false; page_unlock(pe->pd); } return FALSE; } /* * Trylock a page, and if successful, add the page to a collection. * Returns true ("busy") if the page could not be locked; false otherwise. */ static bool page_trylock_add(struct page_collection *set, tb_page_addr_t addr) { tb_page_addr_t index = addr >> TARGET_PAGE_BITS; struct page_entry *pe; PageDesc *pd; pe = g_tree_lookup(set->tree, &index); if (pe) { return false; } pd = page_find(index); if (pd == NULL) { return false; } pe = page_entry_new(pd, index); g_tree_insert(set->tree, &pe->index, pe); /* * If this is either (1) the first insertion or (2) a page whose index * is higher than any other so far, just lock the page and move on. */ if (set->max == NULL || pe->index > set->max->index) { set->max = pe; do_page_entry_lock(pe); return false; } /* * Try to acquire out-of-order lock; if busy, return busy so that we acquire * locks in order. */ return page_entry_trylock(pe); } static gint tb_page_addr_cmp(gconstpointer ap, gconstpointer bp, gpointer udata) { tb_page_addr_t a = *(const tb_page_addr_t *)ap; tb_page_addr_t b = *(const tb_page_addr_t *)bp; if (a == b) { return 0; } else if (a < b) { return -1; } return 1; } /* * Lock a range of pages ([@start,@end[) as well as the pages of all * intersecting TBs. * Locking order: acquire locks in ascending order of page index. */ static struct page_collection *page_collection_lock(tb_page_addr_t start, tb_page_addr_t end) { struct page_collection *set = g_malloc(sizeof(*set)); tb_page_addr_t index; PageDesc *pd; start >>= TARGET_PAGE_BITS; end >>= TARGET_PAGE_BITS; g_assert(start <= end); set->tree = g_tree_new_full(tb_page_addr_cmp, NULL, NULL, page_entry_destroy); set->max = NULL; assert_no_pages_locked(); retry: g_tree_foreach(set->tree, page_entry_lock, NULL); for (index = start; index <= end; index++) { TranslationBlock *tb; PageForEachNext n; pd = page_find(index); if (pd == NULL) { continue; } if (page_trylock_add(set, index << TARGET_PAGE_BITS)) { g_tree_foreach(set->tree, page_entry_unlock, NULL); goto retry; } assert_page_locked(pd); PAGE_FOR_EACH_TB(unused, unused, pd, tb, n) { if (page_trylock_add(set, tb_page_addr0(tb)) || (tb_page_addr1(tb) != -1 && page_trylock_add(set, tb_page_addr1(tb)))) { /* drop all locks, and reacquire in order */ g_tree_foreach(set->tree, page_entry_unlock, NULL); goto retry; } } } return set; } static void page_collection_unlock(struct page_collection *set) { /* entries are unlocked and freed via page_entry_destroy */ g_tree_destroy(set->tree); g_free(set); } /* Set to NULL all the 'first_tb' fields in all PageDescs. */ static void tb_remove_all_1(int level, void **lp) { int i; if (*lp == NULL) { return; } if (level == 0) { PageDesc *pd = *lp; for (i = 0; i < V_L2_SIZE; ++i) { page_lock(&pd[i]); pd[i].first_tb = (uintptr_t)NULL; page_unlock(&pd[i]); } } else { void **pp = *lp; for (i = 0; i < V_L2_SIZE; ++i) { tb_remove_all_1(level - 1, pp + i); } } } static void tb_remove_all(void) { int i, l1_sz = v_l1_size; for (i = 0; i < l1_sz; i++) { tb_remove_all_1(v_l2_levels, l1_map + i); } } /* * Add the tb in the target page and protect it if necessary. * Called with @p->lock held. */ static inline void tb_page_add(PageDesc *p, TranslationBlock *tb, unsigned int n) { bool page_already_protected; assert_page_locked(p); tb->page_next[n] = p->first_tb; page_already_protected = p->first_tb != 0; p->first_tb = (uintptr_t)tb | n; /* * If some code is already present, then the pages are already * protected. So we handle the case where only the first TB is * allocated in a physical page. */ if (!page_already_protected) { tlb_protect_code(tb->page_addr[n] & TARGET_PAGE_MASK); } } static void tb_record(TranslationBlock *tb, PageDesc *p1, PageDesc *p2) { tb_page_add(p1, tb, 0); if (unlikely(p2)) { tb_page_add(p2, tb, 1); } } static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb) { TranslationBlock *tb1; uintptr_t *pprev; PageForEachNext n1; assert_page_locked(pd); pprev = &pd->first_tb; PAGE_FOR_EACH_TB(unused, unused, pd, tb1, n1) { if (tb1 == tb) { *pprev = tb1->page_next[n1]; return; } pprev = &tb1->page_next[n1]; } g_assert_not_reached(); } static void tb_remove(TranslationBlock *tb) { PageDesc *pd; pd = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); tb_page_remove(pd, tb); if (unlikely(tb->page_addr[1] != -1)) { pd = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); tb_page_remove(pd, tb); } } static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1, PageDesc **ret_p2, tb_page_addr_t phys2, bool alloc) { PageDesc *p1, *p2; tb_page_addr_t page1; tb_page_addr_t page2; assert_memory_lock(); g_assert(phys1 != -1); page1 = phys1 >> TARGET_PAGE_BITS; page2 = phys2 >> TARGET_PAGE_BITS; p1 = page_find_alloc(page1, alloc); if (ret_p1) { *ret_p1 = p1; } if (likely(phys2 == -1)) { page_lock(p1); return; } else if (page1 == page2) { page_lock(p1); if (ret_p2) { *ret_p2 = p1; } return; } p2 = page_find_alloc(page2, alloc); if (ret_p2) { *ret_p2 = p2; } if (page1 < page2) { page_lock(p1); page_lock(p2); } else { page_lock(p2); page_lock(p1); } } /* lock the page(s) of a TB in the correct acquisition order */ static void page_lock_tb(const TranslationBlock *tb) { page_lock_pair(NULL, tb_page_addr0(tb), NULL, tb_page_addr1(tb), false); } static void page_unlock_tb(const TranslationBlock *tb) { PageDesc *p1 = page_find(tb_page_addr0(tb) >> TARGET_PAGE_BITS); page_unlock(p1); if (unlikely(tb_page_addr1(tb) != -1)) { PageDesc *p2 = page_find(tb_page_addr1(tb) >> TARGET_PAGE_BITS); if (p2 != p1) { page_unlock(p2); } } } #endif /* CONFIG_USER_ONLY */ /* flush all the translation blocks */ static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) { bool did_flush = false; mmap_lock(); /* If it is already been done on request of another CPU, just retry. */ if (tb_ctx.tb_flush_count != tb_flush_count.host_int) { goto done; } did_flush = true; CPU_FOREACH(cpu) { tcg_flush_jmp_cache(cpu); } qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE); tb_remove_all(); tcg_region_reset_all(); /* XXX: flush processor icache at this point if cache flush is expensive */ qatomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1); done: mmap_unlock(); if (did_flush) { qemu_plugin_flush_cb(); } } void tb_flush(CPUState *cpu) { if (tcg_enabled()) { unsigned tb_flush_count = qatomic_mb_read(&tb_ctx.tb_flush_count); if (cpu_in_exclusive_context(cpu)) { do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count)); } else { async_safe_run_on_cpu(cpu, do_tb_flush, RUN_ON_CPU_HOST_INT(tb_flush_count)); } } } /* remove @orig from its @n_orig-th jump list */ static inline void tb_remove_from_jmp_list(TranslationBlock *orig, int n_orig) { uintptr_t ptr, ptr_locked; TranslationBlock *dest; TranslationBlock *tb; uintptr_t *pprev; int n; /* mark the LSB of jmp_dest[] so that no further jumps can be inserted */ ptr = qatomic_or_fetch(&orig->jmp_dest[n_orig], 1); dest = (TranslationBlock *)(ptr & ~1); if (dest == NULL) { return; } qemu_spin_lock(&dest->jmp_lock); /* * While acquiring the lock, the jump might have been removed if the * destination TB was invalidated; check again. */ ptr_locked = qatomic_read(&orig->jmp_dest[n_orig]); if (ptr_locked != ptr) { qemu_spin_unlock(&dest->jmp_lock); /* * The only possibility is that the jump was unlinked via * tb_jump_unlink(dest). Seeing here another destination would be a bug, * because we set the LSB above. */ g_assert(ptr_locked == 1 && dest->cflags & CF_INVALID); return; } /* * We first acquired the lock, and since the destination pointer matches, * we know for sure that @orig is in the jmp list. */ pprev = &dest->jmp_list_head; TB_FOR_EACH_JMP(dest, tb, n) { if (tb == orig && n == n_orig) { *pprev = tb->jmp_list_next[n]; /* no need to set orig->jmp_dest[n]; setting the LSB was enough */ qemu_spin_unlock(&dest->jmp_lock); return; } pprev = &tb->jmp_list_next[n]; } g_assert_not_reached(); } /* * Reset the jump entry 'n' of a TB so that it is not chained to another TB. */ void tb_reset_jump(TranslationBlock *tb, int n) { uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]); tb_set_jmp_target(tb, n, addr); } /* remove any jumps to the TB */ static inline void tb_jmp_unlink(TranslationBlock *dest) { TranslationBlock *tb; int n; qemu_spin_lock(&dest->jmp_lock); TB_FOR_EACH_JMP(dest, tb, n) { tb_reset_jump(tb, n); qatomic_and(&tb->jmp_dest[n], (uintptr_t)NULL | 1); /* No need to clear the list entry; setting the dest ptr is enough */ } dest->jmp_list_head = (uintptr_t)NULL; qemu_spin_unlock(&dest->jmp_lock); } static void tb_jmp_cache_inval_tb(TranslationBlock *tb) { CPUState *cpu; if (tb_cflags(tb) & CF_PCREL) { /* A TB may be at any virtual address */ CPU_FOREACH(cpu) { tcg_flush_jmp_cache(cpu); } } else { uint32_t h = tb_jmp_cache_hash_func(tb->pc); CPU_FOREACH(cpu) { CPUJumpCache *jc = cpu->tb_jmp_cache; if (qatomic_read(&jc->array[h].tb) == tb) { qatomic_set(&jc->array[h].tb, NULL); } } } } /* * In user-mode, call with mmap_lock held. * In !user-mode, if @rm_from_page_list is set, call with the TB's pages' * locks held. */ static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list) { uint32_t h; tb_page_addr_t phys_pc; uint32_t orig_cflags = tb_cflags(tb); assert_memory_lock(); /* make sure no further incoming jumps will be chained to this TB */ qemu_spin_lock(&tb->jmp_lock); qatomic_set(&tb->cflags, tb->cflags | CF_INVALID); qemu_spin_unlock(&tb->jmp_lock); /* remove the TB from the hash list */ phys_pc = tb_page_addr0(tb); h = tb_hash_func(phys_pc, (orig_cflags & CF_PCREL ? 0 : tb->pc), tb->flags, orig_cflags, tb->trace_vcpu_dstate); if (!qht_remove(&tb_ctx.htable, tb, h)) { return; } /* remove the TB from the page list */ if (rm_from_page_list) { tb_remove(tb); } /* remove the TB from the hash list */ tb_jmp_cache_inval_tb(tb); /* suppress this TB from the two jump lists */ tb_remove_from_jmp_list(tb, 0); tb_remove_from_jmp_list(tb, 1); /* suppress any remaining jumps to this TB */ tb_jmp_unlink(tb); qatomic_set(&tb_ctx.tb_phys_invalidate_count, tb_ctx.tb_phys_invalidate_count + 1); } static void tb_phys_invalidate__locked(TranslationBlock *tb) { qemu_thread_jit_write(); do_tb_phys_invalidate(tb, true); qemu_thread_jit_execute(); } /* * Invalidate one TB. * Called with mmap_lock held in user-mode. */ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) { if (page_addr == -1 && tb_page_addr0(tb) != -1) { page_lock_tb(tb); do_tb_phys_invalidate(tb, true); page_unlock_tb(tb); } else { do_tb_phys_invalidate(tb, false); } } /* * Add a new TB and link it to the physical page tables. phys_page2 is * (-1) to indicate that only one page contains the TB. * * Called with mmap_lock held for user-mode emulation. * * Returns a pointer @tb, or a pointer to an existing TB that matches @tb. * Note that in !user-mode, another thread might have already added a TB * for the same block of guest code that @tb corresponds to. In that case, * the caller should discard the original @tb, and use instead the returned TB. */ TranslationBlock *tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, tb_page_addr_t phys_page2) { PageDesc *p; PageDesc *p2 = NULL; void *existing_tb = NULL; uint32_t h; assert_memory_lock(); tcg_debug_assert(!(tb->cflags & CF_INVALID)); /* * Add the TB to the page list, acquiring first the pages's locks. * We keep the locks held until after inserting the TB in the hash table, * so that if the insertion fails we know for sure that the TBs are still * in the page descriptors. * Note that inserting into the hash table first isn't an option, since * we can only insert TBs that are fully initialized. */ page_lock_pair(&p, phys_pc, &p2, phys_page2, true); tb_record(tb, p, p2); /* add in the hash table */ h = tb_hash_func(phys_pc, (tb->cflags & CF_PCREL ? 0 : tb->pc), tb->flags, tb->cflags, tb->trace_vcpu_dstate); qht_insert(&tb_ctx.htable, tb, h, &existing_tb); /* remove TB from the page(s) if we couldn't insert it */ if (unlikely(existing_tb)) { tb_remove(tb); tb = existing_tb; } if (p2 && p2 != p) { page_unlock(p2); } page_unlock(p); return tb; } #ifdef CONFIG_USER_ONLY /* * Invalidate all TBs which intersect with the target address range. * Called with mmap_lock held for user-mode emulation. * NOTE: this function must not be called while a TB is running. */ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) { TranslationBlock *tb; PageForEachNext n; assert_memory_lock(); PAGE_FOR_EACH_TB(start, end, unused, tb, n) { tb_phys_invalidate__locked(tb); } } /* * Invalidate all TBs which intersect with the target address page @addr. * Called with mmap_lock held for user-mode emulation * NOTE: this function must not be called while a TB is running. */ void tb_invalidate_phys_page(tb_page_addr_t addr) { tb_page_addr_t start, end; start = addr & TARGET_PAGE_MASK; end = start + TARGET_PAGE_SIZE; tb_invalidate_phys_range(start, end); } /* * Called with mmap_lock held. If pc is not 0 then it indicates the * host PC of the faulting store instruction that caused this invalidate. * Returns true if the caller needs to abort execution of the current * TB (because it was modified by this store and the guest CPU has * precise-SMC semantics). */ bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc) { TranslationBlock *current_tb; bool current_tb_modified; TranslationBlock *tb; PageForEachNext n; /* * Without precise smc semantics, or when outside of a TB, * we can skip to invalidate. */ #ifndef TARGET_HAS_PRECISE_SMC pc = 0; #endif if (!pc) { tb_invalidate_phys_page(addr); return false; } assert_memory_lock(); current_tb = tcg_tb_lookup(pc); addr &= TARGET_PAGE_MASK; current_tb_modified = false; PAGE_FOR_EACH_TB(addr, addr + TARGET_PAGE_SIZE, unused, tb, n) { if (current_tb == tb && (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) { /* * If we are modifying the current TB, we must stop its * execution. We could be more precise by checking that * the modification is after the current PC, but it would * require a specialized function to partially restore * the CPU state. */ current_tb_modified = true; cpu_restore_state_from_tb(current_cpu, current_tb, pc); } tb_phys_invalidate__locked(tb); } if (current_tb_modified) { /* Force execution of one insn next time. */ CPUState *cpu = current_cpu; cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(current_cpu); return true; } return false; } #else /* * @p must be non-NULL. * Call with all @pages locked. */ static void tb_invalidate_phys_page_range__locked(struct page_collection *pages, PageDesc *p, tb_page_addr_t start, tb_page_addr_t end, uintptr_t retaddr) { TranslationBlock *tb; tb_page_addr_t tb_start, tb_end; PageForEachNext n; #ifdef TARGET_HAS_PRECISE_SMC bool current_tb_modified = false; TranslationBlock *current_tb = retaddr ? tcg_tb_lookup(retaddr) : NULL; #endif /* TARGET_HAS_PRECISE_SMC */ /* * We remove all the TBs in the range [start, end[. * XXX: see if in some cases it could be faster to invalidate all the code */ PAGE_FOR_EACH_TB(start, end, p, tb, n) { /* NOTE: this is subtle as a TB may span two physical pages */ if (n == 0) { /* NOTE: tb_end may be after the end of the page, but it is not a problem */ tb_start = tb_page_addr0(tb); tb_end = tb_start + tb->size; } else { tb_start = tb_page_addr1(tb); tb_end = tb_start + ((tb_page_addr0(tb) + tb->size) & ~TARGET_PAGE_MASK); } if (!(tb_end <= start || tb_start >= end)) { #ifdef TARGET_HAS_PRECISE_SMC if (current_tb == tb && (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) { /* * If we are modifying the current TB, we must stop * its execution. We could be more precise by checking * that the modification is after the current PC, but it * would require a specialized function to partially * restore the CPU state. */ current_tb_modified = true; cpu_restore_state_from_tb(current_cpu, current_tb, retaddr); } #endif /* TARGET_HAS_PRECISE_SMC */ tb_phys_invalidate__locked(tb); } } /* if no code remaining, no need to continue to use slow writes */ if (!p->first_tb) { tlb_unprotect_code(start); } #ifdef TARGET_HAS_PRECISE_SMC if (current_tb_modified) { page_collection_unlock(pages); /* Force execution of one insn next time. */ current_cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(current_cpu); mmap_unlock(); cpu_loop_exit_noexc(current_cpu); } #endif } /* * Invalidate all TBs which intersect with the target physical * address page @addr. */ void tb_invalidate_phys_page(tb_page_addr_t addr) { struct page_collection *pages; tb_page_addr_t start, end; PageDesc *p; p = page_find(addr >> TARGET_PAGE_BITS); if (p == NULL) { return; } start = addr & TARGET_PAGE_MASK; end = start + TARGET_PAGE_SIZE; pages = page_collection_lock(start, end); tb_invalidate_phys_page_range__locked(pages, p, start, end, 0); page_collection_unlock(pages); } /* * Invalidate all TBs which intersect with the target physical address range * [start;end[. NOTE: start and end may refer to *different* physical pages. * 'is_cpu_write_access' should be true if called from a real cpu write * access: the virtual CPU will exit the current TB if code is modified inside * this TB. */ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) { struct page_collection *pages; tb_page_addr_t next; pages = page_collection_lock(start, end); for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE; start < end; start = next, next += TARGET_PAGE_SIZE) { PageDesc *pd = page_find(start >> TARGET_PAGE_BITS); tb_page_addr_t bound = MIN(next, end); if (pd == NULL) { continue; } assert_page_locked(pd); tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0); } page_collection_unlock(pages); } /* * Call with all @pages in the range [@start, @start + len[ locked. */ static void tb_invalidate_phys_page_fast__locked(struct page_collection *pages, tb_page_addr_t start, unsigned len, uintptr_t ra) { PageDesc *p; p = page_find(start >> TARGET_PAGE_BITS); if (!p) { return; } assert_page_locked(p); tb_invalidate_phys_page_range__locked(pages, p, start, start + len, ra); } /* * len must be <= 8 and start must be a multiple of len. * Called via softmmu_template.h when code areas are written to with * iothread mutex not held. */ void tb_invalidate_phys_range_fast(ram_addr_t ram_addr, unsigned size, uintptr_t retaddr) { struct page_collection *pages; pages = page_collection_lock(ram_addr, ram_addr + size); tb_invalidate_phys_page_fast__locked(pages, ram_addr, size, retaddr); page_collection_unlock(pages); } #endif /* CONFIG_USER_ONLY */