From 89fee74a0f066dfd73830a7b5fa137e87888c870 Mon Sep 17 00:00:00 2001 From: "Emilio G. Cota" Date: Thu, 7 Apr 2016 13:19:22 -0400 Subject: [PATCH 01/39] tb: consistently use uint32_t for tb->flags We are inconsistent with the type of tb->flags: usage varies loosely between int and uint64_t. Settle to uint32_t everywhere, which is superior to both: at least one target (aarch64) uses the most significant bit in the u32, and uint64_t is wasteful. Compile-tested for all targets. Suggested-by: Laurent Desnogues Suggested-by: Richard Henderson Tested-by: Edgar E. Iglesias Reviewed-by: Edgar E. Iglesias Reviewed-by: Laurent Desnogues Signed-off-by: Emilio G. Cota Signed-off-by: Richard Henderson Message-Id: <1460049562-23517-1-git-send-email-cota@braap.org> --- cpu-exec.c | 6 +++--- exec.c | 2 +- hw/i386/kvmvapic.c | 2 +- include/exec/exec-all.h | 5 +++-- target-alpha/cpu.h | 2 +- target-arm/cpu.h | 2 +- target-cris/cpu.h | 2 +- target-i386/cpu.h | 2 +- target-i386/translate.c | 2 +- target-lm32/cpu.h | 2 +- target-m68k/cpu.h | 2 +- target-microblaze/cpu.h | 2 +- target-mips/cpu.h | 2 +- target-moxie/cpu.h | 2 +- target-openrisc/cpu.h | 2 +- target-ppc/cpu.h | 2 +- target-s390x/cpu.h | 2 +- target-sh4/cpu.h | 2 +- target-sparc/cpu.h | 2 +- target-tilegx/cpu.h | 2 +- target-tricore/cpu.h | 2 +- target-unicore32/cpu.h | 2 +- target-xtensa/cpu.h | 2 +- translate-all.c | 10 +++++----- 24 files changed, 32 insertions(+), 31 deletions(-) diff --git a/cpu-exec.c b/cpu-exec.c index bbfcbfb543..debc65ca69 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -220,7 +220,7 @@ static void cpu_exec_nocache(CPUState *cpu, int max_cycles, static TranslationBlock *tb_find_physical(CPUState *cpu, target_ulong pc, target_ulong cs_base, - uint64_t flags) + uint32_t flags) { CPUArchState *env = (CPUArchState *)cpu->env_ptr; TranslationBlock *tb, **ptb1; @@ -271,7 +271,7 @@ static TranslationBlock *tb_find_physical(CPUState *cpu, static TranslationBlock *tb_find_slow(CPUState *cpu, target_ulong pc, target_ulong cs_base, - uint64_t flags) + uint32_t flags) { TranslationBlock *tb; @@ -314,7 +314,7 @@ static inline TranslationBlock *tb_find_fast(CPUState *cpu) CPUArchState *env = (CPUArchState *)cpu->env_ptr; TranslationBlock *tb; target_ulong cs_base, pc; - int flags; + uint32_t flags; /* we record a subset of the CPU state. It will always be the same before a given translated block diff --git a/exec.c b/exec.c index c4f9036184..ee45472cab 100644 --- a/exec.c +++ b/exec.c @@ -2087,7 +2087,7 @@ static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags) target_ulong pc, cs_base; target_ulong vaddr; CPUWatchpoint *wp; - int cpu_flags; + uint32_t cpu_flags; if (cpu->watchpoint_hit) { /* We re-entered the check after replacing the TB. Now raise diff --git a/hw/i386/kvmvapic.c b/hw/i386/kvmvapic.c index c69f374049..4bb695d5c1 100644 --- a/hw/i386/kvmvapic.c +++ b/hw/i386/kvmvapic.c @@ -397,7 +397,7 @@ static void patch_instruction(VAPICROMState *s, X86CPU *cpu, target_ulong ip) uint32_t imm32; target_ulong current_pc = 0; target_ulong current_cs_base = 0; - int current_flags = 0; + uint32_t current_flags = 0; if (smp_cpus == 1) { handlers = &s->rom_state.up; diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 736209505a..c75fb3ac25 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -76,7 +76,8 @@ bool cpu_restore_state(CPUState *cpu, uintptr_t searched_pc); void QEMU_NORETURN cpu_resume_from_signal(CPUState *cpu, void *puc); void QEMU_NORETURN cpu_io_recompile(CPUState *cpu, uintptr_t retaddr); TranslationBlock *tb_gen_code(CPUState *cpu, - target_ulong pc, target_ulong cs_base, int flags, + target_ulong pc, target_ulong cs_base, + uint32_t flags, int cflags); void cpu_exec_init(CPUState *cpu, Error **errp); void QEMU_NORETURN cpu_loop_exit(CPUState *cpu); @@ -235,7 +236,7 @@ static inline void tlb_flush_by_mmuidx(CPUState *cpu, ...) struct TranslationBlock { target_ulong pc; /* simulated PC corresponding to this block (EIP + CS base) */ target_ulong cs_base; /* CS base for this block */ - uint64_t flags; /* flags defining in which context the code was generated */ + uint32_t flags; /* flags defining in which context the code was generated */ uint16_t size; /* size of target code for this block (1 <= size <= TARGET_PAGE_SIZE) */ uint16_t icount; diff --git a/target-alpha/cpu.h b/target-alpha/cpu.h index 420f2a53fe..b25d7d09d0 100644 --- a/target-alpha/cpu.h +++ b/target-alpha/cpu.h @@ -465,7 +465,7 @@ enum { }; static inline void cpu_get_tb_cpu_state(CPUAlphaState *env, target_ulong *pc, - target_ulong *cs_base, int *pflags) + target_ulong *cs_base, uint32_t *pflags) { int flags = 0; diff --git a/target-arm/cpu.h b/target-arm/cpu.h index 066ff678dc..9deef86786 100644 --- a/target-arm/cpu.h +++ b/target-arm/cpu.h @@ -2117,7 +2117,7 @@ static inline bool arm_cpu_bswap_data(CPUARMState *env) #endif static inline void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { if (is_a64(env)) { *pc = env->pc; diff --git a/target-cris/cpu.h b/target-cris/cpu.h index 415cf91436..a492fc687f 100644 --- a/target-cris/cpu.h +++ b/target-cris/cpu.h @@ -249,7 +249,7 @@ int cris_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw, #include "exec/cpu-all.h" static inline void cpu_get_tb_cpu_state(CPUCRISState *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->pc; *cs_base = 0; diff --git a/target-i386/cpu.h b/target-i386/cpu.h index 732eb6d7ec..444fda9ce6 100644 --- a/target-i386/cpu.h +++ b/target-i386/cpu.h @@ -1269,7 +1269,7 @@ void tcg_x86_init(void); #include "exec/exec-all.h" static inline void cpu_get_tb_cpu_state(CPUX86State *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *cs_base = env->segs[R_CS].base; *pc = *cs_base + env->eip; diff --git a/target-i386/translate.c b/target-i386/translate.c index 1a1214dcb1..3a32f65f8d 100644 --- a/target-i386/translate.c +++ b/target-i386/translate.c @@ -8178,7 +8178,7 @@ void gen_intermediate_code(CPUX86State *env, TranslationBlock *tb) CPUState *cs = CPU(cpu); DisasContext dc1, *dc = &dc1; target_ulong pc_ptr; - uint64_t flags; + uint32_t flags; target_ulong pc_start; target_ulong cs_base; int num_insns; diff --git a/target-lm32/cpu.h b/target-lm32/cpu.h index f220fc0bb9..6a0d297b30 100644 --- a/target-lm32/cpu.h +++ b/target-lm32/cpu.h @@ -226,7 +226,7 @@ int lm32_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw, #include "exec/cpu-all.h" static inline void cpu_get_tb_cpu_state(CPULM32State *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->pc; *cs_base = 0; diff --git a/target-m68k/cpu.h b/target-m68k/cpu.h index 48b4c872fc..d2f467ca9b 100644 --- a/target-m68k/cpu.h +++ b/target-m68k/cpu.h @@ -230,7 +230,7 @@ int m68k_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw, #include "exec/cpu-all.h" static inline void cpu_get_tb_cpu_state(CPUM68KState *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->pc; *cs_base = 0; diff --git a/target-microblaze/cpu.h b/target-microblaze/cpu.h index 2f7335eaa7..bf74e2c032 100644 --- a/target-microblaze/cpu.h +++ b/target-microblaze/cpu.h @@ -322,7 +322,7 @@ int mb_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw, #include "exec/cpu-all.h" static inline void cpu_get_tb_cpu_state(CPUMBState *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->sregs[SR_PC]; *cs_base = 0; diff --git a/target-mips/cpu.h b/target-mips/cpu.h index 866924d188..53e826223f 100644 --- a/target-mips/cpu.h +++ b/target-mips/cpu.h @@ -839,7 +839,7 @@ static inline void restore_pamask(CPUMIPSState *env) } static inline void cpu_get_tb_cpu_state(CPUMIPSState *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->active_tc.PC; *cs_base = 0; diff --git a/target-moxie/cpu.h b/target-moxie/cpu.h index 4ee207796c..1b46e52c37 100644 --- a/target-moxie/cpu.h +++ b/target-moxie/cpu.h @@ -132,7 +132,7 @@ static inline int cpu_mmu_index(CPUMoxieState *env, bool ifetch) #include "exec/exec-all.h" static inline void cpu_get_tb_cpu_state(CPUMoxieState *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->pc; *cs_base = 0; diff --git a/target-openrisc/cpu.h b/target-openrisc/cpu.h index 4b63f25804..ed818af0cf 100644 --- a/target-openrisc/cpu.h +++ b/target-openrisc/cpu.h @@ -392,7 +392,7 @@ int cpu_openrisc_get_phys_data(OpenRISCCPU *cpu, static inline void cpu_get_tb_cpu_state(CPUOpenRISCState *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->pc; *cs_base = 0; diff --git a/target-ppc/cpu.h b/target-ppc/cpu.h index 5282533b38..508f03b74d 100644 --- a/target-ppc/cpu.h +++ b/target-ppc/cpu.h @@ -2303,7 +2303,7 @@ static inline void cpu_write_xer(CPUPPCState *env, target_ulong xer) } static inline void cpu_get_tb_cpu_state(CPUPPCState *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->nip; *cs_base = 0; diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h index 6d97c089a4..07f76ad884 100644 --- a/target-s390x/cpu.h +++ b/target-s390x/cpu.h @@ -338,7 +338,7 @@ static inline uint64_t cpu_mmu_idx_to_asc(int mmu_idx) } static inline void cpu_get_tb_cpu_state(CPUS390XState* env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->psw.addr; *cs_base = 0; diff --git a/target-sh4/cpu.h b/target-sh4/cpu.h index 3b23e967bb..10c0191795 100644 --- a/target-sh4/cpu.h +++ b/target-sh4/cpu.h @@ -347,7 +347,7 @@ static inline void cpu_write_sr(CPUSH4State *env, target_ulong sr) } static inline void cpu_get_tb_cpu_state(CPUSH4State *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->pc; *cs_base = 0; diff --git a/target-sparc/cpu.h b/target-sparc/cpu.h index dc46122758..59ec7cafbe 100644 --- a/target-sparc/cpu.h +++ b/target-sparc/cpu.h @@ -688,7 +688,7 @@ trap_state* cpu_tsptr(CPUSPARCState* env); #define TB_FLAG_AM_ENABLED (1 << 5) static inline void cpu_get_tb_cpu_state(CPUSPARCState *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->pc; *cs_base = env->npc; diff --git a/target-tilegx/cpu.h b/target-tilegx/cpu.h index 022cad186a..c9dda127c9 100644 --- a/target-tilegx/cpu.h +++ b/target-tilegx/cpu.h @@ -169,7 +169,7 @@ TileGXCPU *cpu_tilegx_init(const char *cpu_model); #define cpu_signal_handler cpu_tilegx_signal_handler static inline void cpu_get_tb_cpu_state(CPUTLGState *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->pc; *cs_base = 0; diff --git a/target-tricore/cpu.h b/target-tricore/cpu.h index 90045a93d2..eaebdd28e6 100644 --- a/target-tricore/cpu.h +++ b/target-tricore/cpu.h @@ -377,7 +377,7 @@ void tricore_tcg_init(void); int cpu_tricore_signal_handler(int host_signum, void *pinfo, void *puc); static inline void cpu_get_tb_cpu_state(CPUTriCoreState *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->PC; *cs_base = 0; diff --git a/target-unicore32/cpu.h b/target-unicore32/cpu.h index 9c1fbf9b77..e64cb7ecdd 100644 --- a/target-unicore32/cpu.h +++ b/target-unicore32/cpu.h @@ -144,7 +144,7 @@ UniCore32CPU *uc32_cpu_init(const char *cpu_model); #define cpu_init(cpu_model) CPU(uc32_cpu_init(cpu_model)) static inline void cpu_get_tb_cpu_state(CPUUniCore32State *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { *pc = env->regs[31]; *cs_base = 0; diff --git a/target-xtensa/cpu.h b/target-xtensa/cpu.h index d0bd9dada8..7bfc9c841d 100644 --- a/target-xtensa/cpu.h +++ b/target-xtensa/cpu.h @@ -507,7 +507,7 @@ static inline int cpu_mmu_index(CPUXtensaState *env, bool ifetch) #define XTENSA_TBFLAG_WINDOW_SHIFT 15 static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc, - target_ulong *cs_base, int *flags) + target_ulong *cs_base, uint32_t *flags) { CPUState *cs = CPU(xtensa_env_get_cpu(env)); diff --git a/translate-all.c b/translate-all.c index 8329ea60ee..1a8f68bd82 100644 --- a/translate-all.c +++ b/translate-all.c @@ -1051,7 +1051,7 @@ static void build_page_bitmap(PageDesc *p) /* Called with mmap_lock held for user mode emulation. */ TranslationBlock *tb_gen_code(CPUState *cpu, target_ulong pc, target_ulong cs_base, - int flags, int cflags) + uint32_t flags, int cflags) { CPUArchState *env = cpu->env_ptr; TranslationBlock *tb; @@ -1205,7 +1205,7 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, int current_tb_modified = 0; target_ulong current_pc = 0; target_ulong current_cs_base = 0; - int current_flags = 0; + uint32_t current_flags = 0; #endif /* TARGET_HAS_PRECISE_SMC */ p = page_find(start >> TARGET_PAGE_BITS); @@ -1350,7 +1350,7 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr, int current_tb_modified = 0; target_ulong current_pc = 0; target_ulong current_cs_base = 0; - int current_flags = 0; + uint32_t current_flags = 0; #endif addr &= TARGET_PAGE_MASK; @@ -1574,7 +1574,7 @@ void tb_check_watchpoint(CPUState *cpu) CPUArchState *env = cpu->env_ptr; target_ulong pc, cs_base; tb_page_addr_t addr; - int flags; + uint32_t flags; cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); addr = get_page_addr_code(env, pc); @@ -1593,7 +1593,7 @@ void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) TranslationBlock *tb; uint32_t n, cflags; target_ulong pc, cs_base; - uint64_t flags; + uint32_t flags; tb = tb_find_pc(retaddr); if (!tb) { From 18a60a76147569ca9e11b0607e50ce4012fe1aaa Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Fri, 22 Apr 2016 19:08:43 +0300 Subject: [PATCH 02/39] include/qemu/osdep.h: Add a macro to check for alignment Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Message-Id: <1461341333-19646-2-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- include/qemu/osdep.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/qemu/osdep.h b/include/qemu/osdep.h index 408783f532..e3bc50b613 100644 --- a/include/qemu/osdep.h +++ b/include/qemu/osdep.h @@ -158,6 +158,9 @@ extern int daemon(int, int); /* Round number up to multiple */ #define QEMU_ALIGN_UP(n, m) QEMU_ALIGN_DOWN((n) + (m) - 1, (m)) +/* Check if n is a multiple of m */ +#define QEMU_IS_ALIGNED(n, m) (((n) % (m)) == 0) + #ifndef ROUND_UP #define ROUND_UP(n,d) (((n) + (d) - 1) & -(d)) #endif From 6b587d3cda48e7ba26de8d30bf0d8a7063970715 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Fri, 22 Apr 2016 19:08:44 +0300 Subject: [PATCH 03/39] include/qemu/osdep.h: Add macros for pointer alignment These macros provide a convenient way to n-byte align pointers up and down and check if a pointer is n-byte aligned. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Message-Id: <1461341333-19646-3-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- include/qemu/osdep.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/include/qemu/osdep.h b/include/qemu/osdep.h index e3bc50b613..1e3221cbec 100644 --- a/include/qemu/osdep.h +++ b/include/qemu/osdep.h @@ -161,6 +161,17 @@ extern int daemon(int, int); /* Check if n is a multiple of m */ #define QEMU_IS_ALIGNED(n, m) (((n) % (m)) == 0) +/* n-byte align pointer down */ +#define QEMU_ALIGN_PTR_DOWN(p, n) \ + ((typeof(p))QEMU_ALIGN_DOWN((uintptr_t)(p), (n))) + +/* n-byte align pointer up */ +#define QEMU_ALIGN_PTR_UP(p, n) \ + ((typeof(p))QEMU_ALIGN_UP((uintptr_t)(p), (n))) + +/* Check if pointer p is n-bytes aligned */ +#define QEMU_PTR_IS_ALIGNED(p, n) QEMU_IS_ALIGNED((uintptr_t)(p), (n)) + #ifndef ROUND_UP #define ROUND_UP(n,d) (((n) + (d) - 1) & -(d)) #endif From 76442a939ef8fc062b35a890a824de203bb5bd5f Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Fri, 22 Apr 2016 19:08:45 +0300 Subject: [PATCH 04/39] tci: Make direct jump patching thread-safe Ensure direct jump patching in TCI is atomic by: * naturally aligning a location of direct jump address; * using atomic_read()/atomic_set() to load/store the address. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Message-Id: <1461341333-19646-4-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- include/exec/exec-all.h | 2 +- tcg/tci/tcg-target.inc.c | 2 ++ tci.c | 5 ++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index c75fb3ac25..d49befdba5 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -303,7 +303,7 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr); static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr) { /* patch the branch destination */ - *(uint32_t *)jmp_addr = addr - (jmp_addr + 4); + atomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4)); /* no need to flush icache explicitly */ } #elif defined(_ARCH_PPC) diff --git a/tcg/tci/tcg-target.inc.c b/tcg/tci/tcg-target.inc.c index e2fc52a167..85eeb5de24 100644 --- a/tcg/tci/tcg-target.inc.c +++ b/tcg/tci/tcg-target.inc.c @@ -556,6 +556,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, if (s->tb_jmp_offset) { /* Direct jump method. */ tcg_debug_assert(args[0] < ARRAY_SIZE(s->tb_jmp_offset)); + /* Align for atomic patching and thread safety */ + s->code_ptr = QEMU_ALIGN_PTR_UP(s->code_ptr, 4); s->tb_jmp_offset[args[0]] = tcg_current_code_size(s); tcg_out32(s, 0); } else { diff --git a/tci.c b/tci.c index 82705fe772..a8939e6d3c 100644 --- a/tci.c +++ b/tci.c @@ -1089,7 +1089,10 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr) goto exit; break; case INDEX_op_goto_tb: - t0 = tci_read_i32(&tb_ptr); + /* Jump address is aligned */ + tb_ptr = QEMU_ALIGN_PTR_UP(tb_ptr, 4); + t0 = atomic_read((int32_t *)tb_ptr); + tb_ptr += sizeof(int32_t); tci_assert(tb_ptr == old_code_ptr + op_size); tb_ptr += (int32_t)t0; continue; From 399f1648573d49b9411089e4e2ea62b6357b166e Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Fri, 22 Apr 2016 19:08:46 +0300 Subject: [PATCH 05/39] tcg/ppc: Make direct jump patching thread-safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ensure direct jump patching in PPC is atomic by: * limiting translation buffer size in 32-bit mode to be addressable by Branch I-form instruction; * using atomic_read()/atomic_set() for code patching. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Alex Bennée Message-Id: <1461341333-19646-5-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- tcg/ppc/tcg-target.inc.c | 22 ++++++++++++++++++---- translate-all.c | 2 ++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tcg/ppc/tcg-target.inc.c b/tcg/ppc/tcg-target.inc.c index 00bb90fc25..039fa77748 100644 --- a/tcg/ppc/tcg-target.inc.c +++ b/tcg/ppc/tcg-target.inc.c @@ -1237,6 +1237,7 @@ static void tcg_out_brcond2 (TCGContext *s, const TCGArg *args, tcg_out_bc(s, BC | BI(7, CR_EQ) | BO_COND_TRUE, arg_label(args[5])); } +#ifdef __powerpc64__ void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr) { tcg_insn_unit i1, i2; @@ -1265,11 +1266,18 @@ void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr) pair = (uint64_t)i2 << 32 | i1; #endif - /* ??? __atomic_store_8, presuming there's some way to do that - for 32-bit, otherwise this is good enough for 64-bit. */ - *(uint64_t *)jmp_addr = pair; + atomic_set((uint64_t *)jmp_addr, pair); flush_icache_range(jmp_addr, jmp_addr + 8); } +#else +void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr) +{ + intptr_t diff = addr - jmp_addr; + tcg_debug_assert(in_range_b(diff)); + atomic_set((uint32_t *)jmp_addr, B | (diff & 0x3fffffc)); + flush_icache_range(jmp_addr, jmp_addr + 4); +} +#endif static void tcg_out_call(TCGContext *s, tcg_insn_unit *target) { @@ -1895,7 +1903,9 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, break; case INDEX_op_goto_tb: tcg_debug_assert(s->tb_jmp_offset); - /* Direct jump. Ensure the next insns are 8-byte aligned. */ + /* Direct jump. */ +#ifdef __powerpc64__ + /* Ensure the next insns are 8-byte aligned. */ if ((uintptr_t)s->code_ptr & 7) { tcg_out32(s, NOP); } @@ -1904,6 +1914,10 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, s->code_ptr += 2; tcg_out32(s, MTSPR | RS(TCG_REG_TMP1) | CTR); tcg_out32(s, BCCTR | BO_ALWAYS); +#else + /* To be replaced by a branch. */ + s->code_ptr++; +#endif s->tb_next_offset[args[0]] = tcg_current_code_size(s); break; case INDEX_op_br: diff --git a/translate-all.c b/translate-all.c index 1a8f68bd82..781819e940 100644 --- a/translate-all.c +++ b/translate-all.c @@ -464,6 +464,8 @@ static inline PageDesc *page_find(tb_page_addr_t index) # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) #elif defined(__powerpc64__) # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) +#elif defined(__powerpc__) +# define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024) #elif defined(__aarch64__) # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024) #elif defined(__arm__) From 0d07abf05e98903c7faf204a9a90f7d45b7554dc Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Fri, 22 Apr 2016 19:08:47 +0300 Subject: [PATCH 06/39] tcg/i386: Make direct jump patching thread-safe Ensure direct jump patching in i386 is atomic by: * naturally aligning a location of direct jump address; * using atomic_read()/atomic_set() for code patching. tcg_out_nopn() implementation: Suggested-by: Richard Henderson . Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Message-Id: <1461341333-19646-6-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- include/exec/exec-all.h | 2 +- tcg/i386/tcg-target.inc.c | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index d49befdba5..0ab7803259 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -313,7 +313,7 @@ void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr); static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr) { /* patch the branch destination */ - stl_le_p((void*)jmp_addr, addr - (jmp_addr + 4)); + atomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4)); /* no need to flush icache explicitly */ } #elif defined(__s390x__) diff --git a/tcg/i386/tcg-target.inc.c b/tcg/i386/tcg-target.inc.c index 007407c3fc..8d242a6665 100644 --- a/tcg/i386/tcg-target.inc.c +++ b/tcg/i386/tcg-target.inc.c @@ -1123,6 +1123,21 @@ static void tcg_out_jmp(TCGContext *s, tcg_insn_unit *dest) tcg_out_branch(s, 0, dest); } +static void tcg_out_nopn(TCGContext *s, int n) +{ + int i; + /* Emit 1 or 2 operand size prefixes for the standard one byte nop, + * "xchg %eax,%eax", forming "xchg %ax,%ax". All cores accept the + * duplicate prefix, and all of the interesting recent cores can + * decode and discard the duplicates in a single cycle. + */ + tcg_debug_assert(n >= 1); + for (i = 1; i < n; ++i) { + tcg_out8(s, 0x66); + } + tcg_out8(s, 0x90); +} + #if defined(CONFIG_SOFTMMU) /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, * int mmu_idx, uintptr_t ra) @@ -1777,6 +1792,14 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_goto_tb: if (s->tb_jmp_offset) { /* direct jump method */ + int gap; + /* jump displacement must be aligned for atomic patching; + * see if we need to add extra nops before jump + */ + gap = tcg_pcrel_diff(s, QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4)); + if (gap != 1) { + tcg_out_nopn(s, gap - 1); + } tcg_out8(s, OPC_JMP_long); /* jmp im */ s->tb_jmp_offset[args[0]] = tcg_current_code_size(s); tcg_out32(s, 0); From ed3d51ecd7fe248d3959e469d53890ac9ffe0cd2 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Fri, 22 Apr 2016 19:08:48 +0300 Subject: [PATCH 07/39] tcg/s390: Make direct jump patching thread-safe Ensure direct jump patching in s390 is atomic by: * naturally aligning a location of direct jump address; * using atomic_read()/atomic_set() for code patching. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Message-Id: <1461341333-19646-7-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- include/exec/exec-all.h | 2 +- tcg/s390/tcg-target.inc.c | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 0ab7803259..6c095e8636 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -321,7 +321,7 @@ static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr) { /* patch the branch destination */ intptr_t disp = addr - (jmp_addr - 2); - stl_be_p((void*)jmp_addr, disp / 2); + atomic_set((int32_t *)jmp_addr, disp / 2); /* no need to flush icache explicitly */ } #elif defined(__aarch64__) diff --git a/tcg/s390/tcg-target.inc.c b/tcg/s390/tcg-target.inc.c index 5805532398..8e8064ce73 100644 --- a/tcg/s390/tcg-target.inc.c +++ b/tcg/s390/tcg-target.inc.c @@ -219,6 +219,8 @@ typedef enum S390Opcode { RX_ST = 0x50, RX_STC = 0x42, RX_STH = 0x40, + + NOP = 0x0707, } S390Opcode; #ifdef CONFIG_DEBUG_TCG @@ -1716,6 +1718,12 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_goto_tb: if (s->tb_jmp_offset) { + /* branch displacement must be aligned for atomic patching; + * see if we need to add extra nop before branch + */ + if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) { + tcg_out16(s, NOP); + } tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4)); s->tb_jmp_offset[args[0]] = tcg_current_code_size(s); s->code_ptr += 2; From 7d14e0e2d661479985197203589c38840e1066df Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Fri, 22 Apr 2016 19:08:49 +0300 Subject: [PATCH 08/39] tcg/arm: Make direct jump patching thread-safe Ensure direct jump patching in ARM is atomic by using atomic_read()/atomic_set() for code patching. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Message-Id: <1461341333-19646-8-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- include/exec/exec-all.h | 25 ++----------------------- tcg/arm/tcg-target.inc.c | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 6c095e8636..30cdd699f4 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -328,29 +328,8 @@ static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr) void aarch64_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr); #define tb_set_jmp_target1 aarch64_tb_set_jmp_target #elif defined(__arm__) -static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr) -{ -#if !QEMU_GNUC_PREREQ(4, 1) - register unsigned long _beg __asm ("a1"); - register unsigned long _end __asm ("a2"); - register unsigned long _flg __asm ("a3"); -#endif - - /* we could use a ldr pc, [pc, #-4] kind of branch and avoid the flush */ - *(uint32_t *)jmp_addr = - (*(uint32_t *)jmp_addr & ~0xffffff) - | (((addr - (jmp_addr + 8)) >> 2) & 0xffffff); - -#if QEMU_GNUC_PREREQ(4, 1) - __builtin___clear_cache((char *) jmp_addr, (char *) jmp_addr + 4); -#else - /* flush icache */ - _beg = jmp_addr; - _end = jmp_addr + 4; - _flg = 0; - __asm __volatile__ ("swi 0x9f0002" : : "r" (_beg), "r" (_end), "r" (_flg)); -#endif -} +void arm_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr); +#define tb_set_jmp_target1 arm_tb_set_jmp_target #elif defined(__sparc__) || defined(__mips__) void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr); #else diff --git a/tcg/arm/tcg-target.inc.c b/tcg/arm/tcg-target.inc.c index 2b7fbddbf0..977baa05c5 100644 --- a/tcg/arm/tcg-target.inc.c +++ b/tcg/arm/tcg-target.inc.c @@ -121,6 +121,14 @@ static inline void reloc_pc24(tcg_insn_unit *code_ptr, tcg_insn_unit *target) *code_ptr = (*code_ptr & ~0xffffff) | (offset & 0xffffff); } +static inline void reloc_pc24_atomic(tcg_insn_unit *code_ptr, tcg_insn_unit *target) +{ + ptrdiff_t offset = (tcg_ptr_byte_diff(target, code_ptr) - 8) >> 2; + tcg_insn_unit insn = atomic_read(code_ptr); + tcg_debug_assert(offset == sextract32(offset, 0, 24)); + atomic_set(code_ptr, deposit32(insn, 0, 24, offset)); +} + static void patch_reloc(tcg_insn_unit *code_ptr, int type, intptr_t value, intptr_t addend) { @@ -1038,6 +1046,16 @@ static void tcg_out_call(TCGContext *s, tcg_insn_unit *addr) } } +void arm_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr) +{ + tcg_insn_unit *code_ptr = (tcg_insn_unit *)jmp_addr; + tcg_insn_unit *target = (tcg_insn_unit *)addr; + + /* we could use a ldr pc, [pc, #-4] kind of branch and avoid the flush */ + reloc_pc24_atomic(code_ptr, target); + flush_icache_range(jmp_addr, jmp_addr + 4); +} + static inline void tcg_out_goto_label(TCGContext *s, int cond, TCGLabel *l) { if (l->has_value) { From 9e269112953be4d670cb0d25042bd6546fcf3e45 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Fri, 22 Apr 2016 19:08:50 +0300 Subject: [PATCH 09/39] tcg/aarch64: Make direct jump patching thread-safe Ensure direct jump patching in AArch64 is atomic by using atomic_read()/atomic_set() for code patching. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Message-Id: <1461341333-19646-9-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- tcg/aarch64/tcg-target.inc.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tcg/aarch64/tcg-target.inc.c b/tcg/aarch64/tcg-target.inc.c index a8fb4420de..88183c830f 100644 --- a/tcg/aarch64/tcg-target.inc.c +++ b/tcg/aarch64/tcg-target.inc.c @@ -73,6 +73,18 @@ static inline void reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target) *code_ptr = deposit32(*code_ptr, 0, 26, offset); } +static inline void reloc_pc26_atomic(tcg_insn_unit *code_ptr, + tcg_insn_unit *target) +{ + ptrdiff_t offset = target - code_ptr; + tcg_insn_unit insn; + tcg_debug_assert(offset == sextract64(offset, 0, 26)); + /* read instruction, mask away previous PC_REL26 parameter contents, + set the proper offset, then write back the instruction. */ + insn = atomic_read(code_ptr); + atomic_set(code_ptr, deposit32(insn, 0, 26, offset)); +} + static inline void reloc_pc19(tcg_insn_unit *code_ptr, tcg_insn_unit *target) { ptrdiff_t offset = target - code_ptr; @@ -835,7 +847,7 @@ void aarch64_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr) tcg_insn_unit *code_ptr = (tcg_insn_unit *)jmp_addr; tcg_insn_unit *target = (tcg_insn_unit *)addr; - reloc_pc26(code_ptr, target); + reloc_pc26_atomic(code_ptr, target); flush_icache_range(jmp_addr, jmp_addr + 4); } From 84f79fb7c6e857edc807e4a251338243ce0cbac3 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Fri, 22 Apr 2016 19:08:51 +0300 Subject: [PATCH 10/39] tcg/sparc: Make direct jump patching thread-safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ensure direct jump patching in SPARC is atomic by using atomic_read()/atomic_set() for code patching. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Alex Bennée Message-Id: <1461341333-19646-10-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- tcg/sparc/tcg-target.inc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcg/sparc/tcg-target.inc.c b/tcg/sparc/tcg-target.inc.c index d641cfd8c5..c6479e2507 100644 --- a/tcg/sparc/tcg-target.inc.c +++ b/tcg/sparc/tcg-target.inc.c @@ -1647,6 +1647,6 @@ void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr) the code_gen_buffer can't be larger than 2GB. */ tcg_debug_assert(disp == (int32_t)disp); - *ptr = CALL | (uint32_t)disp >> 2; + atomic_set(ptr, deposit32(CALL, 0, 30, disp >> 2)); flush_icache_range(jmp_addr, jmp_addr + 4); } From c82460a560176ef69c2f0662bd280612e274db96 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Fri, 22 Apr 2016 19:08:52 +0300 Subject: [PATCH 11/39] tcg/mips: Make direct jump patching thread-safe Ensure direct jump patching in MIPS is atomic by using atomic_read()/atomic_set() for code patching. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Message-Id: <1461341333-19646-11-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson [rth: Merged the deposit32 followup.] [rth: Merged the following followup.] Message-Id: <1462210518-26522-1-git-send-email-sergey.fedorov@linaro.org> --- tcg/mips/tcg-target.inc.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tcg/mips/tcg-target.inc.c b/tcg/mips/tcg-target.inc.c index aaf881cfd0..1e5a6b41fa 100644 --- a/tcg/mips/tcg-target.inc.c +++ b/tcg/mips/tcg-target.inc.c @@ -1885,7 +1885,6 @@ static void tcg_target_init(TCGContext *s) void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr) { - uint32_t *ptr = (uint32_t *)jmp_addr; - *ptr = deposit32(*ptr, 0, 26, addr >> 2); + atomic_set((uint32_t *)jmp_addr, deposit32(OPC_J, 0, 26, addr >> 2)); flush_icache_range(jmp_addr, jmp_addr + 4); } From 10b4f4855537dd421e193a7d0416513116370558 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Fri, 22 Apr 2016 19:08:53 +0300 Subject: [PATCH 12/39] tcg: Note requirement on atomic direct jump patching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Alex Bennée Message-Id: <1461341333-19646-12-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- include/exec/exec-all.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 30cdd699f4..6c113a3d20 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -230,6 +230,7 @@ static inline void tlb_flush_by_mmuidx(CPUState *cpu, ...) || defined(__sparc__) || defined(__aarch64__) \ || defined(__s390x__) || defined(__mips__) \ || defined(CONFIG_TCG_INTERPRETER) +/* NOTE: Direct jump patching must be atomic to be thread-safe. */ #define USE_DIRECT_JUMP #endif From 835154b6e2200460f04719d0028716a37c178368 Mon Sep 17 00:00:00 2001 From: "Emilio G. Cota" Date: Thu, 21 Apr 2016 20:01:53 -0400 Subject: [PATCH 13/39] translate-all: remove redundant setting of tcg_ctx.code_gen_buffer_size The setting of tcg_ctx.code_gen_buffer_size is done by the only caller of size_code_gen_buffer(), which is code_gen_alloc(): $ git grep size_code_gen_buffer translate-all.c:static inline size_t size_code_gen_buffer(size_t tb_size) translate-all.c: tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size); Signed-off-by: Emilio G. Cota Message-Id: <1461283314-2353-1-git-send-email-cota@braap.org> Signed-off-by: Richard Henderson --- translate-all.c | 1 - 1 file changed, 1 deletion(-) diff --git a/translate-all.c b/translate-all.c index 781819e940..6b0ecb4f9f 100644 --- a/translate-all.c +++ b/translate-all.c @@ -507,7 +507,6 @@ static inline size_t size_code_gen_buffer(size_t tb_size) if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) { tb_size = MAX_CODE_GEN_BUFFER_SIZE; } - tcg_ctx.code_gen_buffer_size = tb_size; return tb_size; } From 8bdf4997823126a39bd4c99e4b2283b02cc7865f Mon Sep 17 00:00:00 2001 From: "Emilio G. Cota" Date: Thu, 21 Apr 2016 20:01:54 -0400 Subject: [PATCH 14/39] translate-all: add missing munmap of the code_gen guard page for MIPS Signed-off-by: Emilio G. Cota Message-Id: <1461283314-2353-2-git-send-email-cota@braap.org> Signed-off-by: Richard Henderson --- translate-all.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/translate-all.c b/translate-all.c index 6b0ecb4f9f..93b91ba5e7 100644 --- a/translate-all.c +++ b/translate-all.c @@ -684,11 +684,11 @@ static inline void *alloc_code_gen_buffer(void) case 1: if (!cross_256mb(buf2, size)) { /* Success! Use the new buffer. */ - munmap(buf, size); + munmap(buf, size + qemu_real_host_page_size); break; } /* Failure. Work with what we had. */ - munmap(buf2, size); + munmap(buf2, size + qemu_real_host_page_size); /* fallthru */ default: /* Split the original buffer. Free the smaller half. */ From 7ba6a512ae439c98c0c1f0f4348c079d90f9dd9d Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 24 Apr 2016 15:55:29 -0700 Subject: [PATCH 15/39] translate-all: Adjust 256mb testing for mips64 Make sure we preserve the high 32-bits when masking for mips64. Signed-off-by: Richard Henderson --- translate-all.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/translate-all.c b/translate-all.c index 93b91ba5e7..79a515dde7 100644 --- a/translate-all.c +++ b/translate-all.c @@ -515,7 +515,7 @@ static inline size_t size_code_gen_buffer(size_t tb_size) that the buffer not cross a 256MB boundary. */ static inline bool cross_256mb(void *addr, size_t size) { - return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & 0xf0000000; + return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful; } /* We weren't able to allocate a buffer without crossing that boundary, @@ -523,7 +523,7 @@ static inline bool cross_256mb(void *addr, size_t size) Returns the new base of the buffer, and adjusts code_gen_buffer_size. */ static inline void *split_cross_256mb(void *buf1, size_t size1) { - void *buf2 = (void *)(((uintptr_t)buf1 + size1) & 0xf0000000); + void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful); size_t size2 = buf1 + size1 - buf2; size1 = buf2 - buf1; From f309101c26b59641fc1aa8fb2a98a5441cdaea03 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Sun, 10 Apr 2016 23:35:45 +0300 Subject: [PATCH 16/39] tcg: Clean up direct block chaining data fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Briefly describe in a comment how direct block chaining is done. It should help in understanding of the following data fields. Rename some fields in TranslationBlock and TCGContext structures to better reflect their purpose (dropping excessive 'tb_' prefix in TranslationBlock but keeping it in TCGContext): tb_next_offset => jmp_reset_offset tb_jmp_offset => jmp_insn_offset tb_next => jmp_target_addr jmp_next => jmp_list_next jmp_first => jmp_list_first Avoid using a magic constant as an invalid offset which is used to indicate that there's no n-th jump generated. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Alex Bennée Signed-off-by: Richard Henderson --- include/exec/exec-all.h | 44 ++++++++++++++++++++----------- tcg/aarch64/tcg-target.inc.c | 7 ++--- tcg/arm/tcg-target.inc.c | 8 +++--- tcg/i386/tcg-target.inc.c | 8 +++--- tcg/ia64/tcg-target.inc.c | 6 ++--- tcg/mips/tcg-target.inc.c | 8 +++--- tcg/ppc/tcg-target.inc.c | 6 ++--- tcg/s390/tcg-target.inc.c | 11 ++++---- tcg/sparc/tcg-target.inc.c | 9 ++++--- tcg/tcg.h | 6 ++--- tcg/tci/tcg-target.inc.c | 10 +++---- translate-all.c | 51 +++++++++++++++++++----------------- 12 files changed, 96 insertions(+), 78 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 6c113a3d20..445d946d84 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -259,20 +259,32 @@ struct TranslationBlock { struct TranslationBlock *page_next[2]; tb_page_addr_t page_addr[2]; - /* the following data are used to directly call another TB from - the code of this one. */ - uint16_t tb_next_offset[2]; /* offset of original jump target */ + /* The following data are used to directly call another TB from + * the code of this one. This can be done either by emitting direct or + * indirect native jump instructions. These jumps are reset so that the TB + * just continue its execution. The TB can be linked to another one by + * setting one of the jump targets (or patching the jump instruction). Only + * two of such jumps are supported. + */ + uint16_t jmp_reset_offset[2]; /* offset of original jump target */ +#define TB_JMP_RESET_OFFSET_INVALID 0xffff /* indicates no jump generated */ #ifdef USE_DIRECT_JUMP - uint16_t tb_jmp_offset[2]; /* offset of jump instruction */ + uint16_t jmp_insn_offset[2]; /* offset of native jump instruction */ #else - uintptr_t tb_next[2]; /* address of jump generated code */ + uintptr_t jmp_target_addr[2]; /* target address for indirect jump */ #endif - /* list of TBs jumping to this one. This is a circular list using - the two least significant bits of the pointers to tell what is - the next pointer: 0 = jmp_next[0], 1 = jmp_next[1], 2 = - jmp_first */ - struct TranslationBlock *jmp_next[2]; - struct TranslationBlock *jmp_first; + /* Each TB has an assosiated circular list of TBs jumping to this one. + * jmp_list_first points to the first TB jumping to this one. + * jmp_list_next is used to point to the next TB in a list. + * Since each TB can have two jumps, it can participate in two lists. + * The two least significant bits of a pointer are used to choose which + * data field holds a pointer to the next TB: + * 0 => jmp_list_next[0], 1 => jmp_list_next[1], 2 => jmp_list_first. + * In other words, 0/1 tells which jump is used in the pointed TB, + * and 2 means that this is a pointer back to the target TB of this list. + */ + struct TranslationBlock *jmp_list_next[2]; + struct TranslationBlock *jmp_list_first; }; #include "qemu/thread.h" @@ -340,7 +352,7 @@ void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr); static inline void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr) { - uint16_t offset = tb->tb_jmp_offset[n]; + uint16_t offset = tb->jmp_insn_offset[n]; tb_set_jmp_target1((uintptr_t)(tb->tc_ptr + offset), addr); } @@ -350,7 +362,7 @@ static inline void tb_set_jmp_target(TranslationBlock *tb, static inline void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr) { - tb->tb_next[n] = addr; + tb->jmp_target_addr[n] = addr; } #endif @@ -359,7 +371,7 @@ static inline void tb_add_jump(TranslationBlock *tb, int n, TranslationBlock *tb_next) { /* NOTE: this test is only needed for thread safety */ - if (!tb->jmp_next[n]) { + if (!tb->jmp_list_next[n]) { qemu_log_mask_and_addr(CPU_LOG_EXEC, tb->pc, "Linking TBs %p [" TARGET_FMT_lx "] index %d -> %p [" TARGET_FMT_lx "]\n", @@ -369,8 +381,8 @@ static inline void tb_add_jump(TranslationBlock *tb, int n, tb_set_jmp_target(tb, n, (uintptr_t)tb_next->tc_ptr); /* add in TB jmp circular list */ - tb->jmp_next[n] = tb_next->jmp_first; - tb_next->jmp_first = (TranslationBlock *)((uintptr_t)(tb) | (n)); + tb->jmp_list_next[n] = tb_next->jmp_list_first; + tb_next->jmp_list_first = (TranslationBlock *)((uintptr_t)tb | n); } } diff --git a/tcg/aarch64/tcg-target.inc.c b/tcg/aarch64/tcg-target.inc.c index 88183c830f..1447f7c216 100644 --- a/tcg/aarch64/tcg-target.inc.c +++ b/tcg/aarch64/tcg-target.inc.c @@ -1306,12 +1306,13 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, #ifndef USE_DIRECT_JUMP #error "USE_DIRECT_JUMP required for aarch64" #endif - tcg_debug_assert(s->tb_jmp_offset != NULL); /* consistency for USE_DIRECT_JUMP */ - s->tb_jmp_offset[a0] = tcg_current_code_size(s); + /* consistency for USE_DIRECT_JUMP */ + tcg_debug_assert(s->tb_jmp_insn_offset != NULL); + s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); /* actual branch destination will be patched by aarch64_tb_set_jmp_target later, beware retranslation. */ tcg_out_goto_noaddr(s); - s->tb_next_offset[a0] = tcg_current_code_size(s); + s->tb_jmp_reset_offset[a0] = tcg_current_code_size(s); break; case INDEX_op_br: diff --git a/tcg/arm/tcg-target.inc.c b/tcg/arm/tcg-target.inc.c index 977baa05c5..f9f54c64c6 100644 --- a/tcg/arm/tcg-target.inc.c +++ b/tcg/arm/tcg-target.inc.c @@ -1665,17 +1665,17 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_goto(s, COND_AL, tb_ret_addr); break; case INDEX_op_goto_tb: - if (s->tb_jmp_offset) { + if (s->tb_jmp_insn_offset) { /* Direct jump method */ - s->tb_jmp_offset[args[0]] = tcg_current_code_size(s); + s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); tcg_out_b_noaddr(s, COND_AL); } else { /* Indirect jump method */ - intptr_t ptr = (intptr_t)(s->tb_next + args[0]); + intptr_t ptr = (intptr_t)(s->tb_jmp_target_addr + args[0]); tcg_out_movi32(s, COND_AL, TCG_REG_R0, ptr & ~0xfff); tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_R0, ptr & 0xfff); } - s->tb_next_offset[args[0]] = tcg_current_code_size(s); + s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s); break; case INDEX_op_br: tcg_out_goto_label(s, COND_AL, arg_label(args[0])); diff --git a/tcg/i386/tcg-target.inc.c b/tcg/i386/tcg-target.inc.c index 8d242a6665..317484cb5d 100644 --- a/tcg/i386/tcg-target.inc.c +++ b/tcg/i386/tcg-target.inc.c @@ -1790,7 +1790,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_jmp(s, tb_ret_addr); break; case INDEX_op_goto_tb: - if (s->tb_jmp_offset) { + if (s->tb_jmp_insn_offset) { /* direct jump method */ int gap; /* jump displacement must be aligned for atomic patching; @@ -1801,14 +1801,14 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out_nopn(s, gap - 1); } tcg_out8(s, OPC_JMP_long); /* jmp im */ - s->tb_jmp_offset[args[0]] = tcg_current_code_size(s); + s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); tcg_out32(s, 0); } else { /* indirect jump method */ tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1, - (intptr_t)(s->tb_next + args[0])); + (intptr_t)(s->tb_jmp_target_addr + args[0])); } - s->tb_next_offset[args[0]] = tcg_current_code_size(s); + s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s); break; case INDEX_op_br: tcg_out_jxx(s, JCC_JMP, arg_label(args[0]), 0); diff --git a/tcg/ia64/tcg-target.inc.c b/tcg/ia64/tcg-target.inc.c index 7557e6a9d4..395223e340 100644 --- a/tcg/ia64/tcg-target.inc.c +++ b/tcg/ia64/tcg-target.inc.c @@ -881,13 +881,13 @@ static void tcg_out_exit_tb(TCGContext *s, tcg_target_long arg) static inline void tcg_out_goto_tb(TCGContext *s, TCGArg arg) { - if (s->tb_jmp_offset) { + if (s->tb_jmp_insn_offset) { /* direct jump method */ tcg_abort(); } else { /* indirect jump method */ tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, - (tcg_target_long)(s->tb_next + arg)); + (tcg_target_long)(s->tb_jmp_target_addr + arg)); tcg_out_bundle(s, MmI, tcg_opc_m1 (TCG_REG_P0, OPC_LD8_M1, TCG_REG_R2, TCG_REG_R2), @@ -900,7 +900,7 @@ static inline void tcg_out_goto_tb(TCGContext *s, TCGArg arg) tcg_opc_b4 (TCG_REG_P0, OPC_BR_SPTK_MANY_B4, TCG_REG_B6)); } - s->tb_next_offset[arg] = tcg_current_code_size(s); + s->tb_jmp_reset_offset[arg] = tcg_current_code_size(s); } static inline void tcg_out_jmp(TCGContext *s, TCGArg addr) diff --git a/tcg/mips/tcg-target.inc.c b/tcg/mips/tcg-target.inc.c index 1e5a6b41fa..50e98ea63a 100644 --- a/tcg/mips/tcg-target.inc.c +++ b/tcg/mips/tcg-target.inc.c @@ -1397,19 +1397,19 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, } break; case INDEX_op_goto_tb: - if (s->tb_jmp_offset) { + if (s->tb_jmp_insn_offset) { /* direct jump method */ - s->tb_jmp_offset[a0] = tcg_current_code_size(s); + s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); /* Avoid clobbering the address during retranslation. */ tcg_out32(s, OPC_J | (*(uint32_t *)s->code_ptr & 0x3ffffff)); } else { /* indirect jump method */ tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO, - (uintptr_t)(s->tb_next + a0)); + (uintptr_t)(s->tb_jmp_target_addr + a0)); tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0); } tcg_out_nop(s); - s->tb_next_offset[a0] = tcg_current_code_size(s); + s->tb_jmp_reset_offset[a0] = tcg_current_code_size(s); break; case INDEX_op_br: tcg_out_brcond(s, TCG_COND_EQ, TCG_REG_ZERO, TCG_REG_ZERO, diff --git a/tcg/ppc/tcg-target.inc.c b/tcg/ppc/tcg-target.inc.c index 039fa77748..da100528ab 100644 --- a/tcg/ppc/tcg-target.inc.c +++ b/tcg/ppc/tcg-target.inc.c @@ -1902,14 +1902,14 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, tcg_out_b(s, 0, tb_ret_addr); break; case INDEX_op_goto_tb: - tcg_debug_assert(s->tb_jmp_offset); + tcg_debug_assert(s->tb_jmp_insn_offset); /* Direct jump. */ #ifdef __powerpc64__ /* Ensure the next insns are 8-byte aligned. */ if ((uintptr_t)s->code_ptr & 7) { tcg_out32(s, NOP); } - s->tb_jmp_offset[args[0]] = tcg_current_code_size(s); + s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); /* To be replaced by either a branch+nop or a load into TMP1. */ s->code_ptr += 2; tcg_out32(s, MTSPR | RS(TCG_REG_TMP1) | CTR); @@ -1918,7 +1918,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, /* To be replaced by a branch. */ s->code_ptr++; #endif - s->tb_next_offset[args[0]] = tcg_current_code_size(s); + s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s); break; case INDEX_op_br: { diff --git a/tcg/s390/tcg-target.inc.c b/tcg/s390/tcg-target.inc.c index 8e8064ce73..e0a60e618c 100644 --- a/tcg/s390/tcg-target.inc.c +++ b/tcg/s390/tcg-target.inc.c @@ -1717,7 +1717,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_goto_tb: - if (s->tb_jmp_offset) { + if (s->tb_jmp_insn_offset) { /* branch displacement must be aligned for atomic patching; * see if we need to add extra nop before branch */ @@ -1725,15 +1725,16 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, tcg_out16(s, NOP); } tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4)); - s->tb_jmp_offset[args[0]] = tcg_current_code_size(s); + s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); s->code_ptr += 2; } else { - /* load address stored at s->tb_next + args[0] */ - tcg_out_ld_abs(s, TCG_TYPE_PTR, TCG_TMP0, s->tb_next + args[0]); + /* load address stored at s->tb_jmp_target_addr + args[0] */ + tcg_out_ld_abs(s, TCG_TYPE_PTR, TCG_TMP0, + s->tb_jmp_target_addr + args[0]); /* and go there */ tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_TMP0); } - s->tb_next_offset[args[0]] = tcg_current_code_size(s); + s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s); break; OP_32_64(ld8u): diff --git a/tcg/sparc/tcg-target.inc.c b/tcg/sparc/tcg-target.inc.c index c6479e2507..9938a5085e 100644 --- a/tcg/sparc/tcg-target.inc.c +++ b/tcg/sparc/tcg-target.inc.c @@ -1229,18 +1229,19 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, } break; case INDEX_op_goto_tb: - if (s->tb_jmp_offset) { + if (s->tb_jmp_insn_offset) { /* direct jump method */ - s->tb_jmp_offset[a0] = tcg_current_code_size(s); + s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s); /* Make sure to preserve links during retranslation. */ tcg_out32(s, CALL | (*s->code_ptr & ~INSN_OP(-1))); } else { /* indirect jump method */ - tcg_out_ld_ptr(s, TCG_REG_T1, (uintptr_t)(s->tb_next + a0)); + tcg_out_ld_ptr(s, TCG_REG_T1, + (uintptr_t)(s->tb_jmp_target_addr + a0)); tcg_out_arithi(s, TCG_REG_G0, TCG_REG_T1, 0, JMPL); } tcg_out_nop(s); - s->tb_next_offset[a0] = tcg_current_code_size(s); + s->tb_jmp_reset_offset[a0] = tcg_current_code_size(s); break; case INDEX_op_br: tcg_out_bpcc(s, COND_A, BPCC_PT, arg_label(a0)); diff --git a/tcg/tcg.h b/tcg/tcg.h index 01dfebd9a2..0fceef5333 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -510,9 +510,9 @@ struct TCGContext { /* goto_tb support */ tcg_insn_unit *code_buf; - uintptr_t *tb_next; - uint16_t *tb_next_offset; - uint16_t *tb_jmp_offset; /* != NULL if USE_DIRECT_JUMP */ + uint16_t *tb_jmp_reset_offset; /* tb->jmp_reset_offset */ + uint16_t *tb_jmp_insn_offset; /* tb->jmp_insn_offset if USE_DIRECT_JUMP */ + uintptr_t *tb_jmp_target_addr; /* tb->jmp_target_addr if !USE_DIRECT_JUMP */ /* liveness analysis */ uint16_t *op_dead_args; /* for each operation, each bit tells if the diff --git a/tcg/tci/tcg-target.inc.c b/tcg/tci/tcg-target.inc.c index 85eeb5de24..fa74d5278e 100644 --- a/tcg/tci/tcg-target.inc.c +++ b/tcg/tci/tcg-target.inc.c @@ -553,19 +553,19 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, tcg_out64(s, args[0]); break; case INDEX_op_goto_tb: - if (s->tb_jmp_offset) { + if (s->tb_jmp_insn_offset) { /* Direct jump method. */ - tcg_debug_assert(args[0] < ARRAY_SIZE(s->tb_jmp_offset)); + tcg_debug_assert(args[0] < ARRAY_SIZE(s->tb_jmp_insn_offset)); /* Align for atomic patching and thread safety */ s->code_ptr = QEMU_ALIGN_PTR_UP(s->code_ptr, 4); - s->tb_jmp_offset[args[0]] = tcg_current_code_size(s); + s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s); tcg_out32(s, 0); } else { /* Indirect jump method. */ TODO(); } - tcg_debug_assert(args[0] < ARRAY_SIZE(s->tb_next_offset)); - s->tb_next_offset[args[0]] = tcg_current_code_size(s); + tcg_debug_assert(args[0] < ARRAY_SIZE(s->tb_jmp_reset_offset)); + s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s); break; case INDEX_op_br: tci_out_label(s, arg_label(args[0])); diff --git a/translate-all.c b/translate-all.c index 79a515dde7..c6613d13c9 100644 --- a/translate-all.c +++ b/translate-all.c @@ -931,7 +931,7 @@ static inline void tb_jmp_remove(TranslationBlock *tb, int n) TranslationBlock *tb1, **ptb; unsigned int n1; - ptb = &tb->jmp_next[n]; + ptb = &tb->jmp_list_next[n]; tb1 = *ptb; if (tb1) { /* find tb(n) in circular list */ @@ -943,15 +943,15 @@ static inline void tb_jmp_remove(TranslationBlock *tb, int n) break; } if (n1 == 2) { - ptb = &tb1->jmp_first; + ptb = &tb1->jmp_list_first; } else { - ptb = &tb1->jmp_next[n1]; + ptb = &tb1->jmp_list_next[n1]; } } /* now we can suppress tb(n) from the list */ - *ptb = tb->jmp_next[n]; + *ptb = tb->jmp_list_next[n]; - tb->jmp_next[n] = NULL; + tb->jmp_list_next[n] = NULL; } } @@ -959,7 +959,8 @@ static inline void tb_jmp_remove(TranslationBlock *tb, int n) another TB */ static inline void tb_reset_jump(TranslationBlock *tb, int n) { - tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n])); + uintptr_t addr = (uintptr_t)(tb->tc_ptr + tb->jmp_reset_offset[n]); + tb_set_jmp_target(tb, n, addr); } /* invalidate one TB */ @@ -1003,19 +1004,21 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) tb_jmp_remove(tb, 1); /* suppress any remaining jumps to this TB */ - tb1 = tb->jmp_first; + tb1 = tb->jmp_list_first; for (;;) { n1 = (uintptr_t)tb1 & 3; if (n1 == 2) { break; } tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3); - tb2 = tb1->jmp_next[n1]; + tb2 = tb1->jmp_list_next[n1]; tb_reset_jump(tb1, n1); - tb1->jmp_next[n1] = NULL; + tb1->jmp_list_next[n1] = NULL; tb1 = tb2; } - tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */ + + /* fail safe */ + tb->jmp_list_first = (TranslationBlock *)((uintptr_t)tb | 2); tcg_ctx.tb_ctx.tb_phys_invalidate_count++; } @@ -1100,15 +1103,15 @@ TranslationBlock *tb_gen_code(CPUState *cpu, trace_translate_block(tb, tb->pc, tb->tc_ptr); /* generate machine code */ - tb->tb_next_offset[0] = 0xffff; - tb->tb_next_offset[1] = 0xffff; - tcg_ctx.tb_next_offset = tb->tb_next_offset; + tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; + tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; + tcg_ctx.tb_jmp_reset_offset = tb->jmp_reset_offset; #ifdef USE_DIRECT_JUMP - tcg_ctx.tb_jmp_offset = tb->tb_jmp_offset; - tcg_ctx.tb_next = NULL; + tcg_ctx.tb_jmp_insn_offset = tb->jmp_insn_offset; + tcg_ctx.tb_jmp_target_addr = NULL; #else - tcg_ctx.tb_jmp_offset = NULL; - tcg_ctx.tb_next = tb->tb_next; + tcg_ctx.tb_jmp_insn_offset = NULL; + tcg_ctx.tb_jmp_target_addr = tb->jmp_target_addr; #endif #ifdef CONFIG_PROFILER @@ -1489,15 +1492,15 @@ static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, tb->page_addr[1] = -1; } - tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); - tb->jmp_next[0] = NULL; - tb->jmp_next[1] = NULL; + tb->jmp_list_first = (TranslationBlock *)((uintptr_t)tb | 2); + tb->jmp_list_next[0] = NULL; + tb->jmp_list_next[1] = NULL; /* init original jump addresses */ - if (tb->tb_next_offset[0] != 0xffff) { + if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { tb_reset_jump(tb, 0); } - if (tb->tb_next_offset[1] != 0xffff) { + if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { tb_reset_jump(tb, 1); } @@ -1690,9 +1693,9 @@ void dump_exec_info(FILE *f, fprintf_function cpu_fprintf) if (tb->page_addr[1] != -1) { cross_page++; } - if (tb->tb_next_offset[0] != 0xffff) { + if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { direct_jmp_count++; - if (tb->tb_next_offset[1] != 0xffff) { + if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { direct_jmp2_count++; } } From c37e6d7e3589ecb96914faa21025ad7ba6654aea Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Mon, 21 Mar 2016 23:11:00 +0300 Subject: [PATCH 17/39] tcg: Use uintptr_t type for jmp_list_{next|first} fields of TB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These fields do not contain pure pointers to a TranslationBlock structure. So uintptr_t is the most appropriate type for them. Also put some asserts to assure that the two least significant bits of the pointer are always zero before assigning it to jmp_list_first. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Alex Bennée Signed-off-by: Richard Henderson --- include/exec/exec-all.h | 12 +++++++----- translate-all.c | 38 ++++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 445d946d84..64c2a660bc 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -277,14 +277,16 @@ struct TranslationBlock { * jmp_list_first points to the first TB jumping to this one. * jmp_list_next is used to point to the next TB in a list. * Since each TB can have two jumps, it can participate in two lists. - * The two least significant bits of a pointer are used to choose which - * data field holds a pointer to the next TB: + * jmp_list_first and jmp_list_next are 4-byte aligned pointers to a + * TranslationBlock structure, but the two least significant bits of + * them are used to encode which data field of the pointed TB should + * be used to traverse the list further from that TB: * 0 => jmp_list_next[0], 1 => jmp_list_next[1], 2 => jmp_list_first. * In other words, 0/1 tells which jump is used in the pointed TB, * and 2 means that this is a pointer back to the target TB of this list. */ - struct TranslationBlock *jmp_list_next[2]; - struct TranslationBlock *jmp_list_first; + uintptr_t jmp_list_next[2]; + uintptr_t jmp_list_first; }; #include "qemu/thread.h" @@ -382,7 +384,7 @@ static inline void tb_add_jump(TranslationBlock *tb, int n, /* add in TB jmp circular list */ tb->jmp_list_next[n] = tb_next->jmp_list_first; - tb_next->jmp_list_first = (TranslationBlock *)((uintptr_t)tb | n); + tb_next->jmp_list_first = (uintptr_t)tb | n; } } diff --git a/translate-all.c b/translate-all.c index c6613d13c9..2fb16466c1 100644 --- a/translate-all.c +++ b/translate-all.c @@ -928,17 +928,17 @@ static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb) static inline void tb_jmp_remove(TranslationBlock *tb, int n) { - TranslationBlock *tb1, **ptb; + TranslationBlock *tb1; + uintptr_t *ptb, ntb; unsigned int n1; ptb = &tb->jmp_list_next[n]; - tb1 = *ptb; - if (tb1) { + if (*ptb) { /* find tb(n) in circular list */ for (;;) { - tb1 = *ptb; - n1 = (uintptr_t)tb1 & 3; - tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3); + ntb = *ptb; + n1 = ntb & 3; + tb1 = (TranslationBlock *)(ntb & ~3); if (n1 == n && tb1 == tb) { break; } @@ -951,7 +951,7 @@ static inline void tb_jmp_remove(TranslationBlock *tb, int n) /* now we can suppress tb(n) from the list */ *ptb = tb->jmp_list_next[n]; - tb->jmp_list_next[n] = NULL; + tb->jmp_list_next[n] = (uintptr_t)NULL; } } @@ -970,7 +970,7 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) PageDesc *p; unsigned int h, n1; tb_page_addr_t phys_pc; - TranslationBlock *tb1, *tb2; + uintptr_t tb1, tb2; /* remove the TB from the hash list */ phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); @@ -1006,19 +1006,20 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) /* suppress any remaining jumps to this TB */ tb1 = tb->jmp_list_first; for (;;) { - n1 = (uintptr_t)tb1 & 3; + TranslationBlock *tmp_tb; + n1 = tb1 & 3; if (n1 == 2) { break; } - tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3); - tb2 = tb1->jmp_list_next[n1]; - tb_reset_jump(tb1, n1); - tb1->jmp_list_next[n1] = NULL; + tmp_tb = (TranslationBlock *)(tb1 & ~3); + tb2 = tmp_tb->jmp_list_next[n1]; + tb_reset_jump(tmp_tb, n1); + tmp_tb->jmp_list_next[n1] = (uintptr_t)NULL; tb1 = tb2; } - /* fail safe */ - tb->jmp_list_first = (TranslationBlock *)((uintptr_t)tb | 2); + assert(((uintptr_t)tb & 3) == 0); + tb->jmp_list_first = (uintptr_t)tb | 2; /* fail safe */ tcg_ctx.tb_ctx.tb_phys_invalidate_count++; } @@ -1492,9 +1493,10 @@ static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, tb->page_addr[1] = -1; } - tb->jmp_list_first = (TranslationBlock *)((uintptr_t)tb | 2); - tb->jmp_list_next[0] = NULL; - tb->jmp_list_next[1] = NULL; + assert(((uintptr_t)tb & 3) == 0); + tb->jmp_list_first = (uintptr_t)tb | 2; + tb->jmp_list_next[0] = (uintptr_t)NULL; + tb->jmp_list_next[1] = (uintptr_t)NULL; /* init original jump addresses */ if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { From e90d96b158665a684ab89b4f002838034b5fafc8 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Tue, 22 Mar 2016 18:47:54 +0300 Subject: [PATCH 18/39] tcg: Rearrange tb_link_page() to avoid forward declaration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Alex Bennée Signed-off-by: Richard Henderson --- translate-all.c | 204 ++++++++++++++++++++++++------------------------ 1 file changed, 101 insertions(+), 103 deletions(-) diff --git a/translate-all.c b/translate-all.c index 2fb16466c1..4a58af43a4 100644 --- a/translate-all.c +++ b/translate-all.c @@ -153,8 +153,6 @@ void tb_lock_reset(void) #endif } -static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, - tb_page_addr_t phys_page2); static TranslationBlock *tb_find_pc(uintptr_t tc_ptr); void cpu_gen_init(void) @@ -1053,6 +1051,107 @@ static void build_page_bitmap(PageDesc *p) } } +/* add the tb in the target page and protect it if necessary + * + * Called with mmap_lock held for user-mode emulation. + */ +static inline void tb_alloc_page(TranslationBlock *tb, + unsigned int n, tb_page_addr_t page_addr) +{ + PageDesc *p; +#ifndef CONFIG_USER_ONLY + bool page_already_protected; +#endif + + tb->page_addr[n] = page_addr; + p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1); + tb->page_next[n] = p->first_tb; +#ifndef CONFIG_USER_ONLY + page_already_protected = p->first_tb != NULL; +#endif + p->first_tb = (TranslationBlock *)((uintptr_t)tb | n); + invalidate_page_bitmap(p); + +#if defined(CONFIG_USER_ONLY) + if (p->flags & PAGE_WRITE) { + target_ulong addr; + PageDesc *p2; + int prot; + + /* force the host page as non writable (writes will have a + page fault + mprotect overhead) */ + page_addr &= qemu_host_page_mask; + prot = 0; + for (addr = page_addr; addr < page_addr + qemu_host_page_size; + addr += TARGET_PAGE_SIZE) { + + p2 = page_find(addr >> TARGET_PAGE_BITS); + if (!p2) { + continue; + } + prot |= p2->flags; + p2->flags &= ~PAGE_WRITE; + } + mprotect(g2h(page_addr), qemu_host_page_size, + (prot & PAGE_BITS) & ~PAGE_WRITE); +#ifdef DEBUG_TB_INVALIDATE + printf("protecting code page: 0x" TARGET_FMT_lx "\n", + page_addr); +#endif + } +#else + /* if some code is already present, then the pages are already + protected. So we handle the case where only the first TB is + allocated in a physical page */ + if (!page_already_protected) { + tlb_protect_code(page_addr); + } +#endif +} + +/* add a new TB and link it to the physical page tables. phys_page2 is + * (-1) to indicate that only one page contains the TB. + * + * Called with mmap_lock held for user-mode emulation. + */ +static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, + tb_page_addr_t phys_page2) +{ + unsigned int h; + TranslationBlock **ptb; + + /* add in the physical hash table */ + h = tb_phys_hash_func(phys_pc); + ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h]; + tb->phys_hash_next = *ptb; + *ptb = tb; + + /* add in the page list */ + tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK); + if (phys_page2 != -1) { + tb_alloc_page(tb, 1, phys_page2); + } else { + tb->page_addr[1] = -1; + } + + assert(((uintptr_t)tb & 3) == 0); + tb->jmp_list_first = (uintptr_t)tb | 2; + tb->jmp_list_next[0] = (uintptr_t)NULL; + tb->jmp_list_next[1] = (uintptr_t)NULL; + + /* init original jump addresses */ + if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { + tb_reset_jump(tb, 0); + } + if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { + tb_reset_jump(tb, 1); + } + +#ifdef DEBUG_TB_CHECK + tb_page_check(); +#endif +} + /* Called with mmap_lock held for user mode emulation. */ TranslationBlock *tb_gen_code(CPUState *cpu, target_ulong pc, target_ulong cs_base, @@ -1410,107 +1509,6 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr, } #endif -/* add the tb in the target page and protect it if necessary - * - * Called with mmap_lock held for user-mode emulation. - */ -static inline void tb_alloc_page(TranslationBlock *tb, - unsigned int n, tb_page_addr_t page_addr) -{ - PageDesc *p; -#ifndef CONFIG_USER_ONLY - bool page_already_protected; -#endif - - tb->page_addr[n] = page_addr; - p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1); - tb->page_next[n] = p->first_tb; -#ifndef CONFIG_USER_ONLY - page_already_protected = p->first_tb != NULL; -#endif - p->first_tb = (TranslationBlock *)((uintptr_t)tb | n); - invalidate_page_bitmap(p); - -#if defined(CONFIG_USER_ONLY) - if (p->flags & PAGE_WRITE) { - target_ulong addr; - PageDesc *p2; - int prot; - - /* force the host page as non writable (writes will have a - page fault + mprotect overhead) */ - page_addr &= qemu_host_page_mask; - prot = 0; - for (addr = page_addr; addr < page_addr + qemu_host_page_size; - addr += TARGET_PAGE_SIZE) { - - p2 = page_find(addr >> TARGET_PAGE_BITS); - if (!p2) { - continue; - } - prot |= p2->flags; - p2->flags &= ~PAGE_WRITE; - } - mprotect(g2h(page_addr), qemu_host_page_size, - (prot & PAGE_BITS) & ~PAGE_WRITE); -#ifdef DEBUG_TB_INVALIDATE - printf("protecting code page: 0x" TARGET_FMT_lx "\n", - page_addr); -#endif - } -#else - /* if some code is already present, then the pages are already - protected. So we handle the case where only the first TB is - allocated in a physical page */ - if (!page_already_protected) { - tlb_protect_code(page_addr); - } -#endif -} - -/* add a new TB and link it to the physical page tables. phys_page2 is - * (-1) to indicate that only one page contains the TB. - * - * Called with mmap_lock held for user-mode emulation. - */ -static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, - tb_page_addr_t phys_page2) -{ - unsigned int h; - TranslationBlock **ptb; - - /* add in the physical hash table */ - h = tb_phys_hash_func(phys_pc); - ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h]; - tb->phys_hash_next = *ptb; - *ptb = tb; - - /* add in the page list */ - tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK); - if (phys_page2 != -1) { - tb_alloc_page(tb, 1, phys_page2); - } else { - tb->page_addr[1] = -1; - } - - assert(((uintptr_t)tb & 3) == 0); - tb->jmp_list_first = (uintptr_t)tb | 2; - tb->jmp_list_next[0] = (uintptr_t)NULL; - tb->jmp_list_next[1] = (uintptr_t)NULL; - - /* init original jump addresses */ - if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { - tb_reset_jump(tb, 0); - } - if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { - tb_reset_jump(tb, 1); - } - -#ifdef DEBUG_TB_CHECK - tb_page_check(); -#endif -} - /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr < tb[1].tc_ptr. Return NULL if not found */ static TranslationBlock *tb_find_pc(uintptr_t tc_ptr) From 901bc3deb43bf37c85e43955905d003be7ae5fa5 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Tue, 22 Mar 2016 19:00:12 +0300 Subject: [PATCH 19/39] tcg: Init TB's direct jumps before making it visible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Initialize TB's direct jump list data fields and reset the jumps before tb_link_page() puts it into the physical hash table and the physical page list. So TB is completely initialized before it becomes visible. This is pure rearrangement of code to a more suitable place, though it could be a preparation for relaxing the locking scheme in future. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Alex Bennée Signed-off-by: Richard Henderson --- translate-all.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/translate-all.c b/translate-all.c index 4a58af43a4..1dc1a73c1d 100644 --- a/translate-all.c +++ b/translate-all.c @@ -1134,19 +1134,6 @@ static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, tb->page_addr[1] = -1; } - assert(((uintptr_t)tb & 3) == 0); - tb->jmp_list_first = (uintptr_t)tb | 2; - tb->jmp_list_next[0] = (uintptr_t)NULL; - tb->jmp_list_next[1] = (uintptr_t)NULL; - - /* init original jump addresses */ - if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { - tb_reset_jump(tb, 0); - } - if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { - tb_reset_jump(tb, 1); - } - #ifdef DEBUG_TB_CHECK tb_page_check(); #endif @@ -1255,12 +1242,31 @@ TranslationBlock *tb_gen_code(CPUState *cpu, ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, CODE_GEN_ALIGN); + /* init jump list */ + assert(((uintptr_t)tb & 3) == 0); + tb->jmp_list_first = (uintptr_t)tb | 2; + tb->jmp_list_next[0] = (uintptr_t)NULL; + tb->jmp_list_next[1] = (uintptr_t)NULL; + + /* init original jump addresses wich has been set during tcg_gen_code() */ + if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { + tb_reset_jump(tb, 0); + } + if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { + tb_reset_jump(tb, 1); + } + /* check next page if needed */ virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; phys_page2 = -1; if ((pc & TARGET_PAGE_MASK) != virt_page2) { phys_page2 = get_page_addr_code(env, virt_page2); } + /* As long as consistency of the TB stuff is provided by tb_lock in user + * mode and is implicit in single-threaded softmmu emulation, no explicit + * memory barrier is required before tb_link_page() makes the TB visible + * through the physical hash table and physical page list. + */ tb_link_page(tb, phys_pc, phys_page2); return tb; } From 9962c478b153a18fe88a6509fe58cd178aff8abc Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 20 Apr 2016 23:15:09 +0300 Subject: [PATCH 20/39] tcg: Clarify thread safety check in tb_add_jump() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The check is to make sure that another thread hasn't already done the same while we were outside of tb_lock. Mention this in a comment. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Alex Bennée Signed-off-by: Richard Henderson --- include/exec/exec-all.h | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 64c2a660bc..06da1bcc45 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -372,20 +372,23 @@ static inline void tb_set_jmp_target(TranslationBlock *tb, static inline void tb_add_jump(TranslationBlock *tb, int n, TranslationBlock *tb_next) { - /* NOTE: this test is only needed for thread safety */ - if (!tb->jmp_list_next[n]) { - qemu_log_mask_and_addr(CPU_LOG_EXEC, tb->pc, - "Linking TBs %p [" TARGET_FMT_lx - "] index %d -> %p [" TARGET_FMT_lx "]\n", - tb->tc_ptr, tb->pc, n, - tb_next->tc_ptr, tb_next->pc); - /* patch the native jump address */ - tb_set_jmp_target(tb, n, (uintptr_t)tb_next->tc_ptr); - - /* add in TB jmp circular list */ - tb->jmp_list_next[n] = tb_next->jmp_list_first; - tb_next->jmp_list_first = (uintptr_t)tb | n; + if (tb->jmp_list_next[n]) { + /* Another thread has already done this while we were + * outside of the lock; nothing to do in this case */ + return; } + qemu_log_mask_and_addr(CPU_LOG_EXEC, tb->pc, + "Linking TBs %p [" TARGET_FMT_lx + "] index %d -> %p [" TARGET_FMT_lx "]\n", + tb->tc_ptr, tb->pc, n, + tb_next->tc_ptr, tb_next->pc); + + /* patch the native jump address */ + tb_set_jmp_target(tb, n, (uintptr_t)tb_next->tc_ptr); + + /* add in TB jmp circular list */ + tb->jmp_list_next[n] = tb_next->jmp_list_first; + tb_next->jmp_list_first = (uintptr_t)tb | n; } /* GETRA is the true target of the return instruction that we'll execute, From 133626783aa5a1bf86332fa3e6f7b8efe005f924 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 23 Mar 2016 18:30:16 +0300 Subject: [PATCH 21/39] tcg: Rename tb_jmp_remove() to tb_remove_from_jmp_list() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tb_jmp_remove() was only used to remove the TB from a list of all TBs jumping to the same TB which is n-th jump destination of the given TB. Put a comment briefly describing the function behavior and rename it to better reflect its purpose. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Alex Bennée Signed-off-by: Richard Henderson --- translate-all.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/translate-all.c b/translate-all.c index 1dc1a73c1d..5e057baaa3 100644 --- a/translate-all.c +++ b/translate-all.c @@ -924,7 +924,8 @@ static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb) } } -static inline void tb_jmp_remove(TranslationBlock *tb, int n) +/* remove the TB from a list of TBs jumping to the n-th jump target of the TB */ +static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n) { TranslationBlock *tb1; uintptr_t *ptb, ntb; @@ -998,8 +999,8 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) } /* suppress this TB from the two jump lists */ - tb_jmp_remove(tb, 0); - tb_jmp_remove(tb, 1); + tb_remove_from_jmp_list(tb, 0); + tb_remove_from_jmp_list(tb, 1); /* suppress any remaining jumps to this TB */ tb1 = tb->jmp_list_first; From 89bba496322d4cf996d42cdd4bb0912231656c3d Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 23 Mar 2016 18:36:31 +0300 Subject: [PATCH 22/39] tcg: Extract removing of jumps to TB from tb_phys_invalidate() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the code for removing jumps to a TB out of tb_phys_invalidate() to a separate static inline function tb_jmp_unlink(). This simplifies tb_phys_invalidate() and improves code structure. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Alex Bennée Signed-off-by: Richard Henderson --- translate-all.c | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/translate-all.c b/translate-all.c index 5e057baaa3..9a57aaba3d 100644 --- a/translate-all.c +++ b/translate-all.c @@ -962,14 +962,37 @@ static inline void tb_reset_jump(TranslationBlock *tb, int n) tb_set_jmp_target(tb, n, addr); } +/* remove any jumps to the TB */ +static inline void tb_jmp_unlink(TranslationBlock *tb) +{ + uintptr_t tb1, tb2; + unsigned int n1; + + tb1 = tb->jmp_list_first; + for (;;) { + TranslationBlock *tmp_tb; + n1 = tb1 & 3; + if (n1 == 2) { + break; + } + tmp_tb = (TranslationBlock *)(tb1 & ~3); + tb2 = tmp_tb->jmp_list_next[n1]; + tb_reset_jump(tmp_tb, n1); + tmp_tb->jmp_list_next[n1] = (uintptr_t)NULL; + tb1 = tb2; + } + + assert(((uintptr_t)tb & 3) == 0); + tb->jmp_list_first = (uintptr_t)tb | 2; /* fail safe */ +} + /* invalidate one TB */ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) { CPUState *cpu; PageDesc *p; - unsigned int h, n1; + unsigned int h; tb_page_addr_t phys_pc; - uintptr_t tb1, tb2; /* remove the TB from the hash list */ phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); @@ -1003,22 +1026,7 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) tb_remove_from_jmp_list(tb, 1); /* suppress any remaining jumps to this TB */ - tb1 = tb->jmp_list_first; - for (;;) { - TranslationBlock *tmp_tb; - n1 = tb1 & 3; - if (n1 == 2) { - break; - } - tmp_tb = (TranslationBlock *)(tb1 & ~3); - tb2 = tmp_tb->jmp_list_next[n1]; - tb_reset_jump(tmp_tb, n1); - tmp_tb->jmp_list_next[n1] = (uintptr_t)NULL; - tb1 = tb2; - } - - assert(((uintptr_t)tb & 3) == 0); - tb->jmp_list_first = (uintptr_t)tb | 2; /* fail safe */ + tb_jmp_unlink(tb); tcg_ctx.tb_ctx.tb_phys_invalidate_count++; } From f9c5b66f487a04d3747dc6997b1503f9258df945 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 23 Mar 2016 21:47:33 +0300 Subject: [PATCH 23/39] tcg: Clean up tb_jmp_unlink() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unify the code of this function with tb_jmp_remove_from_list(). Making these functions similar improves their readability. Also this could be a step towards making this function thread-safe. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Alex Bennée Signed-off-by: Richard Henderson --- translate-all.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/translate-all.c b/translate-all.c index 9a57aaba3d..d679ad129e 100644 --- a/translate-all.c +++ b/translate-all.c @@ -965,25 +965,22 @@ static inline void tb_reset_jump(TranslationBlock *tb, int n) /* remove any jumps to the TB */ static inline void tb_jmp_unlink(TranslationBlock *tb) { - uintptr_t tb1, tb2; + TranslationBlock *tb1; + uintptr_t *ptb, ntb; unsigned int n1; - tb1 = tb->jmp_list_first; + ptb = &tb->jmp_list_first; for (;;) { - TranslationBlock *tmp_tb; - n1 = tb1 & 3; + ntb = *ptb; + n1 = ntb & 3; + tb1 = (TranslationBlock *)(ntb & ~3); if (n1 == 2) { break; } - tmp_tb = (TranslationBlock *)(tb1 & ~3); - tb2 = tmp_tb->jmp_list_next[n1]; - tb_reset_jump(tmp_tb, n1); - tmp_tb->jmp_list_next[n1] = (uintptr_t)NULL; - tb1 = tb2; + tb_reset_jump(tb1, n1); + *ptb = tb1->jmp_list_next[n1]; + tb1->jmp_list_next[n1] = (uintptr_t)NULL; } - - assert(((uintptr_t)tb & 3) == 0); - tb->jmp_list_first = (uintptr_t)tb | 2; /* fail safe */ } /* invalidate one TB */ From 5b053a4a28278bca606eeff7d1c0730df1b047e9 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Fri, 8 Apr 2016 19:48:12 +0300 Subject: [PATCH 24/39] tcg: Clean up direct block chaining safety checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don't take care of direct jumps when address mapping changes. Thus we must be sure to generate direct jumps so that they always keep valid even if address mapping changes. Luckily, we can only allow to execute a TB if it was generated from the pages which match with current mapping. Document tcg_gen_goto_tb() declaration and note the reason for destination PC limitations. Some targets with variable length instructions allow TB to straddle a page boundary. However, we make sure that both of TB pages match the current address mapping when looking up TBs. So it is safe to do direct jumps into the both pages. Correct the checks for some of those targets. Given that, we can safely patch a TB which spans two pages. Remove the unnecessary check in cpu_exec() and allow such TBs to be patched. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Alex Bennée Signed-off-by: Richard Henderson --- cpu-exec.c | 7 ++----- target-arm/translate.c | 3 ++- target-cris/translate.c | 4 +++- target-i386/translate.c | 2 +- target-m68k/translate.c | 2 +- target-s390x/translate.c | 2 +- tcg/tcg-op.h | 10 ++++++++++ 7 files changed, 20 insertions(+), 10 deletions(-) diff --git a/cpu-exec.c b/cpu-exec.c index debc65ca69..f984dc71cb 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -508,11 +508,8 @@ int cpu_exec(CPUState *cpu) next_tb = 0; tcg_ctx.tb_ctx.tb_invalidated_flag = 0; } - /* see if we can patch the calling TB. When the TB - spans two pages, we cannot safely do a direct - jump. */ - if (next_tb != 0 && tb->page_addr[1] == -1 - && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { + /* See if we can patch the calling TB. */ + if (next_tb != 0 && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { tb_add_jump((TranslationBlock *)(next_tb & ~TB_EXIT_MASK), next_tb & TB_EXIT_MASK, tb); } diff --git a/target-arm/translate.c b/target-arm/translate.c index 940ec8d981..34196a8217 100644 --- a/target-arm/translate.c +++ b/target-arm/translate.c @@ -4054,7 +4054,8 @@ static inline void gen_goto_tb(DisasContext *s, int n, target_ulong dest) TranslationBlock *tb; tb = s->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { + if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || + ((s->pc - 1) & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { tcg_gen_goto_tb(n); gen_set_pc_im(s, dest); tcg_gen_exit_tb((uintptr_t)tb + n); diff --git a/target-cris/translate.c b/target-cris/translate.c index a73176c118..9c8ff8f230 100644 --- a/target-cris/translate.c +++ b/target-cris/translate.c @@ -524,7 +524,9 @@ static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest) { TranslationBlock *tb; tb = dc->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { + + if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || + (dc->ppc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { tcg_gen_goto_tb(n); tcg_gen_movi_tl(env_pc, dest); tcg_gen_exit_tb((uintptr_t)tb + n); diff --git a/target-i386/translate.c b/target-i386/translate.c index 3a32f65f8d..058d85a1ab 100644 --- a/target-i386/translate.c +++ b/target-i386/translate.c @@ -2094,7 +2094,7 @@ static inline void gen_goto_tb(DisasContext *s, int tb_num, target_ulong eip) tb = s->tb; /* NOTE: we handle the case where the TB spans two pages here */ if ((pc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) || - (pc & TARGET_PAGE_MASK) == ((s->pc - 1) & TARGET_PAGE_MASK)) { + (pc & TARGET_PAGE_MASK) == (s->pc_start & TARGET_PAGE_MASK)) { /* jump to same page: we can use a direct jump */ tcg_gen_goto_tb(tb_num); gen_jmp_im(eip); diff --git a/target-m68k/translate.c b/target-m68k/translate.c index 7560c3a808..e2ce6c615e 100644 --- a/target-m68k/translate.c +++ b/target-m68k/translate.c @@ -861,7 +861,7 @@ static void gen_jmp_tb(DisasContext *s, int n, uint32_t dest) if (unlikely(s->singlestep_enabled)) { gen_exception(s, dest, EXCP_DEBUG); } else if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || - (s->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { + (s->insn_pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { tcg_gen_goto_tb(n); tcg_gen_movi_i32(QREG_PC, dest); tcg_gen_exit_tb((uintptr_t)tb + n); diff --git a/target-s390x/translate.c b/target-s390x/translate.c index c871ef2bb3..c5179fe05d 100644 --- a/target-s390x/translate.c +++ b/target-s390x/translate.c @@ -610,7 +610,7 @@ static int use_goto_tb(DisasContext *s, uint64_t dest) { /* NOTE: we handle the case where the TB spans two pages here */ return (((dest & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) - || (dest & TARGET_PAGE_MASK) == ((s->pc - 1) & TARGET_PAGE_MASK)) + || (dest & TARGET_PAGE_MASK) == (s->pc & TARGET_PAGE_MASK)) && !s->singlestep_enabled && !(s->tb->cflags & CF_LAST_IO) && !(s->tb->flags & FLAG_MASK_PER)); diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h index c446d3dc72..ace39619ef 100644 --- a/tcg/tcg-op.h +++ b/tcg/tcg-op.h @@ -753,6 +753,16 @@ static inline void tcg_gen_exit_tb(uintptr_t val) tcg_gen_op1i(INDEX_op_exit_tb, val); } +/** + * tcg_gen_goto_tb() - output goto_tb TCG operation + * @idx: Direct jump slot index (0 or 1) + * + * See tcg/README for more info about this TCG operation. + * + * NOTE: Direct jumps with goto_tb are only safe within the pages this TB + * resides in because we don't take care of direct jumps when address mapping + * changes, e.g. in tlb_flush(). + */ void tcg_gen_goto_tb(unsigned idx); #if TARGET_LONG_BITS == 32 From 90aa39a1cc4837360889f0e033ca25cc82100308 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Sat, 9 Apr 2016 01:00:23 +0300 Subject: [PATCH 25/39] tcg: Allow goto_tb to any target PC in user mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In user mode, there's only a static address translation, TBs are always invalidated properly and direct jumps are reset when mapping change. Thus the destination address is always valid for direct jumps and there's no need to restrict it to the pages the TB resides in. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Cc: Riku Voipio Cc: Blue Swirl Reviewed-by: Alex Bennée Signed-off-by: Richard Henderson --- target-alpha/translate.c | 4 ++++ target-arm/translate-a64.c | 2 ++ target-arm/translate.c | 18 ++++++++++++------ target-cris/translate.c | 18 ++++++++++++------ target-i386/translate.c | 21 +++++++++++++-------- target-lm32/translate.c | 21 +++++++++++++++------ target-m68k/translate.c | 18 ++++++++++++------ target-microblaze/translate.c | 15 +++++++++++---- target-mips/translate.c | 20 +++++++++++++++----- target-moxie/translate.c | 21 +++++++++++++++------ target-openrisc/translate.c | 20 +++++++++++++++----- target-ppc/translate.c | 20 +++++++++++++++----- target-s390x/translate.c | 17 +++++++++++------ target-sh4/translate.c | 21 +++++++++++++++------ target-sparc/translate.c | 24 +++++++++++++++++------- target-tricore/translate.c | 20 +++++++++++++++----- target-unicore32/translate.c | 16 +++++++++++----- target-xtensa/translate.c | 4 ++++ tcg/tcg-op.h | 9 ++++++--- 19 files changed, 220 insertions(+), 89 deletions(-) diff --git a/target-alpha/translate.c b/target-alpha/translate.c index 5b86992dd3..8c2183a418 100644 --- a/target-alpha/translate.c +++ b/target-alpha/translate.c @@ -460,12 +460,16 @@ static bool use_goto_tb(DisasContext *ctx, uint64_t dest) || ctx->singlestep_enabled || singlestep) { return false; } +#ifndef CONFIG_USER_ONLY /* If the destination is in the superpage, the page perms can't change. */ if (in_superpage(ctx, dest)) { return true; } /* Check for the dest on the same page as the start of the TB. */ return ((ctx->tb->pc ^ dest) & TARGET_PAGE_MASK) == 0; +#else + return true; +#endif } static ExitStatus gen_bdirect(DisasContext *ctx, int ra, int32_t disp) diff --git a/target-arm/translate-a64.c b/target-arm/translate-a64.c index 24f5e177dd..5526bbda2c 100644 --- a/target-arm/translate-a64.c +++ b/target-arm/translate-a64.c @@ -274,10 +274,12 @@ static inline bool use_goto_tb(DisasContext *s, int n, uint64_t dest) return false; } +#ifndef CONFIG_USER_ONLY /* Only link tbs from inside the same guest page */ if ((s->tb->pc & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) { return false; } +#endif return true; } diff --git a/target-arm/translate.c b/target-arm/translate.c index 34196a8217..a43b1f61cf 100644 --- a/target-arm/translate.c +++ b/target-arm/translate.c @@ -4049,16 +4049,22 @@ static int disas_vfp_insn(DisasContext *s, uint32_t insn) return 0; } +static inline bool use_goto_tb(DisasContext *s, target_ulong dest) +{ +#ifndef CONFIG_USER_ONLY + return (s->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || + ((s->pc - 1) & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(DisasContext *s, int n, target_ulong dest) { - TranslationBlock *tb; - - tb = s->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || - ((s->pc - 1) & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { + if (use_goto_tb(s, dest)) { tcg_gen_goto_tb(n); gen_set_pc_im(s, dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)s->tb + n); } else { gen_set_pc_im(s, dest); tcg_gen_exit_tb(0); diff --git a/target-cris/translate.c b/target-cris/translate.c index 9c8ff8f230..f28b1999a7 100644 --- a/target-cris/translate.c +++ b/target-cris/translate.c @@ -520,16 +520,22 @@ static void t_gen_cc_jmp(TCGv pc_true, TCGv pc_false) gen_set_label(l1); } +static inline bool use_goto_tb(DisasContext *dc, target_ulong dest) +{ +#ifndef CONFIG_USER_ONLY + return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || + (dc->ppc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest) { - TranslationBlock *tb; - tb = dc->tb; - - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || - (dc->ppc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { + if (use_goto_tb(dc, dest)) { tcg_gen_goto_tb(n); tcg_gen_movi_tl(env_pc, dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)dc->tb + n); } else { tcg_gen_movi_tl(env_pc, dest); tcg_gen_exit_tb(0); diff --git a/target-i386/translate.c b/target-i386/translate.c index 058d85a1ab..868c26244b 100644 --- a/target-i386/translate.c +++ b/target-i386/translate.c @@ -2085,20 +2085,25 @@ static inline int insn_const_size(TCGMemOp ot) } } +static inline bool use_goto_tb(DisasContext *s, target_ulong pc) +{ +#ifndef CONFIG_USER_ONLY + return (pc & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) || + (pc & TARGET_PAGE_MASK) == (s->pc_start & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(DisasContext *s, int tb_num, target_ulong eip) { - TranslationBlock *tb; - target_ulong pc; + target_ulong pc = s->cs_base + eip; - pc = s->cs_base + eip; - tb = s->tb; - /* NOTE: we handle the case where the TB spans two pages here */ - if ((pc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) || - (pc & TARGET_PAGE_MASK) == (s->pc_start & TARGET_PAGE_MASK)) { + if (use_goto_tb(s, pc)) { /* jump to same page: we can use a direct jump */ tcg_gen_goto_tb(tb_num); gen_jmp_im(eip); - tcg_gen_exit_tb((uintptr_t)tb + tb_num); + tcg_gen_exit_tb((uintptr_t)s->tb + tb_num); } else { /* jump to another page: currently not optimized */ gen_jmp_im(eip); diff --git a/target-lm32/translate.c b/target-lm32/translate.c index 256a51f849..dd972f5b8c 100644 --- a/target-lm32/translate.c +++ b/target-lm32/translate.c @@ -133,16 +133,25 @@ static inline void t_gen_illegal_insn(DisasContext *dc) gen_helper_ill(cpu_env); } +static inline bool use_goto_tb(DisasContext *dc, target_ulong dest) +{ + if (unlikely(dc->singlestep_enabled)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest) { - TranslationBlock *tb; - - tb = dc->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - likely(!dc->singlestep_enabled)) { + if (use_goto_tb(dc, dest)) { tcg_gen_goto_tb(n); tcg_gen_movi_tl(cpu_pc, dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)dc->tb + n); } else { tcg_gen_movi_tl(cpu_pc, dest); if (dc->singlestep_enabled) { diff --git a/target-m68k/translate.c b/target-m68k/translate.c index e2ce6c615e..e46356e44c 100644 --- a/target-m68k/translate.c +++ b/target-m68k/translate.c @@ -852,19 +852,25 @@ static inline void gen_addr_fault(DisasContext *s) } \ } while (0) +static inline bool use_goto_tb(DisasContext *s, uint32_t dest) +{ +#ifndef CONFIG_USER_ONLY + return (s->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || + (s->insn_pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + /* Generate a jump to an immediate address. */ static void gen_jmp_tb(DisasContext *s, int n, uint32_t dest) { - TranslationBlock *tb; - - tb = s->tb; if (unlikely(s->singlestep_enabled)) { gen_exception(s, dest, EXCP_DEBUG); - } else if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) || - (s->insn_pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { + } else if (use_goto_tb(s, dest)) { tcg_gen_goto_tb(n); tcg_gen_movi_i32(QREG_PC, dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)s->tb + n); } else { gen_jmp_im(s, dest); tcg_gen_exit_tb(0); diff --git a/target-microblaze/translate.c b/target-microblaze/translate.c index f944965a14..a7a8ac8f99 100644 --- a/target-microblaze/translate.c +++ b/target-microblaze/translate.c @@ -124,14 +124,21 @@ static inline void t_gen_raise_exception(DisasContext *dc, uint32_t index) dc->is_jmp = DISAS_UPDATE; } +static inline bool use_goto_tb(DisasContext *dc, target_ulong dest) +{ +#ifndef CONFIG_USER_ONLY + return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest) { - TranslationBlock *tb; - tb = dc->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { + if (use_goto_tb(dc, dest)) { tcg_gen_goto_tb(n); tcg_gen_movi_tl(cpu_SR[SR_PC], dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)dc->tb + n); } else { tcg_gen_movi_tl(cpu_SR[SR_PC], dest); tcg_gen_exit_tb(0); diff --git a/target-mips/translate.c b/target-mips/translate.c index a3a05ec66d..ddfb9244d7 100644 --- a/target-mips/translate.c +++ b/target-mips/translate.c @@ -4191,15 +4191,25 @@ static void gen_trap (DisasContext *ctx, uint32_t opc, tcg_temp_free(t1); } +static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest) +{ + if (unlikely(ctx->singlestep_enabled)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest) { - TranslationBlock *tb; - tb = ctx->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - likely(!ctx->singlestep_enabled)) { + if (use_goto_tb(ctx, dest)) { tcg_gen_goto_tb(n); gen_save_pc(dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)ctx->tb + n); } else { gen_save_pc(dest); if (ctx->singlestep_enabled) { diff --git a/target-moxie/translate.c b/target-moxie/translate.c index a437e2ab60..58200c25d3 100644 --- a/target-moxie/translate.c +++ b/target-moxie/translate.c @@ -121,17 +121,26 @@ void moxie_translate_init(void) done_init = 1; } +static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest) +{ + if (unlikely(ctx->singlestep_enabled)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(CPUMoxieState *env, DisasContext *ctx, int n, target_ulong dest) { - TranslationBlock *tb; - tb = ctx->tb; - - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - !ctx->singlestep_enabled) { + if (use_goto_tb(ctx, dest)) { tcg_gen_goto_tb(n); tcg_gen_movi_i32(cpu_pc, dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)ctx->tb + n); } else { tcg_gen_movi_i32(cpu_pc, dest); if (ctx->singlestep_enabled) { diff --git a/target-openrisc/translate.c b/target-openrisc/translate.c index 5d0ab442a8..d4f1f260e4 100644 --- a/target-openrisc/translate.c +++ b/target-openrisc/translate.c @@ -190,15 +190,25 @@ static void check_ov64s(DisasContext *dc) } #endif*/ +static inline bool use_goto_tb(DisasContext *dc, target_ulong dest) +{ + if (unlikely(dc->singlestep_enabled)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest) { - TranslationBlock *tb; - tb = dc->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - likely(!dc->singlestep_enabled)) { + if (use_goto_tb(dc, dest)) { tcg_gen_movi_tl(cpu_pc, dest); tcg_gen_goto_tb(n); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)dc->tb + n); } else { tcg_gen_movi_tl(cpu_pc, dest); if (dc->singlestep_enabled) { diff --git a/target-ppc/translate.c b/target-ppc/translate.c index b3860ecdea..d485d7c7cb 100644 --- a/target-ppc/translate.c +++ b/target-ppc/translate.c @@ -3822,19 +3822,29 @@ static inline void gen_update_cfar(DisasContext *ctx, target_ulong nip) #endif } +static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest) +{ + if (unlikely(ctx->singlestep_enabled)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + /*** Branch ***/ static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest) { - TranslationBlock *tb; - tb = ctx->tb; if (NARROW_MODE(ctx)) { dest = (uint32_t) dest; } - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - likely(!ctx->singlestep_enabled)) { + if (use_goto_tb(ctx, dest)) { tcg_gen_goto_tb(n); tcg_gen_movi_tl(cpu_nip, dest & ~3); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)ctx->tb + n); } else { tcg_gen_movi_tl(cpu_nip, dest & ~3); if (unlikely(ctx->singlestep_enabled)) { diff --git a/target-s390x/translate.c b/target-s390x/translate.c index c5179fe05d..e99eb5cb01 100644 --- a/target-s390x/translate.c +++ b/target-s390x/translate.c @@ -608,12 +608,17 @@ static void gen_op_calc_cc(DisasContext *s) static int use_goto_tb(DisasContext *s, uint64_t dest) { - /* NOTE: we handle the case where the TB spans two pages here */ - return (((dest & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) - || (dest & TARGET_PAGE_MASK) == (s->pc & TARGET_PAGE_MASK)) - && !s->singlestep_enabled - && !(s->tb->cflags & CF_LAST_IO) - && !(s->tb->flags & FLAG_MASK_PER)); + if (unlikely(s->singlestep_enabled) || + (s->tb->cflags & CF_LAST_IO) || + (s->tb->flags & FLAG_MASK_PER)) { + return false; + } +#ifndef CONFIG_USER_ONLY + return (dest & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) || + (dest & TARGET_PAGE_MASK) == (s->pc & TARGET_PAGE_MASK); +#else + return true; +#endif } static void account_noninline_branch(DisasContext *s, int cc_op) diff --git a/target-sh4/translate.c b/target-sh4/translate.c index 7c189680a7..53f782c054 100644 --- a/target-sh4/translate.c +++ b/target-sh4/translate.c @@ -205,17 +205,26 @@ static void gen_write_sr(TCGv src) tcg_gen_andi_i32(cpu_sr_t, cpu_sr_t, 1); } -static void gen_goto_tb(DisasContext * ctx, int n, target_ulong dest) +static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest) { - TranslationBlock *tb; - tb = ctx->tb; + if (unlikely(ctx->singlestep_enabled)) { + return false; + } - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - !ctx->singlestep_enabled) { +#ifndef CONFIG_USER_ONLY + return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + +static void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest) +{ + if (use_goto_tb(ctx, dest)) { /* Use a direct jump if in same page and singlestep not enabled */ tcg_gen_goto_tb(n); tcg_gen_movi_i32(cpu_pc, dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)ctx->tb + n); } else { tcg_gen_movi_i32(cpu_pc, dest); if (ctx->singlestep_enabled) diff --git a/target-sparc/translate.c b/target-sparc/translate.c index 7998ff57bf..d154e3f7b6 100644 --- a/target-sparc/translate.c +++ b/target-sparc/translate.c @@ -303,20 +303,30 @@ static inline TCGv gen_dest_gpr(DisasContext *dc, int reg) } } +static inline bool use_goto_tb(DisasContext *s, target_ulong pc, + target_ulong npc) +{ + if (unlikely(s->singlestep)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (pc & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) && + (npc & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(DisasContext *s, int tb_num, target_ulong pc, target_ulong npc) { - TranslationBlock *tb; - - tb = s->tb; - if ((pc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) && - (npc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) && - !s->singlestep) { + if (use_goto_tb(s, pc, npc)) { /* jump to same page: we can use a direct jump */ tcg_gen_goto_tb(tb_num); tcg_gen_movi_tl(cpu_pc, pc); tcg_gen_movi_tl(cpu_npc, npc); - tcg_gen_exit_tb((uintptr_t)tb + tb_num); + tcg_gen_exit_tb((uintptr_t)s->tb + tb_num); } else { /* jump to another page: currently not optimized */ tcg_gen_movi_tl(cpu_pc, pc); diff --git a/target-tricore/translate.c b/target-tricore/translate.c index 912bf226be..0237e7bea8 100644 --- a/target-tricore/translate.c +++ b/target-tricore/translate.c @@ -3236,15 +3236,25 @@ static inline void gen_save_pc(target_ulong pc) tcg_gen_movi_tl(cpu_PC, pc); } +static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest) +{ + if (unlikely(ctx->singlestep_enabled)) { + return false; + } + +#ifndef CONFIG_USER_ONLY + return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest) { - TranslationBlock *tb; - tb = ctx->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) && - likely(!ctx->singlestep_enabled)) { + if (use_goto_tb(ctx, dest)) { tcg_gen_goto_tb(n); gen_save_pc(dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)ctx->tb + n); } else { gen_save_pc(dest); if (ctx->singlestep_enabled) { diff --git a/target-unicore32/translate.c b/target-unicore32/translate.c index 39af3af05f..307f7b2059 100644 --- a/target-unicore32/translate.c +++ b/target-unicore32/translate.c @@ -1089,15 +1089,21 @@ static void disas_ucf64_insn(CPUUniCore32State *env, DisasContext *s, uint32_t i } } +static inline bool use_goto_tb(DisasContext *s, uint32_t dest) +{ +#ifndef CONFIG_USER_ONLY + return (s->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK); +#else + return true; +#endif +} + static inline void gen_goto_tb(DisasContext *s, int n, uint32_t dest) { - TranslationBlock *tb; - - tb = s->tb; - if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) { + if (use_goto_tb(s, dest)) { tcg_gen_goto_tb(n); gen_set_pc_im(dest); - tcg_gen_exit_tb((uintptr_t)tb + n); + tcg_gen_exit_tb((uintptr_t)s->tb + n); } else { gen_set_pc_im(dest); tcg_gen_exit_tb(0); diff --git a/target-xtensa/translate.c b/target-xtensa/translate.c index 9894488469..9eac56e2a5 100644 --- a/target-xtensa/translate.c +++ b/target-xtensa/translate.c @@ -418,9 +418,11 @@ static void gen_jump(DisasContext *dc, TCGv dest) static void gen_jumpi(DisasContext *dc, uint32_t dest, int slot) { TCGv_i32 tmp = tcg_const_i32(dest); +#ifndef CONFIG_USER_ONLY if (((dc->tb->pc ^ dest) & TARGET_PAGE_MASK) != 0) { slot = -1; } +#endif gen_jump_slot(dc, tmp, slot); tcg_temp_free(tmp); } @@ -446,9 +448,11 @@ static void gen_callw(DisasContext *dc, int callinc, TCGv_i32 dest) static void gen_callwi(DisasContext *dc, int callinc, uint32_t dest, int slot) { TCGv_i32 tmp = tcg_const_i32(dest); +#ifndef CONFIG_USER_ONLY if (((dc->tb->pc ^ dest) & TARGET_PAGE_MASK) != 0) { slot = -1; } +#endif gen_callw_slot(dc, callinc, tmp, slot); tcg_temp_free(tmp); } diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h index ace39619ef..f217e80747 100644 --- a/tcg/tcg-op.h +++ b/tcg/tcg-op.h @@ -759,9 +759,12 @@ static inline void tcg_gen_exit_tb(uintptr_t val) * * See tcg/README for more info about this TCG operation. * - * NOTE: Direct jumps with goto_tb are only safe within the pages this TB - * resides in because we don't take care of direct jumps when address mapping - * changes, e.g. in tlb_flush(). + * NOTE: In softmmu emulation, direct jumps with goto_tb are only safe within + * the pages this TB resides in because we don't take care of direct jumps when + * address mapping changes, e.g. in tlb_flush(). In user mode, there's only a + * static address translation, so the destination address is always valid, TBs + * are always invalidated properly, and direct jumps are reset when mapping + * changes. */ void tcg_gen_goto_tb(unsigned idx); From 6fad459c91e8a1dedbb6681d3f57ede5222a225c Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 11 Aug 2015 12:42:55 +0200 Subject: [PATCH 26/39] tcg: code_bitmap and code_write_count are not used by user-mode emulation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paolo Bonzini [Sergey Fedorov: eliminate the field entirely in user-mode] Signed-off-by: Sergey Fedorov Reviewed-by: Richard Henderson Reviewed-by: Alex Bennée [rth: merged followup fixup] Message-Id: <1462982777-4513-1-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- translate-all.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/translate-all.c b/translate-all.c index d679ad129e..d5d2bbed3a 100644 --- a/translate-all.c +++ b/translate-all.c @@ -72,11 +72,12 @@ typedef struct PageDesc { /* list of TBs intersecting this ram page */ TranslationBlock *first_tb; +#ifdef CONFIG_SOFTMMU /* in order to optimize self modifying code, we count the number of lookups we do to a given page to use a bitmap */ unsigned int code_write_count; unsigned long *code_bitmap; -#if defined(CONFIG_USER_ONLY) +#else unsigned long flags; #endif } PageDesc; @@ -783,9 +784,11 @@ void tb_free(TranslationBlock *tb) static inline void invalidate_page_bitmap(PageDesc *p) { +#ifdef CONFIG_SOFTMMU g_free(p->code_bitmap); p->code_bitmap = NULL; p->code_write_count = 0; +#endif } /* Set to NULL all the 'first_tb' fields in all PageDescs. */ @@ -1028,6 +1031,7 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) tcg_ctx.tb_ctx.tb_phys_invalidate_count++; } +#ifdef CONFIG_SOFTMMU static void build_page_bitmap(PageDesc *p) { int n, tb_start, tb_end; @@ -1056,6 +1060,7 @@ static void build_page_bitmap(PageDesc *p) tb = tb->page_next[n]; } } +#endif /* add the tb in the target page and protect it if necessary * @@ -1412,6 +1417,7 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, #endif } +#ifdef CONFIG_SOFTMMU /* len must be <= 8 and start must be a multiple of len */ void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len) { @@ -1449,8 +1455,7 @@ void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len) tb_invalidate_phys_page_range(start, start + len, 1); } } - -#if !defined(CONFIG_SOFTMMU) +#else /* Called with mmap_lock held. */ static void tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc, void *puc, From 1279f323d67b482f942c411d8a705fa9cf36e372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 22 Mar 2016 14:59:20 +0000 Subject: [PATCH 27/39] tcg: reorganize tb_find_physical loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Put some comments and improve code structure. This should help reading the code. Signed-off-by: Alex Bennée [Sergey Fedorov: provide commit message; bring back resetting of tb_invalidated_flag] Signed-off-by: Sergey Fedorov Reviewed-by: Richard Henderson Signed-off-by: Richard Henderson --- cpu-exec.c | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/cpu-exec.c b/cpu-exec.c index f984dc71cb..02a49070b9 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -223,10 +223,9 @@ static TranslationBlock *tb_find_physical(CPUState *cpu, uint32_t flags) { CPUArchState *env = (CPUArchState *)cpu->env_ptr; - TranslationBlock *tb, **ptb1; + TranslationBlock *tb, **tb_hash_head, **ptb1; unsigned int h; tb_page_addr_t phys_pc, phys_page1; - target_ulong virt_page2; tcg_ctx.tb_ctx.tb_invalidated_flag = 0; @@ -234,37 +233,42 @@ static TranslationBlock *tb_find_physical(CPUState *cpu, phys_pc = get_page_addr_code(env, pc); phys_page1 = phys_pc & TARGET_PAGE_MASK; h = tb_phys_hash_func(phys_pc); - ptb1 = &tcg_ctx.tb_ctx.tb_phys_hash[h]; - for(;;) { - tb = *ptb1; - if (!tb) { - return NULL; - } + + /* Start at head of the hash entry */ + ptb1 = tb_hash_head = &tcg_ctx.tb_ctx.tb_phys_hash[h]; + tb = *ptb1; + + while (tb) { if (tb->pc == pc && tb->page_addr[0] == phys_page1 && tb->cs_base == cs_base && tb->flags == flags) { - /* check next page if needed */ - if (tb->page_addr[1] != -1) { - tb_page_addr_t phys_page2; - virt_page2 = (pc & TARGET_PAGE_MASK) + - TARGET_PAGE_SIZE; - phys_page2 = get_page_addr_code(env, virt_page2); + if (tb->page_addr[1] == -1) { + /* done, we have a match */ + break; + } else { + /* check next page if needed */ + target_ulong virt_page2 = (pc & TARGET_PAGE_MASK) + + TARGET_PAGE_SIZE; + tb_page_addr_t phys_page2 = get_page_addr_code(env, virt_page2); + if (tb->page_addr[1] == phys_page2) { break; } - } else { - break; } } + ptb1 = &tb->phys_hash_next; + tb = *ptb1; } - /* Move the TB to the head of the list */ - *ptb1 = tb->phys_hash_next; - tb->phys_hash_next = tcg_ctx.tb_ctx.tb_phys_hash[h]; - tcg_ctx.tb_ctx.tb_phys_hash[h] = tb; + if (tb) { + /* Move the TB to the head of the list */ + *ptb1 = tb->phys_hash_next; + tb->phys_hash_next = *tb_hash_head; + *tb_hash_head = tb; + } return tb; } From 7687bf52e5dc39215cde723bb2afb7c53b8bd842 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 11 Aug 2015 11:05:12 +0200 Subject: [PATCH 28/39] cpu-exec: elide more icount code if CONFIG_USER_ONLY MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paolo Bonzini [Alex Bennée: #ifndef replay code to match elided functions] Signed-off-by: Alex Bennée Signed-off-by: Sergey Fedorov Signed-off-by: Richard Henderson --- cpu-exec.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cpu-exec.c b/cpu-exec.c index 02a49070b9..bd831b5a12 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -192,6 +192,7 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb) return next_tb; } +#ifndef CONFIG_USER_ONLY /* Execute the code without caching the generated code. An interpreter could be used if available. */ static void cpu_exec_nocache(CPUState *cpu, int max_cycles, @@ -216,6 +217,7 @@ static void cpu_exec_nocache(CPUState *cpu, int max_cycles, tb_phys_invalidate(tb, -1); tb_free(tb); } +#endif static TranslationBlock *tb_find_physical(CPUState *cpu, target_ulong pc, @@ -430,12 +432,14 @@ int cpu_exec(CPUState *cpu) } #endif } +#ifndef CONFIG_USER_ONLY } else if (replay_has_exception() && cpu->icount_decr.u16.low + cpu->icount_extra == 0) { /* try to cause an exception pending in the log */ cpu_exec_nocache(cpu, 1, tb_find_fast(cpu), true); ret = -1; break; +#endif } next_tb = 0; /* force lookup of first TB */ @@ -542,6 +546,9 @@ int cpu_exec(CPUState *cpu) case TB_EXIT_ICOUNT_EXPIRED: { /* Instruction counter expired. */ +#ifdef CONFIG_USER_ONLY + abort(); +#else int insns_left = cpu->icount_decr.u32; if (cpu->icount_extra && insns_left >= 0) { /* Refill decrementer and continue execution. */ @@ -561,6 +568,7 @@ int cpu_exec(CPUState *cpu) cpu_loop_exit(cpu); } break; +#endif } default: break; From 819af24b9c1e95e6576f1cefd32f4d6bf56dfa56 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 21 Apr 2016 15:58:23 +0300 Subject: [PATCH 29/39] tcg: Clean up from 'next_tb' The value returned from tcg_qemu_tb_exec() is the value passed to the corresponding tcg_gen_exit_tb() at translation time of the last TB attempted to execute. It is a little confusing to store it in a variable named 'next_tb'. In fact, it is a combination of 4-byte aligned pointer and additional information in its two least significant bits. Break it down right away into two variables named 'last_tb' and 'tb_exit' which are a pointer to the last TB attempted to execute and the TB exit reason, correspondingly. This simplifies the code and improves its readability. Correct a misleading documentation comment for tcg_qemu_tb_exec() and fix logging in cpu_tb_exec(). Also rename a misleading 'next_tb' in another couple of places. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Signed-off-by: Richard Henderson --- cpu-exec.c | 59 ++++++++++++++++++++++++++++------------------------ tcg/tcg.h | 19 +++++++++-------- tci.c | 6 +++--- trace-events | 2 +- 4 files changed, 46 insertions(+), 40 deletions(-) diff --git a/cpu-exec.c b/cpu-exec.c index bd831b5a12..9407c66f62 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -136,7 +136,9 @@ static void init_delay_params(SyncClocks *sc, const CPUState *cpu) static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb) { CPUArchState *env = cpu->env_ptr; - uintptr_t next_tb; + uintptr_t ret; + TranslationBlock *last_tb; + int tb_exit; uint8_t *tb_ptr = itb->tc_ptr; qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc, @@ -160,36 +162,37 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb) #endif /* DEBUG_DISAS */ cpu->can_do_io = !use_icount; - next_tb = tcg_qemu_tb_exec(env, tb_ptr); + ret = tcg_qemu_tb_exec(env, tb_ptr); cpu->can_do_io = 1; - trace_exec_tb_exit((void *) (next_tb & ~TB_EXIT_MASK), - next_tb & TB_EXIT_MASK); + last_tb = (TranslationBlock *)(ret & ~TB_EXIT_MASK); + tb_exit = ret & TB_EXIT_MASK; + trace_exec_tb_exit(last_tb, tb_exit); - if ((next_tb & TB_EXIT_MASK) > TB_EXIT_IDX1) { + if (tb_exit > TB_EXIT_IDX1) { /* We didn't start executing this TB (eg because the instruction * counter hit zero); we must restore the guest PC to the address * of the start of the TB. */ CPUClass *cc = CPU_GET_CLASS(cpu); - TranslationBlock *tb = (TranslationBlock *)(next_tb & ~TB_EXIT_MASK); - qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc, + qemu_log_mask_and_addr(CPU_LOG_EXEC, last_tb->pc, "Stopped execution of TB chain before %p [" TARGET_FMT_lx "] %s\n", - itb->tc_ptr, itb->pc, lookup_symbol(itb->pc)); + last_tb->tc_ptr, last_tb->pc, + lookup_symbol(last_tb->pc)); if (cc->synchronize_from_tb) { - cc->synchronize_from_tb(cpu, tb); + cc->synchronize_from_tb(cpu, last_tb); } else { assert(cc->set_pc); - cc->set_pc(cpu, tb->pc); + cc->set_pc(cpu, last_tb->pc); } } - if ((next_tb & TB_EXIT_MASK) == TB_EXIT_REQUESTED) { + if (tb_exit == TB_EXIT_REQUESTED) { /* We were asked to stop executing TBs (probably a pending * interrupt. We've now stopped, so clear the flag. */ cpu->tcg_exit_req = 0; } - return next_tb; + return ret; } #ifndef CONFIG_USER_ONLY @@ -358,8 +361,8 @@ int cpu_exec(CPUState *cpu) CPUArchState *env = &x86_cpu->env; #endif int ret, interrupt_request; - TranslationBlock *tb; - uintptr_t next_tb; + TranslationBlock *tb, *last_tb; + int tb_exit = 0; SyncClocks sc; /* replay_interrupt may need current_cpu */ @@ -442,7 +445,7 @@ int cpu_exec(CPUState *cpu) #endif } - next_tb = 0; /* force lookup of first TB */ + last_tb = NULL; /* forget the last executed TB after exception */ for(;;) { interrupt_request = cpu->interrupt_request; if (unlikely(interrupt_request)) { @@ -487,7 +490,7 @@ int cpu_exec(CPUState *cpu) else { replay_interrupt(); if (cc->cpu_exec_interrupt(cpu, interrupt_request)) { - next_tb = 0; + last_tb = NULL; } } /* Don't use the cached interrupt_request value, @@ -496,7 +499,7 @@ int cpu_exec(CPUState *cpu) cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB; /* ensure that no TB jump will be modified as the program flow was changed */ - next_tb = 0; + last_tb = NULL; } } if (unlikely(cpu->exit_request @@ -513,22 +516,24 @@ int cpu_exec(CPUState *cpu) /* as some TB could have been invalidated because of memory exceptions while generating the code, we must recompute the hash index here */ - next_tb = 0; + last_tb = NULL; tcg_ctx.tb_ctx.tb_invalidated_flag = 0; } /* See if we can patch the calling TB. */ - if (next_tb != 0 && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { - tb_add_jump((TranslationBlock *)(next_tb & ~TB_EXIT_MASK), - next_tb & TB_EXIT_MASK, tb); + if (last_tb && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { + tb_add_jump(last_tb, tb_exit, tb); } tb_unlock(); if (likely(!cpu->exit_request)) { + uintptr_t ret; trace_exec_tb(tb, tb->pc); /* execute the generated code */ cpu->current_tb = tb; - next_tb = cpu_tb_exec(cpu, tb); + ret = cpu_tb_exec(cpu, tb); cpu->current_tb = NULL; - switch (next_tb & TB_EXIT_MASK) { + last_tb = (TranslationBlock *)(ret & ~TB_EXIT_MASK); + tb_exit = ret & TB_EXIT_MASK; + switch (tb_exit) { case TB_EXIT_REQUESTED: /* Something asked us to stop executing * chained TBs; just continue round the main @@ -541,7 +546,7 @@ int cpu_exec(CPUState *cpu) * or cpu->interrupt_request. */ smp_rmb(); - next_tb = 0; + last_tb = NULL; break; case TB_EXIT_ICOUNT_EXPIRED: { @@ -559,12 +564,12 @@ int cpu_exec(CPUState *cpu) } else { if (insns_left > 0) { /* Execute remaining instructions. */ - tb = (TranslationBlock *)(next_tb & ~TB_EXIT_MASK); - cpu_exec_nocache(cpu, insns_left, tb, false); + cpu_exec_nocache(cpu, insns_left, + last_tb, false); align_clocks(&sc, cpu); } cpu->exception_index = EXCP_INTERRUPT; - next_tb = 0; + last_tb = NULL; cpu_loop_exit(cpu); } break; diff --git a/tcg/tcg.h b/tcg/tcg.h index 0fceef5333..a013d77a33 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -925,7 +925,7 @@ static inline unsigned get_mmuidx(TCGMemOpIdx oi) /** * tcg_qemu_tb_exec: - * @env: CPUArchState * for the CPU + * @env: pointer to CPUArchState for the CPU * @tb_ptr: address of generated code for the TB to execute * * Start executing code from a given translation block. @@ -936,30 +936,31 @@ static inline unsigned get_mmuidx(TCGMemOpIdx oi) * which has not yet been directly linked, or an asynchronous * event such as an interrupt needs handling. * - * The return value is a pointer to the next TB to execute - * (if known; otherwise zero). This pointer is assumed to be - * 4-aligned, and the bottom two bits are used to return further - * information: + * Return: The return value is the value passed to the corresponding + * tcg_gen_exit_tb() at translation time of the last TB attempted to execute. + * The value is either zero or a 4-byte aligned pointer to that TB combined + * with additional information in its two least significant bits. The + * additional information is encoded as follows: * 0, 1: the link between this TB and the next is via the specified * TB index (0 or 1). That is, we left the TB via (the equivalent * of) "goto_tb ". The main loop uses this to determine * how to link the TB just executed to the next. * 2: we are using instruction counting code generation, and we * did not start executing this TB because the instruction counter - * would hit zero midway through it. In this case the next-TB pointer + * would hit zero midway through it. In this case the pointer * returned is the TB we were about to execute, and the caller must * arrange to execute the remaining count of instructions. * 3: we stopped because the CPU's exit_request flag was set * (usually meaning that there is an interrupt that needs to be - * handled). The next-TB pointer returned is the TB we were - * about to execute when we noticed the pending exit request. + * handled). The pointer returned is the TB we were about to execute + * when we noticed the pending exit request. * * If the bottom two bits indicate an exit-via-index then the CPU * state is correctly synchronised and ready for execution of the next * TB (and in particular the guest PC is the address to execute next). * Otherwise, we gave up on execution of this TB before it started, and * the caller must fix up the CPU state by calling the CPU's - * synchronize_from_tb() method with the next-TB pointer we return (falling + * synchronize_from_tb() method with the TB pointer we return (falling * back to calling the CPU's set_pc method with tb->pb if no * synchronize_from_tb() method exists). * diff --git a/tci.c b/tci.c index a8939e6d3c..0fdc4e2c12 100644 --- a/tci.c +++ b/tci.c @@ -467,7 +467,7 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr) { long tcg_temps[CPU_TEMP_BUF_NLONGS]; uintptr_t sp_value = (uintptr_t)(tcg_temps + CPU_TEMP_BUF_NLONGS); - uintptr_t next_tb = 0; + uintptr_t ret = 0; tci_reg[TCG_AREG0] = (tcg_target_ulong)env; tci_reg[TCG_REG_CALL_STACK] = sp_value; @@ -1085,7 +1085,7 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr) /* QEMU specific operations. */ case INDEX_op_exit_tb: - next_tb = *(uint64_t *)tb_ptr; + ret = *(uint64_t *)tb_ptr; goto exit; break; case INDEX_op_goto_tb: @@ -1243,5 +1243,5 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr) tci_assert(tb_ptr == old_code_ptr + op_size); } exit: - return next_tb; + return ret; } diff --git a/trace-events b/trace-events index b588091b6c..4fce005afd 100644 --- a/trace-events +++ b/trace-events @@ -1614,7 +1614,7 @@ kvm_failed_spr_get(int str, const char *msg) "Warning: Unable to retrieve SPR %d # cpu-exec.c disable exec_tb(void *tb, uintptr_t pc) "tb:%p pc=0x%"PRIxPTR disable exec_tb_nocache(void *tb, uintptr_t pc) "tb:%p pc=0x%"PRIxPTR -disable exec_tb_exit(void *next_tb, unsigned int flags) "tb:%p flags=%x" +disable exec_tb_exit(void *last_tb, unsigned int flags) "tb:%p flags=%x" # translate-all.c translate_block(void *tb, uintptr_t pc, uint8_t *tb_code) "tb:%p, pc:0x%"PRIxPTR", tb_code:%p" From 6f789be56d3f38e9214dafcfab3bf9be7191f370 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 13 Apr 2016 22:40:18 +0300 Subject: [PATCH 30/39] tcg: Rework tb_invalidated_flag 'tb_invalidated_flag' was meant to catch two events: * some TB has been invalidated by tb_phys_invalidate(); * the whole translation buffer has been flushed by tb_flush(). Then it was checked: * in cpu_exec() to ensure that the last executed TB can be safely linked to directly call the next one; * in cpu_exec_nocache() to decide if the original TB should be provided for further possible invalidation along with the temporarily generated TB. It is always safe to patch an invalidated TB since it is not going to be used anyway. It is also safe to call tb_phys_invalidate() for an already invalidated TB. Thus, setting this flag in tb_phys_invalidate() is simply unnecessary. Moreover, it can prevent from pretty proper linking of TBs, if any arbitrary TB has been invalidated. So just don't touch it in tb_phys_invalidate(). If this flag is only used to catch whether tb_flush() has been called then rename it to 'tb_flushed'. Declare it as 'bool' and stick to using only 'true' and 'false' to set its value. Also, instead of setting it in tb_gen_code(), just after tb_flush() has been called, do it right inside of tb_flush(). In cpu_exec(), this flag is used to track if tb_flush() has been called and have made 'next_tb' (a reference to the last executed TB) invalid for linking it to directly call the next TB. tb_flush() can be called during the CPU execution loop from tb_gen_code(), during TB execution or by another thread while 'tb_lock' is released. Catch for translation buffer flush reliably by resetting this flag once before first TB lookup and each time we find it set before trying to add a direct jump. Don't touch in in tb_find_physical(). Each vCPU has its own execution loop in multithreaded mode and thus should have its own copy of the flag to be able to reset it with its own 'next_tb' and don't affect any other vCPU execution thread. So make this flag per-vCPU and move it to CPUState. In cpu_exec_nocache(), we only need to check if tb_flush() has been called from tb_gen_code() called by cpu_exec_nocache() itself. To do this reliably, preserve the old value of the flag, reset it before calling tb_gen_code(), check afterwards, and combine the saved value back to the flag. This patch is based on the patch "tcg: move tb_invalidated_flag to CPUState" from Paolo Bonzini . Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Signed-off-by: Richard Henderson --- cpu-exec.c | 21 +++++++++++---------- include/exec/exec-all.h | 2 -- include/qom/cpu.h | 2 ++ translate-all.c | 5 +---- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/cpu-exec.c b/cpu-exec.c index 9407c66f62..f49a436e1a 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -202,16 +202,20 @@ static void cpu_exec_nocache(CPUState *cpu, int max_cycles, TranslationBlock *orig_tb, bool ignore_icount) { TranslationBlock *tb; + bool old_tb_flushed; /* Should never happen. We only end up here when an existing TB is too long. */ if (max_cycles > CF_COUNT_MASK) max_cycles = CF_COUNT_MASK; + old_tb_flushed = cpu->tb_flushed; + cpu->tb_flushed = false; tb = tb_gen_code(cpu, orig_tb->pc, orig_tb->cs_base, orig_tb->flags, max_cycles | CF_NOCACHE | (ignore_icount ? CF_IGNORE_ICOUNT : 0)); - tb->orig_tb = tcg_ctx.tb_ctx.tb_invalidated_flag ? NULL : orig_tb; + tb->orig_tb = cpu->tb_flushed ? NULL : orig_tb; + cpu->tb_flushed |= old_tb_flushed; cpu->current_tb = tb; /* execute the generated code */ trace_exec_tb_nocache(tb, tb->pc); @@ -232,8 +236,6 @@ static TranslationBlock *tb_find_physical(CPUState *cpu, unsigned int h; tb_page_addr_t phys_pc, phys_page1; - tcg_ctx.tb_ctx.tb_invalidated_flag = 0; - /* find translated block using physical mappings */ phys_pc = get_page_addr_code(env, pc); phys_page1 = phys_pc & TARGET_PAGE_MASK; @@ -446,6 +448,7 @@ int cpu_exec(CPUState *cpu) } last_tb = NULL; /* forget the last executed TB after exception */ + cpu->tb_flushed = false; /* reset before first TB lookup */ for(;;) { interrupt_request = cpu->interrupt_request; if (unlikely(interrupt_request)) { @@ -510,14 +513,12 @@ int cpu_exec(CPUState *cpu) } tb_lock(); tb = tb_find_fast(cpu); - /* Note: we do it here to avoid a gcc bug on Mac OS X when - doing it in tb_find_slow */ - if (tcg_ctx.tb_ctx.tb_invalidated_flag) { - /* as some TB could have been invalidated because - of memory exceptions while generating the code, we - must recompute the hash index here */ + if (cpu->tb_flushed) { + /* Ensure that no TB jump will be modified as the + * translation buffer has been flushed. + */ last_tb = NULL; - tcg_ctx.tb_ctx.tb_invalidated_flag = 0; + cpu->tb_flushed = false; } /* See if we can patch the calling TB. */ if (last_tb && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 06da1bcc45..85528f9941 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -304,8 +304,6 @@ struct TBContext { /* statistics */ int tb_flush_count; int tb_phys_invalidate_count; - - int tb_invalidated_flag; }; void tb_free(TranslationBlock *tb); diff --git a/include/qom/cpu.h b/include/qom/cpu.h index b7a10f791a..c1ae24d1fc 100644 --- a/include/qom/cpu.h +++ b/include/qom/cpu.h @@ -238,6 +238,7 @@ struct kvm_run; * @crash_occurred: Indicates the OS reported a crash (panic) for this CPU * @tcg_exit_req: Set to force TCG to stop executing linked TBs for this * CPU and return to its top level loop. + * @tb_flushed: Indicates the translation buffer has been flushed. * @singlestep_enabled: Flags for single-stepping. * @icount_extra: Instructions until next timer event. * @icount_decr: Number of cycles left, with interrupt flag in high bit. @@ -289,6 +290,7 @@ struct CPUState { bool stopped; bool crash_occurred; bool exit_request; + bool tb_flushed; uint32_t interrupt_request; int singlestep_enabled; int64_t icount_extra; diff --git a/translate-all.c b/translate-all.c index d5d2bbed3a..2d1957bfa3 100644 --- a/translate-all.c +++ b/translate-all.c @@ -843,6 +843,7 @@ void tb_flush(CPUState *cpu) CPU_FOREACH(cpu) { memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache)); + cpu->tb_flushed = true; } memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash)); @@ -1011,8 +1012,6 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) invalidate_page_bitmap(p); } - tcg_ctx.tb_ctx.tb_invalidated_flag = 1; - /* remove the TB from the hash list */ h = tb_jmp_cache_hash_func(tb->pc); CPU_FOREACH(cpu) { @@ -1178,8 +1177,6 @@ TranslationBlock *tb_gen_code(CPUState *cpu, /* cannot fail at this point */ tb = tb_alloc(pc); assert(tb != NULL); - /* Don't forget to invalidate previous TB info. */ - tcg_ctx.tb_ctx.tb_invalidated_flag = 1; } gen_code_buf = tcg_ctx.code_gen_ptr; From a0522c7a55cc8ac76d82884cf8e52f76daa664cc Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Mon, 25 Apr 2016 18:17:30 +0300 Subject: [PATCH 31/39] cpu-exec: Move TB chaining into tb_find_fast() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move tb_add_jump() call and surrounding code from cpu_exec() into tb_find_fast(). That simplifies cpu_exec() a little by hiding the direct chaining optimization details into tb_find_fast(). It also allows to move tb_lock()/tb_unlock() pair into tb_find_fast(), putting it closer to tb_find_slow() which also manipulates the lock. Suggested-by: Alex Bennée Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Signed-off-by: Richard Henderson [rth: Fixed rebase typo in nochain test.] --- cpu-exec.c | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/cpu-exec.c b/cpu-exec.c index f49a436e1a..7380b1eb4d 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -320,7 +320,9 @@ found: return tb; } -static inline TranslationBlock *tb_find_fast(CPUState *cpu) +static inline TranslationBlock *tb_find_fast(CPUState *cpu, + TranslationBlock **last_tb, + int tb_exit) { CPUArchState *env = (CPUArchState *)cpu->env_ptr; TranslationBlock *tb; @@ -331,11 +333,24 @@ static inline TranslationBlock *tb_find_fast(CPUState *cpu) always be the same before a given translated block is executed. */ cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); + tb_lock(); tb = cpu->tb_jmp_cache[tb_jmp_cache_hash_func(pc)]; if (unlikely(!tb || tb->pc != pc || tb->cs_base != cs_base || tb->flags != flags)) { tb = tb_find_slow(cpu, pc, cs_base, flags); } + if (cpu->tb_flushed) { + /* Ensure that no TB jump will be modified as the + * translation buffer has been flushed. + */ + *last_tb = NULL; + cpu->tb_flushed = false; + } + /* See if we can patch the calling TB. */ + if (*last_tb && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { + tb_add_jump(*last_tb, tb_exit, tb); + } + tb_unlock(); return tb; } @@ -441,7 +456,8 @@ int cpu_exec(CPUState *cpu) } else if (replay_has_exception() && cpu->icount_decr.u16.low + cpu->icount_extra == 0) { /* try to cause an exception pending in the log */ - cpu_exec_nocache(cpu, 1, tb_find_fast(cpu), true); + last_tb = NULL; /* Avoid chaining TBs */ + cpu_exec_nocache(cpu, 1, tb_find_fast(cpu, &last_tb, 0), true); ret = -1; break; #endif @@ -511,20 +527,7 @@ int cpu_exec(CPUState *cpu) cpu->exception_index = EXCP_INTERRUPT; cpu_loop_exit(cpu); } - tb_lock(); - tb = tb_find_fast(cpu); - if (cpu->tb_flushed) { - /* Ensure that no TB jump will be modified as the - * translation buffer has been flushed. - */ - last_tb = NULL; - cpu->tb_flushed = false; - } - /* See if we can patch the calling TB. */ - if (last_tb && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { - tb_add_jump(last_tb, tb_exit, tb); - } - tb_unlock(); + tb = tb_find_fast(cpu, &last_tb, tb_exit); if (likely(!cpu->exit_request)) { uintptr_t ret; trace_exec_tb(tb, tb->pc); From 3213525f8ab48742db09dab18cb9ae6f36a6c921 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Tue, 3 May 2016 14:04:22 +0300 Subject: [PATCH 32/39] tcg: Remove needless CPUState::current_tb This field was used for telling cpu_interrupt() to unlink a chain of TBs being executed when it worked that way. Now, cpu_interrupt() don't do this anymore. So we don't need this field anymore. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Message-Id: <1462273462-14036-1-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- cpu-exec-common.c | 2 -- cpu-exec.c | 4 ---- cputlb.c | 13 ------------- hw/i386/kvmvapic.c | 1 - include/qom/cpu.h | 2 -- qom/cpu.c | 1 - translate-all.c | 20 ++------------------ 7 files changed, 2 insertions(+), 41 deletions(-) diff --git a/cpu-exec-common.c b/cpu-exec-common.c index 1b1731cd83..6bdda6b6b0 100644 --- a/cpu-exec-common.c +++ b/cpu-exec-common.c @@ -68,7 +68,6 @@ void cpu_reloading_memory_map(void) void cpu_loop_exit(CPUState *cpu) { - cpu->current_tb = NULL; siglongjmp(cpu->jmp_env, 1); } @@ -77,6 +76,5 @@ void cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc) if (pc) { cpu_restore_state(cpu, pc); } - cpu->current_tb = NULL; siglongjmp(cpu->jmp_env, 1); } diff --git a/cpu-exec.c b/cpu-exec.c index 7380b1eb4d..d43d5ae729 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -216,11 +216,9 @@ static void cpu_exec_nocache(CPUState *cpu, int max_cycles, | (ignore_icount ? CF_IGNORE_ICOUNT : 0)); tb->orig_tb = cpu->tb_flushed ? NULL : orig_tb; cpu->tb_flushed |= old_tb_flushed; - cpu->current_tb = tb; /* execute the generated code */ trace_exec_tb_nocache(tb, tb->pc); cpu_tb_exec(cpu, tb); - cpu->current_tb = NULL; tb_phys_invalidate(tb, -1); tb_free(tb); } @@ -532,9 +530,7 @@ int cpu_exec(CPUState *cpu) uintptr_t ret; trace_exec_tb(tb, tb->pc); /* execute the generated code */ - cpu->current_tb = tb; ret = cpu_tb_exec(cpu, tb); - cpu->current_tb = NULL; last_tb = (TranslationBlock *)(ret & ~TB_EXIT_MASK); tb_exit = ret & TB_EXIT_MASK; switch (tb_exit) { diff --git a/cputlb.c b/cputlb.c index 466663b56c..167280ae96 100644 --- a/cputlb.c +++ b/cputlb.c @@ -76,10 +76,6 @@ void tlb_flush(CPUState *cpu, int flush_global) tlb_debug("(%d)\n", flush_global); - /* must reset current TB so that interrupts cannot modify the - links while we are modifying them */ - cpu->current_tb = NULL; - memset(env->tlb_table, -1, sizeof(env->tlb_table)); memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table)); memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache)); @@ -95,9 +91,6 @@ static inline void v_tlb_flush_by_mmuidx(CPUState *cpu, va_list argp) CPUArchState *env = cpu->env_ptr; tlb_debug("start\n"); - /* must reset current TB so that interrupts cannot modify the - links while we are modifying them */ - cpu->current_tb = NULL; for (;;) { int mmu_idx = va_arg(argp, int); @@ -152,9 +145,6 @@ void tlb_flush_page(CPUState *cpu, target_ulong addr) tlb_flush(cpu, 1); return; } - /* must reset current TB so that interrupts cannot modify the - links while we are modifying them */ - cpu->current_tb = NULL; addr &= TARGET_PAGE_MASK; i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); @@ -193,9 +183,6 @@ void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, ...) va_end(argp); return; } - /* must reset current TB so that interrupts cannot modify the - links while we are modifying them */ - cpu->current_tb = NULL; addr &= TARGET_PAGE_MASK; i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); diff --git a/hw/i386/kvmvapic.c b/hw/i386/kvmvapic.c index 4bb695d5c1..f14445d4fb 100644 --- a/hw/i386/kvmvapic.c +++ b/hw/i386/kvmvapic.c @@ -446,7 +446,6 @@ static void patch_instruction(VAPICROMState *s, X86CPU *cpu, target_ulong ip) resume_all_vcpus(); if (!kvm_enabled()) { - cs->current_tb = NULL; tb_gen_code(cs, current_pc, current_cs_base, current_flags, 1); cpu_resume_from_signal(cs, NULL); } diff --git a/include/qom/cpu.h b/include/qom/cpu.h index c1ae24d1fc..4349c465c5 100644 --- a/include/qom/cpu.h +++ b/include/qom/cpu.h @@ -253,7 +253,6 @@ struct kvm_run; * @as: Pointer to the first AddressSpace, for the convenience of targets which * only have a single AddressSpace * @env_ptr: Pointer to subclass-specific CPUArchState field. - * @current_tb: Currently executing TB. * @gdb_regs: Additional GDB registers. * @gdb_num_regs: Number of total registers accessible to GDB. * @gdb_num_g_regs: Number of registers in GDB 'g' packets. @@ -305,7 +304,6 @@ struct CPUState { MemoryRegion *memory; void *env_ptr; /* CPUArchState */ - struct TranslationBlock *current_tb; struct TranslationBlock *tb_jmp_cache[TB_JMP_CACHE_SIZE]; struct GDBRegisterState *gdb_regs; int gdb_num_regs; diff --git a/qom/cpu.c b/qom/cpu.c index c9007d3d06..751e992de8 100644 --- a/qom/cpu.c +++ b/qom/cpu.c @@ -254,7 +254,6 @@ static void cpu_common_reset(CPUState *cpu) } cpu->interrupt_request = 0; - cpu->current_tb = NULL; cpu->halted = 0; cpu->mem_io_pc = 0; cpu->mem_io_vaddr = 0; diff --git a/translate-all.c b/translate-all.c index 2d1957bfa3..b54f472531 100644 --- a/translate-all.c +++ b/translate-all.c @@ -305,7 +305,6 @@ bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr) cpu_restore_state_from_tb(cpu, tb, retaddr); if (tb->cflags & CF_NOCACHE) { /* one-shot translation, invalidate it immediately */ - cpu->current_tb = NULL; tb_phys_invalidate(tb, -1); tb_free(tb); } @@ -1309,9 +1308,9 @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end) void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, int is_cpu_write_access) { - TranslationBlock *tb, *tb_next, *saved_tb; - CPUState *cpu = current_cpu; + TranslationBlock *tb, *tb_next; #if defined(TARGET_HAS_PRECISE_SMC) + CPUState *cpu = current_cpu; CPUArchState *env = NULL; #endif tb_page_addr_t tb_start, tb_end; @@ -1378,20 +1377,7 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, ¤t_flags); } #endif /* TARGET_HAS_PRECISE_SMC */ - /* we need to do that to handle the case where a signal - occurs while doing tb_phys_invalidate() */ - saved_tb = NULL; - if (cpu != NULL) { - saved_tb = cpu->current_tb; - cpu->current_tb = NULL; - } tb_phys_invalidate(tb, -1); - if (cpu != NULL) { - cpu->current_tb = saved_tb; - if (cpu->interrupt_request && cpu->current_tb) { - cpu_interrupt(cpu, cpu->interrupt_request); - } - } } tb = tb_next; } @@ -1407,7 +1393,6 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, /* we generate a block containing just the instruction modifying the memory. It will ensure that it cannot modify itself */ - cpu->current_tb = NULL; tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1); cpu_resume_from_signal(cpu, NULL); } @@ -1512,7 +1497,6 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr, /* we generate a block containing just the instruction modifying the memory. It will ensure that it cannot modify itself */ - cpu->current_tb = NULL; tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1); if (locked) { mmap_unlock(); From c6f0d9f84c43ae973270df1a77482466558ee487 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Tue, 3 May 2016 17:34:10 +0300 Subject: [PATCH 33/39] cpu-exec: Remove relic orphaned comment This comment should have been deleted by commit 0ac087f1f3ae ("removed unused code") but somehow it is still here. There's no point to keep it. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Message-Id: <1462286050-21778-1-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- cpu-exec.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpu-exec.c b/cpu-exec.c index d43d5ae729..d55faa5114 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -582,8 +582,6 @@ int cpu_exec(CPUState *cpu) /* Try to align the host and virtual clocks if the guest is in advance */ align_clocks(&sc, cpu); - /* reset soft MMU for next block (it can currently - only be set by a memory fault) */ } /* for(;;) */ } else { #if defined(__clang__) || !QEMU_GNUC_PREREQ(4, 6) From 8b2d34e997371c9729a0f41e3cc624d4300bbe78 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 11 May 2016 13:21:47 +0300 Subject: [PATCH 34/39] cpu-exec: Move halt handling out of cpu_exec() Simplify cpu_exec() by extracting CPU halt state handling code out of cpu_exec() into a new static inline function cpu_handle_halt(). Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Richard Henderson Message-Id: <1462962111-32237-2-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- cpu-exec.c | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/cpu-exec.c b/cpu-exec.c index d55faa5114..529cac2213 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -352,6 +352,28 @@ static inline TranslationBlock *tb_find_fast(CPUState *cpu, return tb; } +static inline bool cpu_handle_halt(CPUState *cpu) +{ + if (cpu->halted) { +#if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY) + if ((cpu->interrupt_request & CPU_INTERRUPT_POLL) + && replay_interrupt()) { + X86CPU *x86_cpu = X86_CPU(cpu); + apic_poll_irq(x86_cpu->apic_state); + cpu_reset_interrupt(cpu, CPU_INTERRUPT_POLL); + } +#endif + if (!cpu_has_work(cpu)) { + current_cpu = NULL; + return true; + } + + cpu->halted = 0; + } + + return false; +} + static void cpu_handle_debug_exception(CPUState *cpu) { CPUClass *cc = CPU_GET_CLASS(cpu); @@ -383,20 +405,8 @@ int cpu_exec(CPUState *cpu) /* replay_interrupt may need current_cpu */ current_cpu = cpu; - if (cpu->halted) { -#if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY) - if ((cpu->interrupt_request & CPU_INTERRUPT_POLL) - && replay_interrupt()) { - apic_poll_irq(x86_cpu->apic_state); - cpu_reset_interrupt(cpu, CPU_INTERRUPT_POLL); - } -#endif - if (!cpu_has_work(cpu)) { - current_cpu = NULL; - return EXCP_HALTED; - } - - cpu->halted = 0; + if (cpu_handle_halt(cpu)) { + return EXCP_HALTED; } atomic_mb_set(&tcg_current_cpu, cpu); From ea284766ec6b9f1712369249566b4c372f3cec8b Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 11 May 2016 13:21:48 +0300 Subject: [PATCH 35/39] cpu-exec: Move exception handling out of cpu_exec() Simplify cpu_exec() by extracting exception handling code out of cpu_exec() into a new static inline function cpu_handle_exception(). Also make cpu_handle_debug_exception() inline as it is used only once. Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Richard Henderson Message-Id: <1462962111-32237-3-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- cpu-exec.c | 93 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 52 insertions(+), 41 deletions(-) diff --git a/cpu-exec.c b/cpu-exec.c index 529cac2213..36df39543b 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -374,7 +374,7 @@ static inline bool cpu_handle_halt(CPUState *cpu) return false; } -static void cpu_handle_debug_exception(CPUState *cpu) +static inline void cpu_handle_debug_exception(CPUState *cpu) { CPUClass *cc = CPU_GET_CLASS(cpu); CPUWatchpoint *wp; @@ -388,6 +388,55 @@ static void cpu_handle_debug_exception(CPUState *cpu) cc->debug_excp_handler(cpu); } +static inline bool cpu_handle_exception(CPUState *cpu, int *ret) +{ + if (cpu->exception_index >= 0) { + if (cpu->exception_index >= EXCP_INTERRUPT) { + /* exit request from the cpu execution loop */ + *ret = cpu->exception_index; + if (*ret == EXCP_DEBUG) { + cpu_handle_debug_exception(cpu); + } + cpu->exception_index = -1; + return true; + } else { +#if defined(CONFIG_USER_ONLY) + /* if user mode only, we simulate a fake exception + which will be handled outside the cpu execution + loop */ +#if defined(TARGET_I386) + CPUClass *cc = CPU_GET_CLASS(cpu); + cc->do_interrupt(cpu); +#endif + *ret = cpu->exception_index; + cpu->exception_index = -1; + return true; +#else + if (replay_exception()) { + CPUClass *cc = CPU_GET_CLASS(cpu); + cc->do_interrupt(cpu); + cpu->exception_index = -1; + } else if (!replay_has_interrupt()) { + /* give a chance to iothread in replay mode */ + *ret = EXCP_INTERRUPT; + return true; + } +#endif + } +#ifndef CONFIG_USER_ONLY + } else if (replay_has_exception() + && cpu->icount_decr.u16.low + cpu->icount_extra == 0) { + /* try to cause an exception pending in the log */ + TranslationBlock *last_tb = NULL; /* Avoid chaining TBs */ + cpu_exec_nocache(cpu, 1, tb_find_fast(cpu, &last_tb, 0), true); + *ret = -1; + return true; +#endif + } + + return false; +} + /* main execution loop */ int cpu_exec(CPUState *cpu) @@ -425,50 +474,12 @@ int cpu_exec(CPUState *cpu) */ init_delay_params(&sc, cpu); - /* prepare setjmp context for exception handling */ for(;;) { + /* prepare setjmp context for exception handling */ if (sigsetjmp(cpu->jmp_env, 0) == 0) { /* if an exception is pending, we execute it here */ - if (cpu->exception_index >= 0) { - if (cpu->exception_index >= EXCP_INTERRUPT) { - /* exit request from the cpu execution loop */ - ret = cpu->exception_index; - if (ret == EXCP_DEBUG) { - cpu_handle_debug_exception(cpu); - } - cpu->exception_index = -1; - break; - } else { -#if defined(CONFIG_USER_ONLY) - /* if user mode only, we simulate a fake exception - which will be handled outside the cpu execution - loop */ -#if defined(TARGET_I386) - cc->do_interrupt(cpu); -#endif - ret = cpu->exception_index; - cpu->exception_index = -1; - break; -#else - if (replay_exception()) { - cc->do_interrupt(cpu); - cpu->exception_index = -1; - } else if (!replay_has_interrupt()) { - /* give a chance to iothread in replay mode */ - ret = EXCP_INTERRUPT; - break; - } -#endif - } -#ifndef CONFIG_USER_ONLY - } else if (replay_has_exception() - && cpu->icount_decr.u16.low + cpu->icount_extra == 0) { - /* try to cause an exception pending in the log */ - last_tb = NULL; /* Avoid chaining TBs */ - cpu_exec_nocache(cpu, 1, tb_find_fast(cpu, &last_tb, 0), true); - ret = -1; + if (cpu_handle_exception(cpu, &ret)) { break; -#endif } last_tb = NULL; /* forget the last executed TB after exception */ From c385e6e49763c6dd5dbbd90fadde95d986f8bd38 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 11 May 2016 13:21:49 +0300 Subject: [PATCH 36/39] cpu-exec: Move interrupt handling out of cpu_exec() Simplify cpu_exec() by extracting interrupt handling code outside of cpu_exec() into a new static inline function cpu_handle_interrupt(). Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Richard Henderson Message-Id: <1462962111-32237-4-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- cpu-exec.c | 132 ++++++++++++++++++++++++++++------------------------- 1 file changed, 70 insertions(+), 62 deletions(-) diff --git a/cpu-exec.c b/cpu-exec.c index 36df39543b..c83b3540de 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -437,6 +437,74 @@ static inline bool cpu_handle_exception(CPUState *cpu, int *ret) return false; } +static inline void cpu_handle_interrupt(CPUState *cpu, + TranslationBlock **last_tb) +{ + CPUClass *cc = CPU_GET_CLASS(cpu); + int interrupt_request = cpu->interrupt_request; + + if (unlikely(interrupt_request)) { + if (unlikely(cpu->singlestep_enabled & SSTEP_NOIRQ)) { + /* Mask out external interrupts for this step. */ + interrupt_request &= ~CPU_INTERRUPT_SSTEP_MASK; + } + if (interrupt_request & CPU_INTERRUPT_DEBUG) { + cpu->interrupt_request &= ~CPU_INTERRUPT_DEBUG; + cpu->exception_index = EXCP_DEBUG; + cpu_loop_exit(cpu); + } + if (replay_mode == REPLAY_MODE_PLAY && !replay_has_interrupt()) { + /* Do nothing */ + } else if (interrupt_request & CPU_INTERRUPT_HALT) { + replay_interrupt(); + cpu->interrupt_request &= ~CPU_INTERRUPT_HALT; + cpu->halted = 1; + cpu->exception_index = EXCP_HLT; + cpu_loop_exit(cpu); + } +#if defined(TARGET_I386) + else if (interrupt_request & CPU_INTERRUPT_INIT) { + X86CPU *x86_cpu = X86_CPU(cpu); + CPUArchState *env = &x86_cpu->env; + replay_interrupt(); + cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0); + do_cpu_init(x86_cpu); + cpu->exception_index = EXCP_HALTED; + cpu_loop_exit(cpu); + } +#else + else if (interrupt_request & CPU_INTERRUPT_RESET) { + replay_interrupt(); + cpu_reset(cpu); + cpu_loop_exit(cpu); + } +#endif + /* The target hook has 3 exit conditions: + False when the interrupt isn't processed, + True when it is, and we should restart on a new TB, + and via longjmp via cpu_loop_exit. */ + else { + replay_interrupt(); + if (cc->cpu_exec_interrupt(cpu, interrupt_request)) { + *last_tb = NULL; + } + } + /* Don't use the cached interrupt_request value, + do_interrupt may have updated the EXITTB flag. */ + if (cpu->interrupt_request & CPU_INTERRUPT_EXITTB) { + cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB; + /* ensure that no TB jump will be modified as + the program flow was changed */ + *last_tb = NULL; + } + } + if (unlikely(cpu->exit_request || replay_has_interrupt())) { + cpu->exit_request = 0; + cpu->exception_index = EXCP_INTERRUPT; + cpu_loop_exit(cpu); + } +} + /* main execution loop */ int cpu_exec(CPUState *cpu) @@ -446,7 +514,7 @@ int cpu_exec(CPUState *cpu) X86CPU *x86_cpu = X86_CPU(cpu); CPUArchState *env = &x86_cpu->env; #endif - int ret, interrupt_request; + int ret; TranslationBlock *tb, *last_tb; int tb_exit = 0; SyncClocks sc; @@ -485,67 +553,7 @@ int cpu_exec(CPUState *cpu) last_tb = NULL; /* forget the last executed TB after exception */ cpu->tb_flushed = false; /* reset before first TB lookup */ for(;;) { - interrupt_request = cpu->interrupt_request; - if (unlikely(interrupt_request)) { - if (unlikely(cpu->singlestep_enabled & SSTEP_NOIRQ)) { - /* Mask out external interrupts for this step. */ - interrupt_request &= ~CPU_INTERRUPT_SSTEP_MASK; - } - if (interrupt_request & CPU_INTERRUPT_DEBUG) { - cpu->interrupt_request &= ~CPU_INTERRUPT_DEBUG; - cpu->exception_index = EXCP_DEBUG; - cpu_loop_exit(cpu); - } - if (replay_mode == REPLAY_MODE_PLAY - && !replay_has_interrupt()) { - /* Do nothing */ - } else if (interrupt_request & CPU_INTERRUPT_HALT) { - replay_interrupt(); - cpu->interrupt_request &= ~CPU_INTERRUPT_HALT; - cpu->halted = 1; - cpu->exception_index = EXCP_HLT; - cpu_loop_exit(cpu); - } -#if defined(TARGET_I386) - else if (interrupt_request & CPU_INTERRUPT_INIT) { - replay_interrupt(); - cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0); - do_cpu_init(x86_cpu); - cpu->exception_index = EXCP_HALTED; - cpu_loop_exit(cpu); - } -#else - else if (interrupt_request & CPU_INTERRUPT_RESET) { - replay_interrupt(); - cpu_reset(cpu); - cpu_loop_exit(cpu); - } -#endif - /* The target hook has 3 exit conditions: - False when the interrupt isn't processed, - True when it is, and we should restart on a new TB, - and via longjmp via cpu_loop_exit. */ - else { - replay_interrupt(); - if (cc->cpu_exec_interrupt(cpu, interrupt_request)) { - last_tb = NULL; - } - } - /* Don't use the cached interrupt_request value, - do_interrupt may have updated the EXITTB flag. */ - if (cpu->interrupt_request & CPU_INTERRUPT_EXITTB) { - cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB; - /* ensure that no TB jump will be modified as - the program flow was changed */ - last_tb = NULL; - } - } - if (unlikely(cpu->exit_request - || replay_has_interrupt())) { - cpu->exit_request = 0; - cpu->exception_index = EXCP_INTERRUPT; - cpu_loop_exit(cpu); - } + cpu_handle_interrupt(cpu, &last_tb); tb = tb_find_fast(cpu, &last_tb, tb_exit); if (likely(!cpu->exit_request)) { uintptr_t ret; From 928de9ee14b0b63ee9f9275732ed3e1c8b5f4790 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 11 May 2016 13:21:50 +0300 Subject: [PATCH 37/39] cpu-exec: Move TB execution stuff out of cpu_exec() Simplify cpu_exec() by extracting TB execution code outside of cpu_exec() into a new static inline function cpu_loop_exec_tb(). Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Richard Henderson Message-Id: <1462962111-32237-5-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- cpu-exec.c | 119 ++++++++++++++++++++++++++++------------------------- 1 file changed, 64 insertions(+), 55 deletions(-) diff --git a/cpu-exec.c b/cpu-exec.c index c83b3540de..251988b0e7 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -505,6 +505,66 @@ static inline void cpu_handle_interrupt(CPUState *cpu, } } +static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb, + TranslationBlock **last_tb, int *tb_exit, + SyncClocks *sc) +{ + uintptr_t ret; + + if (unlikely(cpu->exit_request)) { + return; + } + + trace_exec_tb(tb, tb->pc); + ret = cpu_tb_exec(cpu, tb); + *last_tb = (TranslationBlock *)(ret & ~TB_EXIT_MASK); + *tb_exit = ret & TB_EXIT_MASK; + switch (*tb_exit) { + case TB_EXIT_REQUESTED: + /* Something asked us to stop executing + * chained TBs; just continue round the main + * loop. Whatever requested the exit will also + * have set something else (eg exit_request or + * interrupt_request) which we will handle + * next time around the loop. But we need to + * ensure the tcg_exit_req read in generated code + * comes before the next read of cpu->exit_request + * or cpu->interrupt_request. + */ + smp_rmb(); + *last_tb = NULL; + break; + case TB_EXIT_ICOUNT_EXPIRED: + { + /* Instruction counter expired. */ +#ifdef CONFIG_USER_ONLY + abort(); +#else + int insns_left = cpu->icount_decr.u32; + if (cpu->icount_extra && insns_left >= 0) { + /* Refill decrementer and continue execution. */ + cpu->icount_extra += insns_left; + insns_left = MIN(0xffff, cpu->icount_extra); + cpu->icount_extra -= insns_left; + cpu->icount_decr.u16.low = insns_left; + } else { + if (insns_left > 0) { + /* Execute remaining instructions. */ + cpu_exec_nocache(cpu, insns_left, *last_tb, false); + align_clocks(sc, cpu); + } + cpu->exception_index = EXCP_INTERRUPT; + *last_tb = NULL; + cpu_loop_exit(cpu); + } + break; +#endif + } + default: + break; + } +} + /* main execution loop */ int cpu_exec(CPUState *cpu) @@ -515,8 +575,6 @@ int cpu_exec(CPUState *cpu) CPUArchState *env = &x86_cpu->env; #endif int ret; - TranslationBlock *tb, *last_tb; - int tb_exit = 0; SyncClocks sc; /* replay_interrupt may need current_cpu */ @@ -543,6 +601,9 @@ int cpu_exec(CPUState *cpu) init_delay_params(&sc, cpu); for(;;) { + TranslationBlock *tb, *last_tb; + int tb_exit = 0; + /* prepare setjmp context for exception handling */ if (sigsetjmp(cpu->jmp_env, 0) == 0) { /* if an exception is pending, we execute it here */ @@ -555,59 +616,7 @@ int cpu_exec(CPUState *cpu) for(;;) { cpu_handle_interrupt(cpu, &last_tb); tb = tb_find_fast(cpu, &last_tb, tb_exit); - if (likely(!cpu->exit_request)) { - uintptr_t ret; - trace_exec_tb(tb, tb->pc); - /* execute the generated code */ - ret = cpu_tb_exec(cpu, tb); - last_tb = (TranslationBlock *)(ret & ~TB_EXIT_MASK); - tb_exit = ret & TB_EXIT_MASK; - switch (tb_exit) { - case TB_EXIT_REQUESTED: - /* Something asked us to stop executing - * chained TBs; just continue round the main - * loop. Whatever requested the exit will also - * have set something else (eg exit_request or - * interrupt_request) which we will handle - * next time around the loop. But we need to - * ensure the tcg_exit_req read in generated code - * comes before the next read of cpu->exit_request - * or cpu->interrupt_request. - */ - smp_rmb(); - last_tb = NULL; - break; - case TB_EXIT_ICOUNT_EXPIRED: - { - /* Instruction counter expired. */ -#ifdef CONFIG_USER_ONLY - abort(); -#else - int insns_left = cpu->icount_decr.u32; - if (cpu->icount_extra && insns_left >= 0) { - /* Refill decrementer and continue execution. */ - cpu->icount_extra += insns_left; - insns_left = MIN(0xffff, cpu->icount_extra); - cpu->icount_extra -= insns_left; - cpu->icount_decr.u16.low = insns_left; - } else { - if (insns_left > 0) { - /* Execute remaining instructions. */ - cpu_exec_nocache(cpu, insns_left, - last_tb, false); - align_clocks(&sc, cpu); - } - cpu->exception_index = EXCP_INTERRUPT; - last_tb = NULL; - cpu_loop_exit(cpu); - } - break; -#endif - } - default: - break; - } - } + cpu_loop_exec_tb(cpu, tb, &last_tb, &tb_exit, &sc); /* Try to align the host and virtual clocks if the guest is in advance */ align_clocks(&sc, cpu); From ba048a4ae15ba0f70c6dcb12ee05db120408de78 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 11 May 2016 13:21:51 +0300 Subject: [PATCH 38/39] cpu-exec: Remove unused 'x86_cpu' and 'env' from cpu_exec() Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Reviewed-by: Richard Henderson Message-Id: <1462962111-32237-6-git-send-email-sergey.fedorov@linaro.org> Signed-off-by: Richard Henderson --- cpu-exec.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/cpu-exec.c b/cpu-exec.c index 251988b0e7..0ea47e997d 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -570,10 +570,6 @@ static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb, int cpu_exec(CPUState *cpu) { CPUClass *cc = CPU_GET_CLASS(cpu); -#ifdef TARGET_I386 - X86CPU *x86_cpu = X86_CPU(cpu); - CPUArchState *env = &x86_cpu->env; -#endif int ret; SyncClocks sc; @@ -629,18 +625,10 @@ int cpu_exec(CPUState *cpu) * Newer versions of gcc would complain about this code (-Wclobbered). */ cpu = current_cpu; cc = CPU_GET_CLASS(cpu); -#ifdef TARGET_I386 - x86_cpu = X86_CPU(cpu); - env = &x86_cpu->env; -#endif #else /* buggy compiler */ /* Assert that the compiler does not smash local variables. */ g_assert(cpu == current_cpu); g_assert(cc == CPU_GET_CLASS(cpu)); -#ifdef TARGET_I386 - g_assert(x86_cpu == X86_CPU(cpu)); - g_assert(env == &x86_cpu->env); -#endif #endif /* buggy compiler */ cpu->can_do_io = 1; tb_lock_reset(); From 8b1fe3f439eaa2f0a6ee7737942bb6c405725867 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 12 May 2016 19:52:17 +0300 Subject: [PATCH 39/39] cpu-exec: Clean up 'interrupt_request' reloading in cpu_handle_interrupt() Suggested-by: Richard Henderson Signed-off-by: Sergey Fedorov Signed-off-by: Sergey Fedorov Message-Id: <1463071937-26607-1-git-send-email-sergey.fedorov@linaro.org> Reviewed-by: Richard Henderson Signed-off-by: Richard Henderson --- cpu-exec.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cpu-exec.c b/cpu-exec.c index 0ea47e997d..14df1aacf4 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -488,10 +488,11 @@ static inline void cpu_handle_interrupt(CPUState *cpu, if (cc->cpu_exec_interrupt(cpu, interrupt_request)) { *last_tb = NULL; } + /* The target hook may have updated the 'cpu->interrupt_request'; + * reload the 'interrupt_request' value */ + interrupt_request = cpu->interrupt_request; } - /* Don't use the cached interrupt_request value, - do_interrupt may have updated the EXITTB flag. */ - if (cpu->interrupt_request & CPU_INTERRUPT_EXITTB) { + if (interrupt_request & CPU_INTERRUPT_EXITTB) { cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB; /* ensure that no TB jump will be modified as the program flow was changed */