From 6d06fdd10eecc89d73af0b9e3a7cf1be64a9988b Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:16:22 +0000 Subject: [PATCH 01/42] elf.h: Add the RISCV ELF magic numbers Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: <02fc0b3a733f5f08eb396bee5afd3d327941f0c9.1545246859.git.alistair.francis@wdc.com> Signed-off-by: Richard Henderson --- include/elf.h | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/include/elf.h b/include/elf.h index c151164b63..0ac7911b7b 100644 --- a/include/elf.h +++ b/include/elf.h @@ -1338,6 +1338,61 @@ typedef struct { #define R_IA64_DTPREL64LSB 0xb7 /* @dtprel(sym + add), data8 LSB */ #define R_IA64_LTOFF_DTPREL22 0xba /* @ltoff(@dtprel(s+a)), imm22 */ +/* RISC-V relocations. */ +#define R_RISCV_NONE 0 +#define R_RISCV_32 1 +#define R_RISCV_64 2 +#define R_RISCV_RELATIVE 3 +#define R_RISCV_COPY 4 +#define R_RISCV_JUMP_SLOT 5 +#define R_RISCV_TLS_DTPMOD32 6 +#define R_RISCV_TLS_DTPMOD64 7 +#define R_RISCV_TLS_DTPREL32 8 +#define R_RISCV_TLS_DTPREL64 9 +#define R_RISCV_TLS_TPREL32 10 +#define R_RISCV_TLS_TPREL64 11 +#define R_RISCV_BRANCH 16 +#define R_RISCV_JAL 17 +#define R_RISCV_CALL 18 +#define R_RISCV_CALL_PLT 19 +#define R_RISCV_GOT_HI20 20 +#define R_RISCV_TLS_GOT_HI20 21 +#define R_RISCV_TLS_GD_HI20 22 +#define R_RISCV_PCREL_HI20 23 +#define R_RISCV_PCREL_LO12_I 24 +#define R_RISCV_PCREL_LO12_S 25 +#define R_RISCV_HI20 26 +#define R_RISCV_LO12_I 27 +#define R_RISCV_LO12_S 28 +#define R_RISCV_TPREL_HI20 29 +#define R_RISCV_TPREL_LO12_I 30 +#define R_RISCV_TPREL_LO12_S 31 +#define R_RISCV_TPREL_ADD 32 +#define R_RISCV_ADD8 33 +#define R_RISCV_ADD16 34 +#define R_RISCV_ADD32 35 +#define R_RISCV_ADD64 36 +#define R_RISCV_SUB8 37 +#define R_RISCV_SUB16 38 +#define R_RISCV_SUB32 39 +#define R_RISCV_SUB64 40 +#define R_RISCV_GNU_VTINHERIT 41 +#define R_RISCV_GNU_VTENTRY 42 +#define R_RISCV_ALIGN 43 +#define R_RISCV_RVC_BRANCH 44 +#define R_RISCV_RVC_JUMP 45 +#define R_RISCV_RVC_LUI 46 +#define R_RISCV_GPREL_I 47 +#define R_RISCV_GPREL_S 48 +#define R_RISCV_TPREL_I 49 +#define R_RISCV_TPREL_S 50 +#define R_RISCV_RELAX 51 +#define R_RISCV_SUB6 52 +#define R_RISCV_SET6 53 +#define R_RISCV_SET8 54 +#define R_RISCV_SET16 55 +#define R_RISCV_SET32 56 + typedef struct elf32_rel { Elf32_Addr r_offset; Elf32_Word r_info; From f936eac8082ece746c543386a234087d0b32e419 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:16:31 +0000 Subject: [PATCH 02/42] linux-user: Add host dependency for RISC-V 32-bit Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: <76f8f9383a766dbcade883e897dec8cfef669799.1545246859.git.alistair.francis@wdc.com> Signed-off-by: Richard Henderson --- MAINTAINERS | 1 + linux-user/host/riscv32/hostdep.h | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 linux-user/host/riscv32/hostdep.h diff --git a/MAINTAINERS b/MAINTAINERS index 180695f5d3..067629b10e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -266,6 +266,7 @@ S: Maintained F: target/riscv/ F: hw/riscv/ F: include/hw/riscv/ +F: linux-user/host/riscv32/ F: disas/riscv.c S390 diff --git a/linux-user/host/riscv32/hostdep.h b/linux-user/host/riscv32/hostdep.h new file mode 100644 index 0000000000..adf9edbf2d --- /dev/null +++ b/linux-user/host/riscv32/hostdep.h @@ -0,0 +1,11 @@ +/* + * hostdep.h : things which are dependent on the host architecture + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef RISCV32_HOSTDEP_H +#define RISCV32_HOSTDEP_H + +#endif From 511f3138e4af5c1022e0f746cb4f5454fd58fab0 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:16:41 +0000 Subject: [PATCH 03/42] linux-user: Add host dependency for RISC-V 64-bit Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: <9d777f619840a8dd8e4f3834dcfc3bd28e052ccd.1545246859.git.alistair.francis@wdc.com> Signed-off-by: Richard Henderson --- MAINTAINERS | 1 + linux-user/host/riscv64/hostdep.h | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 linux-user/host/riscv64/hostdep.h diff --git a/MAINTAINERS b/MAINTAINERS index 067629b10e..7a4cf30a04 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -267,6 +267,7 @@ F: target/riscv/ F: hw/riscv/ F: include/hw/riscv/ F: linux-user/host/riscv32/ +F: linux-user/host/riscv64/ F: disas/riscv.c S390 diff --git a/linux-user/host/riscv64/hostdep.h b/linux-user/host/riscv64/hostdep.h new file mode 100644 index 0000000000..28467ba00b --- /dev/null +++ b/linux-user/host/riscv64/hostdep.h @@ -0,0 +1,11 @@ +/* + * hostdep.h : things which are dependent on the host architecture + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef RISCV64_HOSTDEP_H +#define RISCV64_HOSTDEP_H + +#endif From e4041f669fc7abb966247d194743e2d91fdc7d5b Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:16:52 +0000 Subject: [PATCH 04/42] exec: Add RISC-V GCC poison macro Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: <00d02e34f10b87fd61f8dc69ac93d1eb63df949c.1545246859.git.alistair.francis@wdc.com> Signed-off-by: Richard Henderson --- include/exec/poison.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/exec/poison.h b/include/exec/poison.h index 32d53789f8..ecdc83c147 100644 --- a/include/exec/poison.h +++ b/include/exec/poison.h @@ -79,6 +79,7 @@ #pragma GCC poison CONFIG_MOXIE_DIS #pragma GCC poison CONFIG_NIOS2_DIS #pragma GCC poison CONFIG_PPC_DIS +#pragma GCC poison CONFIG_RISCV_DIS #pragma GCC poison CONFIG_S390_DIS #pragma GCC poison CONFIG_SH4_DIS #pragma GCC poison CONFIG_SPARC_DIS From fb1f70f3685bae613d91626578bce96590ed2cb7 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:17:07 +0000 Subject: [PATCH 05/42] tcg/riscv: Add the tcg-target.h file Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: Signed-off-by: Richard Henderson --- MAINTAINERS | 10 ++- tcg/riscv/tcg-target.h | 177 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 tcg/riscv/tcg-target.h diff --git a/MAINTAINERS b/MAINTAINERS index 7a4cf30a04..827d6445d3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -268,7 +268,6 @@ F: hw/riscv/ F: include/hw/riscv/ F: linux-user/host/riscv32/ F: linux-user/host/riscv64/ -F: disas/riscv.c S390 M: Richard Henderson @@ -2174,6 +2173,15 @@ S: Odd Fixes F: tcg/ppc/ F: disas/ppc.c +RISC-V +M: Michael Clark +M: Palmer Dabbelt +M: Alistair Francis +L: qemu-riscv@nongnu.org +S: Maintained +F: tcg/riscv/ +F: disas/riscv.c + S390 target M: Richard Henderson S: Maintained diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h new file mode 100644 index 0000000000..60918cacb4 --- /dev/null +++ b/tcg/riscv/tcg-target.h @@ -0,0 +1,177 @@ +/* + * Tiny Code Generator for QEMU + * + * Copyright (c) 2018 SiFive, Inc + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef RISCV_TCG_TARGET_H +#define RISCV_TCG_TARGET_H + +#if __riscv_xlen == 32 +# define TCG_TARGET_REG_BITS 32 +#elif __riscv_xlen == 64 +# define TCG_TARGET_REG_BITS 64 +#endif + +#define TCG_TARGET_INSN_UNIT_SIZE 4 +#define TCG_TARGET_TLB_DISPLACEMENT_BITS 20 +#define TCG_TARGET_NB_REGS 32 + +typedef enum { + TCG_REG_ZERO, + TCG_REG_RA, + TCG_REG_SP, + TCG_REG_GP, + TCG_REG_TP, + TCG_REG_T0, + TCG_REG_T1, + TCG_REG_T2, + TCG_REG_S0, + TCG_REG_S1, + TCG_REG_A0, + TCG_REG_A1, + TCG_REG_A2, + TCG_REG_A3, + TCG_REG_A4, + TCG_REG_A5, + TCG_REG_A6, + TCG_REG_A7, + TCG_REG_S2, + TCG_REG_S3, + TCG_REG_S4, + TCG_REG_S5, + TCG_REG_S6, + TCG_REG_S7, + TCG_REG_S8, + TCG_REG_S9, + TCG_REG_S10, + TCG_REG_S11, + TCG_REG_T3, + TCG_REG_T4, + TCG_REG_T5, + TCG_REG_T6, + + /* aliases */ + TCG_AREG0 = TCG_REG_S0, + TCG_GUEST_BASE_REG = TCG_REG_S1, + TCG_REG_TMP0 = TCG_REG_T6, + TCG_REG_TMP1 = TCG_REG_T5, + TCG_REG_TMP2 = TCG_REG_T4, +} TCGReg; + +/* used for function call generation */ +#define TCG_REG_CALL_STACK TCG_REG_SP +#define TCG_TARGET_STACK_ALIGN 16 +#define TCG_TARGET_CALL_ALIGN_ARGS 1 +#define TCG_TARGET_CALL_STACK_OFFSET 0 + +/* optional instructions */ +#define TCG_TARGET_HAS_goto_ptr 1 +#define TCG_TARGET_HAS_movcond_i32 0 +#define TCG_TARGET_HAS_div_i32 1 +#define TCG_TARGET_HAS_rem_i32 1 +#define TCG_TARGET_HAS_div2_i32 0 +#define TCG_TARGET_HAS_rot_i32 0 +#define TCG_TARGET_HAS_deposit_i32 0 +#define TCG_TARGET_HAS_extract_i32 0 +#define TCG_TARGET_HAS_sextract_i32 0 +#define TCG_TARGET_HAS_add2_i32 1 +#define TCG_TARGET_HAS_sub2_i32 1 +#define TCG_TARGET_HAS_mulu2_i32 0 +#define TCG_TARGET_HAS_muls2_i32 0 +#define TCG_TARGET_HAS_muluh_i32 (TCG_TARGET_REG_BITS == 32) +#define TCG_TARGET_HAS_mulsh_i32 (TCG_TARGET_REG_BITS == 32) +#define TCG_TARGET_HAS_ext8s_i32 1 +#define TCG_TARGET_HAS_ext16s_i32 1 +#define TCG_TARGET_HAS_ext8u_i32 1 +#define TCG_TARGET_HAS_ext16u_i32 1 +#define TCG_TARGET_HAS_bswap16_i32 0 +#define TCG_TARGET_HAS_bswap32_i32 0 +#define TCG_TARGET_HAS_not_i32 1 +#define TCG_TARGET_HAS_neg_i32 1 +#define TCG_TARGET_HAS_andc_i32 0 +#define TCG_TARGET_HAS_orc_i32 0 +#define TCG_TARGET_HAS_eqv_i32 0 +#define TCG_TARGET_HAS_nand_i32 0 +#define TCG_TARGET_HAS_nor_i32 0 +#define TCG_TARGET_HAS_clz_i32 0 +#define TCG_TARGET_HAS_ctz_i32 0 +#define TCG_TARGET_HAS_ctpop_i32 0 +#define TCG_TARGET_HAS_direct_jump 0 +#define TCG_TARGET_HAS_brcond2 1 +#define TCG_TARGET_HAS_setcond2 1 + +#if TCG_TARGET_REG_BITS == 64 +#define TCG_TARGET_HAS_movcond_i64 0 +#define TCG_TARGET_HAS_div_i64 1 +#define TCG_TARGET_HAS_rem_i64 1 +#define TCG_TARGET_HAS_div2_i64 0 +#define TCG_TARGET_HAS_rot_i64 0 +#define TCG_TARGET_HAS_deposit_i64 0 +#define TCG_TARGET_HAS_extract_i64 0 +#define TCG_TARGET_HAS_sextract_i64 0 +#define TCG_TARGET_HAS_extrl_i64_i32 1 +#define TCG_TARGET_HAS_extrh_i64_i32 1 +#define TCG_TARGET_HAS_ext8s_i64 1 +#define TCG_TARGET_HAS_ext16s_i64 1 +#define TCG_TARGET_HAS_ext32s_i64 1 +#define TCG_TARGET_HAS_ext8u_i64 1 +#define TCG_TARGET_HAS_ext16u_i64 1 +#define TCG_TARGET_HAS_ext32u_i64 1 +#define TCG_TARGET_HAS_bswap16_i64 0 +#define TCG_TARGET_HAS_bswap32_i64 0 +#define TCG_TARGET_HAS_bswap64_i64 0 +#define TCG_TARGET_HAS_not_i64 1 +#define TCG_TARGET_HAS_neg_i64 1 +#define TCG_TARGET_HAS_andc_i64 0 +#define TCG_TARGET_HAS_orc_i64 0 +#define TCG_TARGET_HAS_eqv_i64 0 +#define TCG_TARGET_HAS_nand_i64 0 +#define TCG_TARGET_HAS_nor_i64 0 +#define TCG_TARGET_HAS_clz_i64 0 +#define TCG_TARGET_HAS_ctz_i64 0 +#define TCG_TARGET_HAS_ctpop_i64 0 +#define TCG_TARGET_HAS_add2_i64 1 +#define TCG_TARGET_HAS_sub2_i64 1 +#define TCG_TARGET_HAS_mulu2_i64 0 +#define TCG_TARGET_HAS_muls2_i64 0 +#define TCG_TARGET_HAS_muluh_i64 1 +#define TCG_TARGET_HAS_mulsh_i64 1 +#endif + +static inline void flush_icache_range(uintptr_t start, uintptr_t stop) +{ + __builtin___clear_cache((char *)start, (char *)stop); +} + +/* not defined -- call should be eliminated at compile time */ +void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t); + +#define TCG_TARGET_DEFAULT_MO (0) + +#ifdef CONFIG_SOFTMMU +#define TCG_TARGET_NEED_LDST_LABELS +#endif +#define TCG_TARGET_NEED_POOL_LABELS + +#define TCG_TARGET_HAS_MEMORY_BSWAP 0 + +#endif From 505e75c592825e3a8fed38af252fa4425e2c2b09 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:17:20 +0000 Subject: [PATCH 06/42] tcg/riscv: Add the tcg target registers Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: <6e43abaa64361d57b9bc9439820d0e7701f2d47e.1545246859.git.alistair.francis@wdc.com> Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 118 +++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 tcg/riscv/tcg-target.inc.c diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c new file mode 100644 index 0000000000..6c969e3973 --- /dev/null +++ b/tcg/riscv/tcg-target.inc.c @@ -0,0 +1,118 @@ +/* + * Tiny Code Generator for QEMU + * + * Copyright (c) 2018 SiFive, Inc + * Copyright (c) 2008-2009 Arnaud Patard + * Copyright (c) 2009 Aurelien Jarno + * Copyright (c) 2008 Fabrice Bellard + * + * Based on i386/tcg-target.c and mips/tcg-target.c + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "tcg-pool.inc.c" + +#ifdef CONFIG_DEBUG_TCG +static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { + "zero", + "ra", + "sp", + "gp", + "tp", + "t0", + "t1", + "t2", + "s0", + "s1", + "a0", + "a1", + "a2", + "a3", + "a4", + "a5", + "a6", + "a7", + "s2", + "s3", + "s4", + "s5", + "s6", + "s7", + "s8", + "s9", + "s10", + "s11", + "t3", + "t4", + "t5", + "t6" +}; +#endif + +static const int tcg_target_reg_alloc_order[] = { + /* Call saved registers */ + /* TCG_REG_S0 reservered for TCG_AREG0 */ + TCG_REG_S1, + TCG_REG_S2, + TCG_REG_S3, + TCG_REG_S4, + TCG_REG_S5, + TCG_REG_S6, + TCG_REG_S7, + TCG_REG_S8, + TCG_REG_S9, + TCG_REG_S10, + TCG_REG_S11, + + /* Call clobbered registers */ + TCG_REG_T0, + TCG_REG_T1, + TCG_REG_T2, + TCG_REG_T3, + TCG_REG_T4, + TCG_REG_T5, + TCG_REG_T6, + + /* Argument registers */ + TCG_REG_A0, + TCG_REG_A1, + TCG_REG_A2, + TCG_REG_A3, + TCG_REG_A4, + TCG_REG_A5, + TCG_REG_A6, + TCG_REG_A7, +}; + +static const int tcg_target_call_iarg_regs[] = { + TCG_REG_A0, + TCG_REG_A1, + TCG_REG_A2, + TCG_REG_A3, + TCG_REG_A4, + TCG_REG_A5, + TCG_REG_A6, + TCG_REG_A7, +}; + +static const int tcg_target_call_oarg_regs[] = { + TCG_REG_A0, + TCG_REG_A1, +}; From 8ce23a131254387bbd6135802aef3c6dce010249 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:17:32 +0000 Subject: [PATCH 07/42] tcg/riscv: Add support for the constraints Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 168 +++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index 6c969e3973..f853d01803 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -116,3 +116,171 @@ static const int tcg_target_call_oarg_regs[] = { TCG_REG_A0, TCG_REG_A1, }; + +#define TCG_CT_CONST_ZERO 0x100 +#define TCG_CT_CONST_S12 0x200 +#define TCG_CT_CONST_N12 0x400 +#define TCG_CT_CONST_M12 0x800 + +static inline tcg_target_long sextreg(tcg_target_long val, int pos, int len) +{ + if (TCG_TARGET_REG_BITS == 32) { + return sextract32(val, pos, len); + } else { + return sextract64(val, pos, len); + } +} + +/* parse target specific constraints */ +static const char *target_parse_constraint(TCGArgConstraint *ct, + const char *ct_str, TCGType type) +{ + switch (*ct_str++) { + case 'r': + ct->ct |= TCG_CT_REG; + ct->u.regs = 0xffffffff; + break; + case 'L': + /* qemu_ld/qemu_st constraint */ + ct->ct |= TCG_CT_REG; + ct->u.regs = 0xffffffff; + /* qemu_ld/qemu_st uses TCG_REG_TMP0 */ +#if defined(CONFIG_SOFTMMU) + tcg_regset_reset_reg(ct->u.regs, tcg_target_call_iarg_regs[0]); + tcg_regset_reset_reg(ct->u.regs, tcg_target_call_iarg_regs[1]); + tcg_regset_reset_reg(ct->u.regs, tcg_target_call_iarg_regs[2]); + tcg_regset_reset_reg(ct->u.regs, tcg_target_call_iarg_regs[3]); + tcg_regset_reset_reg(ct->u.regs, tcg_target_call_iarg_regs[4]); +#endif + break; + case 'I': + ct->ct |= TCG_CT_CONST_S12; + break; + case 'N': + ct->ct |= TCG_CT_CONST_N12; + break; + case 'M': + ct->ct |= TCG_CT_CONST_M12; + break; + case 'Z': + /* we can use a zero immediate as a zero register argument. */ + ct->ct |= TCG_CT_CONST_ZERO; + break; + default: + return NULL; + } + return ct_str; +} + +/* test if a constant matches the constraint */ +static int tcg_target_const_match(tcg_target_long val, TCGType type, + const TCGArgConstraint *arg_ct) +{ + int ct = arg_ct->ct; + if (ct & TCG_CT_CONST) { + return 1; + } + if ((ct & TCG_CT_CONST_ZERO) && val == 0) { + return 1; + } + if ((ct & TCG_CT_CONST_S12) && val == sextreg(val, 0, 12)) { + return 1; + } + if ((ct & TCG_CT_CONST_N12) && -val == sextreg(-val, 0, 12)) { + return 1; + } + if ((ct & TCG_CT_CONST_M12) && val >= -0xfff && val <= 0xfff) { + return 1; + } + return 0; +} + +/* + * RISC-V Base ISA opcodes (IM) + */ + +typedef enum { + OPC_ADD = 0x33, + OPC_ADDI = 0x13, + OPC_AND = 0x7033, + OPC_ANDI = 0x7013, + OPC_AUIPC = 0x17, + OPC_BEQ = 0x63, + OPC_BGE = 0x5063, + OPC_BGEU = 0x7063, + OPC_BLT = 0x4063, + OPC_BLTU = 0x6063, + OPC_BNE = 0x1063, + OPC_DIV = 0x2004033, + OPC_DIVU = 0x2005033, + OPC_JAL = 0x6f, + OPC_JALR = 0x67, + OPC_LB = 0x3, + OPC_LBU = 0x4003, + OPC_LD = 0x3003, + OPC_LH = 0x1003, + OPC_LHU = 0x5003, + OPC_LUI = 0x37, + OPC_LW = 0x2003, + OPC_LWU = 0x6003, + OPC_MUL = 0x2000033, + OPC_MULH = 0x2001033, + OPC_MULHSU = 0x2002033, + OPC_MULHU = 0x2003033, + OPC_OR = 0x6033, + OPC_ORI = 0x6013, + OPC_REM = 0x2006033, + OPC_REMU = 0x2007033, + OPC_SB = 0x23, + OPC_SD = 0x3023, + OPC_SH = 0x1023, + OPC_SLL = 0x1033, + OPC_SLLI = 0x1013, + OPC_SLT = 0x2033, + OPC_SLTI = 0x2013, + OPC_SLTIU = 0x3013, + OPC_SLTU = 0x3033, + OPC_SRA = 0x40005033, + OPC_SRAI = 0x40005013, + OPC_SRL = 0x5033, + OPC_SRLI = 0x5013, + OPC_SUB = 0x40000033, + OPC_SW = 0x2023, + OPC_XOR = 0x4033, + OPC_XORI = 0x4013, + +#if TCG_TARGET_REG_BITS == 64 + OPC_ADDIW = 0x1b, + OPC_ADDW = 0x3b, + OPC_DIVUW = 0x200503b, + OPC_DIVW = 0x200403b, + OPC_MULW = 0x200003b, + OPC_REMUW = 0x200703b, + OPC_REMW = 0x200603b, + OPC_SLLIW = 0x101b, + OPC_SLLW = 0x103b, + OPC_SRAIW = 0x4000501b, + OPC_SRAW = 0x4000503b, + OPC_SRLIW = 0x501b, + OPC_SRLW = 0x503b, + OPC_SUBW = 0x4000003b, +#else + /* Simplify code throughout by defining aliases for RV32. */ + OPC_ADDIW = OPC_ADDI, + OPC_ADDW = OPC_ADD, + OPC_DIVUW = OPC_DIVU, + OPC_DIVW = OPC_DIV, + OPC_MULW = OPC_MUL, + OPC_REMUW = OPC_REMU, + OPC_REMW = OPC_REM, + OPC_SLLIW = OPC_SLLI, + OPC_SLLW = OPC_SLL, + OPC_SRAIW = OPC_SRAI, + OPC_SRAW = OPC_SRA, + OPC_SRLIW = OPC_SRLI, + OPC_SRLW = OPC_SRL, + OPC_SUBW = OPC_SUB, +#endif + + OPC_FENCE = 0x0000000f, +} RISCVInsn; From 54a9ce0f683ab69b0a16e59f5ef28c1ded604def Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:17:43 +0000 Subject: [PATCH 08/42] tcg/riscv: Add the immediate encoders Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index f853d01803..08838027cd 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -284,3 +284,93 @@ typedef enum { OPC_FENCE = 0x0000000f, } RISCVInsn; + +/* + * RISC-V immediate and instruction encoders (excludes 16-bit RVC) + */ + +/* Type-R */ + +static int32_t encode_r(RISCVInsn opc, TCGReg rd, TCGReg rs1, TCGReg rs2) +{ + return opc | (rd & 0x1f) << 7 | (rs1 & 0x1f) << 15 | (rs2 & 0x1f) << 20; +} + +/* Type-I */ + +static int32_t encode_imm12(uint32_t imm) +{ + return (imm & 0xfff) << 20; +} + +static int32_t encode_i(RISCVInsn opc, TCGReg rd, TCGReg rs1, uint32_t imm) +{ + return opc | (rd & 0x1f) << 7 | (rs1 & 0x1f) << 15 | encode_imm12(imm); +} + +/* Type-S */ + +static int32_t encode_simm12(uint32_t imm) +{ + int32_t ret = 0; + + ret |= (imm & 0xFE0) << 20; + ret |= (imm & 0x1F) << 7; + + return ret; +} + +static int32_t encode_s(RISCVInsn opc, TCGReg rs1, TCGReg rs2, uint32_t imm) +{ + return opc | (rs1 & 0x1f) << 15 | (rs2 & 0x1f) << 20 | encode_simm12(imm); +} + +/* Type-SB */ + +static int32_t encode_sbimm12(uint32_t imm) +{ + int32_t ret = 0; + + ret |= (imm & 0x1000) << 19; + ret |= (imm & 0x7e0) << 20; + ret |= (imm & 0x1e) << 7; + ret |= (imm & 0x800) >> 4; + + return ret; +} + +static int32_t encode_sb(RISCVInsn opc, TCGReg rs1, TCGReg rs2, uint32_t imm) +{ + return opc | (rs1 & 0x1f) << 15 | (rs2 & 0x1f) << 20 | encode_sbimm12(imm); +} + +/* Type-U */ + +static int32_t encode_uimm20(uint32_t imm) +{ + return imm & 0xfffff000; +} + +static int32_t encode_u(RISCVInsn opc, TCGReg rd, uint32_t imm) +{ + return opc | (rd & 0x1f) << 7 | encode_uimm20(imm); +} + +/* Type-UJ */ + +static int32_t encode_ujimm20(uint32_t imm) +{ + int32_t ret = 0; + + ret |= (imm & 0x0007fe) << (21 - 1); + ret |= (imm & 0x000800) << (20 - 11); + ret |= (imm & 0x0ff000) << (12 - 12); + ret |= (imm & 0x100000) << (31 - 20); + + return ret; +} + +static int32_t encode_uj(RISCVInsn opc, TCGReg rd, uint32_t imm) +{ + return opc | (rd & 0x1f) << 7 | encode_ujimm20(imm); +} From bedf14e335f5cb1339e9f35834ca7fe49d62a083 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:17:55 +0000 Subject: [PATCH 09/42] tcg/riscv: Add the instruction emitters Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 48 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index 08838027cd..d198cfd5f7 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -374,3 +374,51 @@ static int32_t encode_uj(RISCVInsn opc, TCGReg rd, uint32_t imm) { return opc | (rd & 0x1f) << 7 | encode_ujimm20(imm); } + +/* + * RISC-V instruction emitters + */ + +static void tcg_out_opc_reg(TCGContext *s, RISCVInsn opc, + TCGReg rd, TCGReg rs1, TCGReg rs2) +{ + tcg_out32(s, encode_r(opc, rd, rs1, rs2)); +} + +static void tcg_out_opc_imm(TCGContext *s, RISCVInsn opc, + TCGReg rd, TCGReg rs1, TCGArg imm) +{ + tcg_out32(s, encode_i(opc, rd, rs1, imm)); +} + +static void tcg_out_opc_store(TCGContext *s, RISCVInsn opc, + TCGReg rs1, TCGReg rs2, uint32_t imm) +{ + tcg_out32(s, encode_s(opc, rs1, rs2, imm)); +} + +static void tcg_out_opc_branch(TCGContext *s, RISCVInsn opc, + TCGReg rs1, TCGReg rs2, uint32_t imm) +{ + tcg_out32(s, encode_sb(opc, rs1, rs2, imm)); +} + +static void tcg_out_opc_upper(TCGContext *s, RISCVInsn opc, + TCGReg rd, uint32_t imm) +{ + tcg_out32(s, encode_u(opc, rd, imm)); +} + +static void tcg_out_opc_jump(TCGContext *s, RISCVInsn opc, + TCGReg rd, uint32_t imm) +{ + tcg_out32(s, encode_uj(opc, rd, imm)); +} + +static void tcg_out_nop_fill(tcg_insn_unit *p, int count) +{ + int i; + for (i = 0; i < count; ++i) { + p[i] = encode_i(OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0); + } +} From dfa8e74f946347accb834da0f47126c39394c31f Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:18:06 +0000 Subject: [PATCH 10/42] tcg/riscv: Add the relocation functions Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: <6ac4f4b0d5ea93cb0ee9a3b8b47ee9f7b3711494.1545246859.git.alistair.francis@wdc.com> Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 88 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index d198cfd5f7..a26744052f 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -422,3 +422,91 @@ static void tcg_out_nop_fill(tcg_insn_unit *p, int count) p[i] = encode_i(OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0); } } + +/* + * Relocations + */ + +static bool reloc_sbimm12(tcg_insn_unit *code_ptr, tcg_insn_unit *target) +{ + intptr_t offset = (intptr_t)target - (intptr_t)code_ptr; + + if (offset == sextreg(offset, 1, 12) << 1) { + code_ptr[0] |= encode_sbimm12(offset); + return true; + } + + return false; +} + +static bool reloc_jimm20(tcg_insn_unit *code_ptr, tcg_insn_unit *target) +{ + intptr_t offset = (intptr_t)target - (intptr_t)code_ptr; + + if (offset == sextreg(offset, 1, 20) << 1) { + code_ptr[0] |= encode_ujimm20(offset); + return true; + } + + return false; +} + +static bool reloc_call(tcg_insn_unit *code_ptr, tcg_insn_unit *target) +{ + intptr_t offset = (intptr_t)target - (intptr_t)code_ptr; + int32_t lo = sextreg(offset, 0, 12); + int32_t hi = offset - lo; + + if (offset == hi + lo) { + code_ptr[0] |= encode_uimm20(hi); + code_ptr[1] |= encode_imm12(lo); + return true; + } + + return false; +} + +static bool patch_reloc(tcg_insn_unit *code_ptr, int type, + intptr_t value, intptr_t addend) +{ + uint32_t insn = *code_ptr; + intptr_t diff; + bool short_jmp; + + tcg_debug_assert(addend == 0); + + switch (type) { + case R_RISCV_BRANCH: + diff = value - (uintptr_t)code_ptr; + short_jmp = diff == sextreg(diff, 0, 12); + if (short_jmp) { + return reloc_sbimm12(code_ptr, (tcg_insn_unit *)value); + } else { + /* Invert the condition */ + insn = insn ^ (1 << 12); + /* Clear the offset */ + insn &= 0x01fff07f; + /* Set the offset to the PC + 8 */ + insn |= encode_sbimm12(8); + + /* Move forward */ + code_ptr[0] = insn; + + /* Overwrite the NOP with jal x0,value */ + diff = value - (uintptr_t)(code_ptr + 1); + insn = encode_uj(OPC_JAL, TCG_REG_ZERO, diff); + code_ptr[1] = insn; + + return true; + } + break; + case R_RISCV_JAL: + return reloc_jimm20(code_ptr, (tcg_insn_unit *)value); + break; + case R_RISCV_CALL: + return reloc_call(code_ptr, (tcg_insn_unit *)value); + break; + default: + tcg_abort(); + } +} From 6cd2eda39f648ed398b0dca69cd7aaf6be50e30d Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:18:17 +0000 Subject: [PATCH 11/42] tcg/riscv: Add the mov and movi instruction Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 86 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index a26744052f..01b4443d6d 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -510,3 +510,89 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type, tcg_abort(); } } + +/* + * TCG intrinsics + */ + +static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) +{ + if (ret == arg) { + return; + } + switch (type) { + case TCG_TYPE_I32: + case TCG_TYPE_I64: + tcg_out_opc_imm(s, OPC_ADDI, ret, arg, 0); + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, + tcg_target_long val) +{ + tcg_target_long lo, hi, tmp; + int shift, ret; + + if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { + val = (int32_t)val; + } + + lo = sextreg(val, 0, 12); + if (val == lo) { + tcg_out_opc_imm(s, OPC_ADDI, rd, TCG_REG_ZERO, lo); + return; + } + + hi = val - lo; + if (TCG_TARGET_REG_BITS == 32 || val == (int32_t)val) { + tcg_out_opc_upper(s, OPC_LUI, rd, hi); + if (lo != 0) { + tcg_out_opc_imm(s, OPC_ADDIW, rd, rd, lo); + } + return; + } + + /* We can only be here if TCG_TARGET_REG_BITS != 32 */ + tmp = tcg_pcrel_diff(s, (void *)val); + if (tmp == (int32_t)tmp) { + tcg_out_opc_upper(s, OPC_AUIPC, rd, 0); + tcg_out_opc_imm(s, OPC_ADDI, rd, rd, 0); + ret = reloc_call(s->code_ptr - 2, (tcg_insn_unit *)val); + tcg_debug_assert(ret == true); + return; + } + + /* Look for a single 20-bit section. */ + shift = ctz64(val); + tmp = val >> shift; + if (tmp == sextreg(tmp, 0, 20)) { + tcg_out_opc_upper(s, OPC_LUI, rd, tmp << 12); + if (shift > 12) { + tcg_out_opc_imm(s, OPC_SLLI, rd, rd, shift - 12); + } else { + tcg_out_opc_imm(s, OPC_SRAI, rd, rd, 12 - shift); + } + return; + } + + /* Look for a few high zero bits, with lots of bits set in the middle. */ + shift = clz64(val); + tmp = val << shift; + if (tmp == sextreg(tmp, 12, 20) << 12) { + tcg_out_opc_upper(s, OPC_LUI, rd, tmp); + tcg_out_opc_imm(s, OPC_SRLI, rd, rd, shift); + return; + } else if (tmp == sextreg(tmp, 0, 12)) { + tcg_out_opc_imm(s, OPC_ADDI, rd, TCG_REG_ZERO, tmp); + tcg_out_opc_imm(s, OPC_SRLI, rd, rd, shift); + return; + } + + /* Drop into the constant pool. */ + new_pool_label(s, val, R_RISCV_CALL, s->code_ptr, 0); + tcg_out_opc_upper(s, OPC_AUIPC, rd, 0); + tcg_out_opc_imm(s, OPC_LD, rd, rd, 0); +} From 27fd64144bebc34a815fd8a69af790b080e80c61 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:18:28 +0000 Subject: [PATCH 12/42] tcg/riscv: Add the extract instructions Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index 01b4443d6d..48399480b2 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -596,3 +596,37 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, tcg_out_opc_upper(s, OPC_AUIPC, rd, 0); tcg_out_opc_imm(s, OPC_LD, rd, rd, 0); } + +static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_out_opc_imm(s, OPC_ANDI, ret, arg, 0xff); +} + +static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); + tcg_out_opc_imm(s, OPC_SRLIW, ret, ret, 16); +} + +static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_out_opc_imm(s, OPC_SLLI, ret, arg, 32); + tcg_out_opc_imm(s, OPC_SRLI, ret, ret, 32); +} + +static void tcg_out_ext8s(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 24); + tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 24); +} + +static void tcg_out_ext16s(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); + tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 16); +} + +static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_out_opc_imm(s, OPC_ADDIW, ret, arg, 0); +} From 61535d49881a93b9c765eef0290a47f310eb9067 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:18:39 +0000 Subject: [PATCH 13/42] tcg/riscv: Add the out load and store instructions Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 65 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index 48399480b2..65718df7ad 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -630,3 +630,68 @@ static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg) { tcg_out_opc_imm(s, OPC_ADDIW, ret, arg, 0); } + +static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data, + TCGReg addr, intptr_t offset) +{ + intptr_t imm12 = sextreg(offset, 0, 12); + + if (offset != imm12) { + intptr_t diff = offset - (uintptr_t)s->code_ptr; + + if (addr == TCG_REG_ZERO && diff == (int32_t)diff) { + imm12 = sextreg(diff, 0, 12); + tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP2, diff - imm12); + } else { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP2, offset - imm12); + if (addr != TCG_REG_ZERO) { + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, addr); + } + } + addr = TCG_REG_TMP2; + } + + switch (opc) { + case OPC_SB: + case OPC_SH: + case OPC_SW: + case OPC_SD: + tcg_out_opc_store(s, opc, addr, data, imm12); + break; + case OPC_LB: + case OPC_LBU: + case OPC_LH: + case OPC_LHU: + case OPC_LW: + case OPC_LWU: + case OPC_LD: + tcg_out_opc_imm(s, opc, data, addr, imm12); + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg, + TCGReg arg1, intptr_t arg2) +{ + bool is32bit = (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32); + tcg_out_ldst(s, is32bit ? OPC_LW : OPC_LD, arg, arg1, arg2); +} + +static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, + TCGReg arg1, intptr_t arg2) +{ + bool is32bit = (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32); + tcg_out_ldst(s, is32bit ? OPC_SW : OPC_SD, arg, arg1, arg2); +} + +static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, + TCGReg base, intptr_t ofs) +{ + if (val == 0) { + tcg_out_st(s, type, TCG_REG_ZERO, base, ofs); + return true; + } + return false; +} From 28ca738e9db89abffe329e8924a141b90c1b91e1 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:18:49 +0000 Subject: [PATCH 14/42] tcg/riscv: Add the add2 and sub2 instructions Signed-off-by: Alistair Francis Reviewed-by: Richard Henderson Message-Id: <5665a57809e32b35775e8e98fdab898853af37b8.1545246859.git.alistair.francis@wdc.com> Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 55 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index 65718df7ad..5da850b957 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -695,3 +695,58 @@ static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, } return false; } + +static void tcg_out_addsub2(TCGContext *s, + TCGReg rl, TCGReg rh, + TCGReg al, TCGReg ah, + TCGArg bl, TCGArg bh, + bool cbl, bool cbh, bool is_sub, bool is32bit) +{ + const RISCVInsn opc_add = is32bit ? OPC_ADDW : OPC_ADD; + const RISCVInsn opc_addi = is32bit ? OPC_ADDIW : OPC_ADDI; + const RISCVInsn opc_sub = is32bit ? OPC_SUBW : OPC_SUB; + TCGReg th = TCG_REG_TMP1; + + /* If we have a negative constant such that negating it would + make the high part zero, we can (usually) eliminate one insn. */ + if (cbl && cbh && bh == -1 && bl != 0) { + bl = -bl; + bh = 0; + is_sub = !is_sub; + } + + /* By operating on the high part first, we get to use the final + carry operation to move back from the temporary. */ + if (!cbh) { + tcg_out_opc_reg(s, (is_sub ? opc_sub : opc_add), th, ah, bh); + } else if (bh != 0 || ah == rl) { + tcg_out_opc_imm(s, opc_addi, th, ah, (is_sub ? -bh : bh)); + } else { + th = ah; + } + + /* Note that tcg optimization should eliminate the bl == 0 case. */ + if (is_sub) { + if (cbl) { + tcg_out_opc_imm(s, OPC_SLTIU, TCG_REG_TMP0, al, bl); + tcg_out_opc_imm(s, opc_addi, rl, al, -bl); + } else { + tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_TMP0, al, bl); + tcg_out_opc_reg(s, opc_sub, rl, al, bl); + } + tcg_out_opc_reg(s, opc_sub, rh, th, TCG_REG_TMP0); + } else { + if (cbl) { + tcg_out_opc_imm(s, opc_addi, rl, al, bl); + tcg_out_opc_imm(s, OPC_SLTIU, TCG_REG_TMP0, rl, bl); + } else if (rl == al && rl == bl) { + tcg_out_opc_imm(s, OPC_SLTI, TCG_REG_TMP0, al, 0); + tcg_out_opc_reg(s, opc_addi, rl, al, bl); + } else { + tcg_out_opc_reg(s, opc_add, rl, al, bl); + tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_TMP0, + rl, (rl == bl ? al : bl)); + } + tcg_out_opc_reg(s, opc_add, rh, th, TCG_REG_TMP0); + } +} From 15840069e17e133e0295c183565d0101334b8476 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:19:00 +0000 Subject: [PATCH 15/42] tcg/riscv: Add branch and jump instructions Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 145 +++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index 5da850b957..ecc76c9ef8 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -750,3 +750,148 @@ static void tcg_out_addsub2(TCGContext *s, tcg_out_opc_reg(s, opc_add, rh, th, TCG_REG_TMP0); } } + +static const struct { + RISCVInsn op; + bool swap; +} tcg_brcond_to_riscv[] = { + [TCG_COND_EQ] = { OPC_BEQ, false }, + [TCG_COND_NE] = { OPC_BNE, false }, + [TCG_COND_LT] = { OPC_BLT, false }, + [TCG_COND_GE] = { OPC_BGE, false }, + [TCG_COND_LE] = { OPC_BGE, true }, + [TCG_COND_GT] = { OPC_BLT, true }, + [TCG_COND_LTU] = { OPC_BLTU, false }, + [TCG_COND_GEU] = { OPC_BGEU, false }, + [TCG_COND_LEU] = { OPC_BGEU, true }, + [TCG_COND_GTU] = { OPC_BLTU, true } +}; + +static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, + TCGReg arg2, TCGLabel *l) +{ + RISCVInsn op = tcg_brcond_to_riscv[cond].op; + + tcg_debug_assert(op != 0); + + if (tcg_brcond_to_riscv[cond].swap) { + TCGReg t = arg1; + arg1 = arg2; + arg2 = t; + } + + if (l->has_value) { + intptr_t diff = tcg_pcrel_diff(s, l->u.value_ptr); + if (diff == sextreg(diff, 0, 12)) { + tcg_out_opc_branch(s, op, arg1, arg2, diff); + } else { + /* Invert the conditional branch. */ + tcg_out_opc_branch(s, op ^ (1 << 12), arg1, arg2, 8); + tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, diff - 4); + } + } else { + tcg_out_reloc(s, s->code_ptr, R_RISCV_BRANCH, l, 0); + tcg_out_opc_branch(s, op, arg1, arg2, 0); + /* NOP to allow patching later */ + tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0); + } +} + +static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg arg1, TCGReg arg2) +{ + switch (cond) { + case TCG_COND_EQ: + tcg_out_opc_reg(s, OPC_SUB, ret, arg1, arg2); + tcg_out_opc_imm(s, OPC_SLTIU, ret, ret, 1); + break; + case TCG_COND_NE: + tcg_out_opc_reg(s, OPC_SUB, ret, arg1, arg2); + tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, ret); + break; + case TCG_COND_LT: + tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2); + break; + case TCG_COND_GE: + tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2); + tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); + break; + case TCG_COND_LE: + tcg_out_opc_reg(s, OPC_SLT, ret, arg2, arg1); + tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); + break; + case TCG_COND_GT: + tcg_out_opc_reg(s, OPC_SLT, ret, arg2, arg1); + break; + case TCG_COND_LTU: + tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2); + break; + case TCG_COND_GEU: + tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2); + tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); + break; + case TCG_COND_LEU: + tcg_out_opc_reg(s, OPC_SLTU, ret, arg2, arg1); + tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); + break; + case TCG_COND_GTU: + tcg_out_opc_reg(s, OPC_SLTU, ret, arg2, arg1); + break; + default: + g_assert_not_reached(); + break; + } +} + +static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, + TCGReg bl, TCGReg bh, TCGLabel *l) +{ + /* todo */ + g_assert_not_reached(); +} + +static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh) +{ + /* todo */ + g_assert_not_reached(); +} + +static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target) +{ + ptrdiff_t offset = tcg_pcrel_diff(s, target); + tcg_debug_assert(offset == sextreg(offset, 1, 20) << 1); + tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, offset); +} + +static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail) +{ + TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA; + ptrdiff_t offset = tcg_pcrel_diff(s, arg); + int ret; + + if (offset == sextreg(offset, 1, 20) << 1) { + /* short jump: -2097150 to 2097152 */ + tcg_out_opc_jump(s, OPC_JAL, link, offset); + } else if (TCG_TARGET_REG_BITS == 32 || + offset == sextreg(offset, 1, 31) << 1) { + /* long jump: -2147483646 to 2147483648 */ + tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP0, 0); + tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, 0); + ret = reloc_call(s->code_ptr - 2, arg);\ + tcg_debug_assert(ret == true); + } else if (TCG_TARGET_REG_BITS == 64) { + /* far jump: 64-bit */ + tcg_target_long imm = sextreg((tcg_target_long)arg, 0, 12); + tcg_target_long base = (tcg_target_long)arg - imm; + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, base); + tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, imm); + } else { + g_assert_not_reached(); + } +} + +static void tcg_out_call(TCGContext *s, tcg_insn_unit *arg) +{ + tcg_out_call_int(s, arg, false); +} From efbea94c76b25d9655900e744799ad542527a45f Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:19:11 +0000 Subject: [PATCH 16/42] tcg/riscv: Add slowpath load and store instructions Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: <1a0a7e8f3347764f212c5efa5c07c9be17efdec6.1545246859.git.alistair.francis@wdc.com> Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 256 +++++++++++++++++++++++++++++++++++++ 1 file changed, 256 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index ecc76c9ef8..7216bad086 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -895,3 +895,259 @@ static void tcg_out_call(TCGContext *s, tcg_insn_unit *arg) { tcg_out_call_int(s, arg, false); } + +static void tcg_out_mb(TCGContext *s, TCGArg a0) +{ + tcg_insn_unit insn = OPC_FENCE; + + if (a0 & TCG_MO_LD_LD) { + insn |= 0x02200000; + } + if (a0 & TCG_MO_ST_LD) { + insn |= 0x01200000; + } + if (a0 & TCG_MO_LD_ST) { + insn |= 0x02100000; + } + if (a0 & TCG_MO_ST_ST) { + insn |= 0x02200000; + } + tcg_out32(s, insn); +} + +/* + * Load/store and TLB + */ + +#if defined(CONFIG_SOFTMMU) +#include "tcg-ldst.inc.c" + +/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, + * TCGMemOpIdx oi, uintptr_t ra) + */ +static void * const qemu_ld_helpers[16] = { + [MO_UB] = helper_ret_ldub_mmu, + [MO_SB] = helper_ret_ldsb_mmu, + [MO_LEUW] = helper_le_lduw_mmu, + [MO_LESW] = helper_le_ldsw_mmu, + [MO_LEUL] = helper_le_ldul_mmu, +#if TCG_TARGET_REG_BITS == 64 + [MO_LESL] = helper_le_ldsl_mmu, +#endif + [MO_LEQ] = helper_le_ldq_mmu, + [MO_BEUW] = helper_be_lduw_mmu, + [MO_BESW] = helper_be_ldsw_mmu, + [MO_BEUL] = helper_be_ldul_mmu, +#if TCG_TARGET_REG_BITS == 64 + [MO_BESL] = helper_be_ldsl_mmu, +#endif + [MO_BEQ] = helper_be_ldq_mmu, +}; + +/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, + * uintxx_t val, TCGMemOpIdx oi, + * uintptr_t ra) + */ +static void * const qemu_st_helpers[16] = { + [MO_UB] = helper_ret_stb_mmu, + [MO_LEUW] = helper_le_stw_mmu, + [MO_LEUL] = helper_le_stl_mmu, + [MO_LEQ] = helper_le_stq_mmu, + [MO_BEUW] = helper_be_stw_mmu, + [MO_BEUL] = helper_be_stl_mmu, + [MO_BEQ] = helper_be_stq_mmu, +}; + +static void tcg_out_tlb_load(TCGContext *s, TCGReg addrl, + TCGReg addrh, TCGMemOpIdx oi, + tcg_insn_unit **label_ptr, bool is_load) +{ + TCGMemOp opc = get_memop(oi); + unsigned s_bits = opc & MO_SIZE; + unsigned a_bits = get_alignment_bits(opc); + target_ulong mask; + int mem_index = get_mmuidx(oi); + int cmp_off + = (is_load + ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read) + : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write)); + int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend); + RISCVInsn load_cmp_op = (TARGET_LONG_BITS == 64 ? OPC_LD : + TCG_TARGET_REG_BITS == 64 ? OPC_LWU : OPC_LW); + RISCVInsn load_add_op = TCG_TARGET_REG_BITS == 64 ? OPC_LD : OPC_LW; + TCGReg base = TCG_AREG0; + + /* We don't support oversize guests */ + if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { + g_assert_not_reached(); + } + + /* We don't support unaligned accesses. */ + if (a_bits < s_bits) { + a_bits = s_bits; + } + mask = (target_ulong)TARGET_PAGE_MASK | ((1 << a_bits) - 1); + + + /* Compensate for very large offsets. */ + if (add_off >= 0x1000) { + int adj; + base = TCG_REG_TMP2; + if (cmp_off <= 2 * 0xfff) { + adj = 0xfff; + tcg_out_opc_imm(s, OPC_ADDI, base, TCG_AREG0, adj); + } else { + adj = cmp_off - sextreg(cmp_off, 0, 12); + tcg_debug_assert(add_off - adj >= -0x1000 + && add_off - adj < 0x1000); + + tcg_out_opc_upper(s, OPC_LUI, base, adj); + tcg_out_opc_reg(s, OPC_ADD, base, base, TCG_AREG0); + } + add_off -= adj; + cmp_off -= adj; + } + + /* Extract the page index. */ + if (CPU_TLB_BITS + CPU_TLB_ENTRY_BITS < 12) { + tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP0, addrl, + TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); + tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP0, TCG_REG_TMP0, + MAKE_64BIT_MASK(CPU_TLB_ENTRY_BITS, CPU_TLB_BITS)); + } else if (TARGET_PAGE_BITS >= 12) { + tcg_out_opc_upper(s, OPC_LUI, TCG_REG_TMP0, + MAKE_64BIT_MASK(TARGET_PAGE_BITS, CPU_TLB_BITS)); + tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP0, TCG_REG_TMP0, addrl); + tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP0, TCG_REG_TMP0, + CPU_TLB_BITS - CPU_TLB_ENTRY_BITS); + } else { + tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP0, addrl, TARGET_PAGE_BITS); + tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP0, TCG_REG_TMP0, + MAKE_64BIT_MASK(0, CPU_TLB_BITS)); + tcg_out_opc_imm(s, OPC_SLLI, TCG_REG_TMP0, TCG_REG_TMP0, + CPU_TLB_ENTRY_BITS); + } + + /* Add that to the base address to index the tlb. */ + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, base, TCG_REG_TMP0); + base = TCG_REG_TMP2; + + /* Load the tlb comparator and the addend. */ + tcg_out_ldst(s, load_cmp_op, TCG_REG_TMP0, base, cmp_off); + tcg_out_ldst(s, load_add_op, TCG_REG_TMP2, base, add_off); + + /* Clear the non-page, non-alignment bits from the address. */ + if (mask == sextreg(mask, 0, 12)) { + tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addrl, mask); + } else { + tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP1, mask); + tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addrl); + } + + /* Compare masked address with the TLB entry. */ + label_ptr[0] = s->code_ptr; + tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); + /* NOP to allow patching later */ + tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0); + /* TODO: Move this out of line + * see: + * https://lists.nongnu.org/archive/html/qemu-devel/2018-11/msg02234.html + */ + + /* TLB Hit - translate address using addend. */ + if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + tcg_out_ext32u(s, TCG_REG_TMP0, addrl); + addrl = TCG_REG_TMP0; + } + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addrl); +} + +static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi, + TCGType ext, + TCGReg datalo, TCGReg datahi, + TCGReg addrlo, TCGReg addrhi, + void *raddr, tcg_insn_unit **label_ptr) +{ + TCGLabelQemuLdst *label = new_ldst_label(s); + + label->is_ld = is_ld; + label->oi = oi; + label->type = ext; + label->datalo_reg = datalo; + label->datahi_reg = datahi; + label->addrlo_reg = addrlo; + label->addrhi_reg = addrhi; + label->raddr = raddr; + label->label_ptr[0] = label_ptr[0]; +} + +static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + TCGMemOpIdx oi = l->oi; + TCGMemOp opc = get_memop(oi); + TCGReg a0 = tcg_target_call_iarg_regs[0]; + TCGReg a1 = tcg_target_call_iarg_regs[1]; + TCGReg a2 = tcg_target_call_iarg_regs[2]; + TCGReg a3 = tcg_target_call_iarg_regs[3]; + + /* We don't support oversize guests */ + if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { + g_assert_not_reached(); + } + + /* resolve label address */ + patch_reloc(l->label_ptr[0], R_RISCV_BRANCH, (intptr_t) s->code_ptr, 0); + + /* call load helper */ + tcg_out_mov(s, TCG_TYPE_PTR, a0, TCG_AREG0); + tcg_out_mov(s, TCG_TYPE_PTR, a1, l->addrlo_reg); + tcg_out_movi(s, TCG_TYPE_PTR, a2, oi); + tcg_out_movi(s, TCG_TYPE_PTR, a3, (tcg_target_long)l->raddr); + + tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SSIZE)]); + tcg_out_mov(s, (opc & MO_SIZE) == MO_64, l->datalo_reg, a0); + + tcg_out_goto(s, l->raddr); +} + +static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) +{ + TCGMemOpIdx oi = l->oi; + TCGMemOp opc = get_memop(oi); + TCGMemOp s_bits = opc & MO_SIZE; + TCGReg a0 = tcg_target_call_iarg_regs[0]; + TCGReg a1 = tcg_target_call_iarg_regs[1]; + TCGReg a2 = tcg_target_call_iarg_regs[2]; + TCGReg a3 = tcg_target_call_iarg_regs[3]; + TCGReg a4 = tcg_target_call_iarg_regs[4]; + + /* We don't support oversize guests */ + if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { + g_assert_not_reached(); + } + + /* resolve label address */ + patch_reloc(l->label_ptr[0], R_RISCV_BRANCH, (intptr_t) s->code_ptr, 0); + + /* call store helper */ + tcg_out_mov(s, TCG_TYPE_PTR, a0, TCG_AREG0); + tcg_out_mov(s, TCG_TYPE_PTR, a1, l->addrlo_reg); + tcg_out_mov(s, TCG_TYPE_PTR, a2, l->datalo_reg); + switch (s_bits) { + case MO_8: + tcg_out_ext8u(s, a2, a2); + break; + case MO_16: + tcg_out_ext16u(s, a2, a2); + break; + default: + break; + } + tcg_out_movi(s, TCG_TYPE_PTR, a3, oi); + tcg_out_movi(s, TCG_TYPE_PTR, a4, (tcg_target_long)l->raddr); + + tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SSIZE)]); + + tcg_out_goto(s, l->raddr); +} +#endif /* CONFIG_SOFTMMU */ From 03a7d0213d6b6ffd0781dbd0c41ba73160bec9a1 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:19:20 +0000 Subject: [PATCH 17/42] tcg/riscv: Add direct load and store instructions Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: <2e047a95c39c007c66cda024c095e29b0ac4c43e.1545246859.git.alistair.francis@wdc.com> Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 158 +++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index 7216bad086..154315787c 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -1151,3 +1151,161 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) tcg_out_goto(s, l->raddr); } #endif /* CONFIG_SOFTMMU */ + +static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, + TCGReg base, TCGMemOp opc, bool is_64) +{ + const TCGMemOp bswap = opc & MO_BSWAP; + + /* We don't yet handle byteswapping, assert */ + g_assert(!bswap); + + switch (opc & (MO_SSIZE)) { + case MO_UB: + tcg_out_opc_imm(s, OPC_LBU, lo, base, 0); + break; + case MO_SB: + tcg_out_opc_imm(s, OPC_LB, lo, base, 0); + break; + case MO_UW: + tcg_out_opc_imm(s, OPC_LHU, lo, base, 0); + break; + case MO_SW: + tcg_out_opc_imm(s, OPC_LH, lo, base, 0); + break; + case MO_UL: + if (TCG_TARGET_REG_BITS == 64 && is_64) { + tcg_out_opc_imm(s, OPC_LWU, lo, base, 0); + break; + } + /* FALLTHRU */ + case MO_SL: + tcg_out_opc_imm(s, OPC_LW, lo, base, 0); + break; + case MO_Q: + /* Prefer to load from offset 0 first, but allow for overlap. */ + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_opc_imm(s, OPC_LD, lo, base, 0); + } else if (lo != base) { + tcg_out_opc_imm(s, OPC_LW, lo, base, 0); + tcg_out_opc_imm(s, OPC_LW, hi, base, 4); + } else { + tcg_out_opc_imm(s, OPC_LW, hi, base, 4); + tcg_out_opc_imm(s, OPC_LW, lo, base, 0); + } + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) +{ + TCGReg addr_regl, addr_regh __attribute__((unused)); + TCGReg data_regl, data_regh; + TCGMemOpIdx oi; + TCGMemOp opc; +#if defined(CONFIG_SOFTMMU) + tcg_insn_unit *label_ptr[1]; +#endif + TCGReg base = TCG_REG_TMP0; + + data_regl = *args++; + data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); + addr_regl = *args++; + addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); + oi = *args++; + opc = get_memop(oi); + +#if defined(CONFIG_SOFTMMU) + tcg_out_tlb_load(s, addr_regl, addr_regh, oi, label_ptr, 1); + tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); + add_qemu_ldst_label(s, 1, oi, + (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), + data_regl, data_regh, addr_regl, addr_regh, + s->code_ptr, label_ptr); +#else + if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + tcg_out_ext32u(s, base, addr_regl); + addr_regl = base; + } + + if (guest_base == 0) { + tcg_out_opc_reg(s, OPC_ADD, base, addr_regl, TCG_REG_ZERO); + } else { + tcg_out_opc_reg(s, OPC_ADD, base, TCG_GUEST_BASE_REG, addr_regl); + } + tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); +#endif +} + +static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, + TCGReg base, TCGMemOp opc) +{ + const TCGMemOp bswap = opc & MO_BSWAP; + + /* We don't yet handle byteswapping, assert */ + g_assert(!bswap); + + switch (opc & (MO_SSIZE)) { + case MO_8: + tcg_out_opc_store(s, OPC_SB, base, lo, 0); + break; + case MO_16: + tcg_out_opc_store(s, OPC_SH, base, lo, 0); + break; + case MO_32: + tcg_out_opc_store(s, OPC_SW, base, lo, 0); + break; + case MO_64: + if (TCG_TARGET_REG_BITS == 64) { + tcg_out_opc_store(s, OPC_SD, base, lo, 0); + } else { + tcg_out_opc_store(s, OPC_SW, base, lo, 0); + tcg_out_opc_store(s, OPC_SW, base, hi, 4); + } + break; + default: + g_assert_not_reached(); + } +} + +static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) +{ + TCGReg addr_regl, addr_regh __attribute__((unused)); + TCGReg data_regl, data_regh; + TCGMemOpIdx oi; + TCGMemOp opc; +#if defined(CONFIG_SOFTMMU) + tcg_insn_unit *label_ptr[1]; +#endif + TCGReg base = TCG_REG_TMP0; + + data_regl = *args++; + data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); + addr_regl = *args++; + addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); + oi = *args++; + opc = get_memop(oi); + +#if defined(CONFIG_SOFTMMU) + tcg_out_tlb_load(s, addr_regl, addr_regh, oi, label_ptr, 0); + tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); + add_qemu_ldst_label(s, 0, oi, + (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), + data_regl, data_regh, addr_regl, addr_regh, + s->code_ptr, label_ptr); +#else + if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { + tcg_out_ext32u(s, base, addr_regl); + addr_regl = base; + } + + if (guest_base == 0) { + tcg_out_opc_reg(s, OPC_ADD, base, addr_regl, TCG_REG_ZERO); + } else { + tcg_out_opc_reg(s, OPC_ADD, base, TCG_GUEST_BASE_REG, addr_regl); + } + tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); +#endif +} From bdf503819ee188765f0249cb4507c3b2646ae884 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:19:29 +0000 Subject: [PATCH 18/42] tcg/riscv: Add the out op decoder Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: <7c47f00cb4a9a777120456e0704b4076a5d943ab.1545246859.git.alistair.francis@wdc.com> Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 496 +++++++++++++++++++++++++++++++++++++ 1 file changed, 496 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index 154315787c..014c5287f5 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -1309,3 +1309,499 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); #endif } + +static tcg_insn_unit *tb_ret_addr; + +static void tcg_out_op(TCGContext *s, TCGOpcode opc, + const TCGArg *args, const int *const_args) +{ + TCGArg a0 = args[0]; + TCGArg a1 = args[1]; + TCGArg a2 = args[2]; + int c2 = const_args[2]; + + switch (opc) { + case INDEX_op_exit_tb: + /* Reuse the zeroing that exists for goto_ptr. */ + if (a0 == 0) { + tcg_out_call_int(s, s->code_gen_epilogue, true); + } else { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); + tcg_out_call_int(s, tb_ret_addr, true); + } + break; + + case INDEX_op_goto_tb: + assert(s->tb_jmp_insn_offset == 0); + /* indirect jump method */ + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, + (uintptr_t)(s->tb_jmp_target_addr + a0)); + tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0); + set_jmp_reset_offset(s, a0); + break; + + case INDEX_op_goto_ptr: + tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, a0, 0); + break; + + case INDEX_op_br: + tcg_out_reloc(s, s->code_ptr, R_RISCV_JAL, arg_label(a0), 0); + tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0); + break; + + case INDEX_op_ld8u_i32: + case INDEX_op_ld8u_i64: + tcg_out_ldst(s, OPC_LBU, a0, a1, a2); + break; + case INDEX_op_ld8s_i32: + case INDEX_op_ld8s_i64: + tcg_out_ldst(s, OPC_LB, a0, a1, a2); + break; + case INDEX_op_ld16u_i32: + case INDEX_op_ld16u_i64: + tcg_out_ldst(s, OPC_LHU, a0, a1, a2); + break; + case INDEX_op_ld16s_i32: + case INDEX_op_ld16s_i64: + tcg_out_ldst(s, OPC_LH, a0, a1, a2); + break; + case INDEX_op_ld32u_i64: + tcg_out_ldst(s, OPC_LWU, a0, a1, a2); + break; + case INDEX_op_ld_i32: + case INDEX_op_ld32s_i64: + tcg_out_ldst(s, OPC_LW, a0, a1, a2); + break; + case INDEX_op_ld_i64: + tcg_out_ldst(s, OPC_LD, a0, a1, a2); + break; + + case INDEX_op_st8_i32: + case INDEX_op_st8_i64: + tcg_out_ldst(s, OPC_SB, a0, a1, a2); + break; + case INDEX_op_st16_i32: + case INDEX_op_st16_i64: + tcg_out_ldst(s, OPC_SH, a0, a1, a2); + break; + case INDEX_op_st_i32: + case INDEX_op_st32_i64: + tcg_out_ldst(s, OPC_SW, a0, a1, a2); + break; + case INDEX_op_st_i64: + tcg_out_ldst(s, OPC_SD, a0, a1, a2); + break; + + case INDEX_op_add_i32: + if (c2) { + tcg_out_opc_imm(s, OPC_ADDIW, a0, a1, a2); + } else { + tcg_out_opc_reg(s, OPC_ADDW, a0, a1, a2); + } + break; + case INDEX_op_add_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_ADDI, a0, a1, a2); + } else { + tcg_out_opc_reg(s, OPC_ADD, a0, a1, a2); + } + break; + + case INDEX_op_sub_i32: + if (c2) { + tcg_out_opc_imm(s, OPC_ADDIW, a0, a1, -a2); + } else { + tcg_out_opc_reg(s, OPC_SUBW, a0, a1, a2); + } + break; + case INDEX_op_sub_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_ADDI, a0, a1, -a2); + } else { + tcg_out_opc_reg(s, OPC_SUB, a0, a1, a2); + } + break; + + case INDEX_op_and_i32: + case INDEX_op_and_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_ANDI, a0, a1, a2); + } else { + tcg_out_opc_reg(s, OPC_AND, a0, a1, a2); + } + break; + + case INDEX_op_or_i32: + case INDEX_op_or_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_ORI, a0, a1, a2); + } else { + tcg_out_opc_reg(s, OPC_OR, a0, a1, a2); + } + break; + + case INDEX_op_xor_i32: + case INDEX_op_xor_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_XORI, a0, a1, a2); + } else { + tcg_out_opc_reg(s, OPC_XOR, a0, a1, a2); + } + break; + + case INDEX_op_not_i32: + case INDEX_op_not_i64: + tcg_out_opc_imm(s, OPC_XORI, a0, a1, -1); + break; + + case INDEX_op_neg_i32: + tcg_out_opc_reg(s, OPC_SUBW, a0, TCG_REG_ZERO, a1); + break; + case INDEX_op_neg_i64: + tcg_out_opc_reg(s, OPC_SUB, a0, TCG_REG_ZERO, a1); + break; + + case INDEX_op_mul_i32: + tcg_out_opc_reg(s, OPC_MULW, a0, a1, a2); + break; + case INDEX_op_mul_i64: + tcg_out_opc_reg(s, OPC_MUL, a0, a1, a2); + break; + + case INDEX_op_div_i32: + tcg_out_opc_reg(s, OPC_DIVW, a0, a1, a2); + break; + case INDEX_op_div_i64: + tcg_out_opc_reg(s, OPC_DIV, a0, a1, a2); + break; + + case INDEX_op_divu_i32: + tcg_out_opc_reg(s, OPC_DIVUW, a0, a1, a2); + break; + case INDEX_op_divu_i64: + tcg_out_opc_reg(s, OPC_DIVU, a0, a1, a2); + break; + + case INDEX_op_rem_i32: + tcg_out_opc_reg(s, OPC_REMW, a0, a1, a2); + break; + case INDEX_op_rem_i64: + tcg_out_opc_reg(s, OPC_REM, a0, a1, a2); + break; + + case INDEX_op_remu_i32: + tcg_out_opc_reg(s, OPC_REMUW, a0, a1, a2); + break; + case INDEX_op_remu_i64: + tcg_out_opc_reg(s, OPC_REMU, a0, a1, a2); + break; + + case INDEX_op_shl_i32: + if (c2) { + tcg_out_opc_imm(s, OPC_SLLIW, a0, a1, a2); + } else { + tcg_out_opc_reg(s, OPC_SLLW, a0, a1, a2); + } + break; + case INDEX_op_shl_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_SLLI, a0, a1, a2); + } else { + tcg_out_opc_reg(s, OPC_SLL, a0, a1, a2); + } + break; + + case INDEX_op_shr_i32: + if (c2) { + tcg_out_opc_imm(s, OPC_SRLIW, a0, a1, a2); + } else { + tcg_out_opc_reg(s, OPC_SRLW, a0, a1, a2); + } + break; + case INDEX_op_shr_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_SRLI, a0, a1, a2); + } else { + tcg_out_opc_reg(s, OPC_SRL, a0, a1, a2); + } + break; + + case INDEX_op_sar_i32: + if (c2) { + tcg_out_opc_imm(s, OPC_SRAIW, a0, a1, a2); + } else { + tcg_out_opc_reg(s, OPC_SRAW, a0, a1, a2); + } + break; + case INDEX_op_sar_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_SRAI, a0, a1, a2); + } else { + tcg_out_opc_reg(s, OPC_SRA, a0, a1, a2); + } + break; + + case INDEX_op_add2_i32: + tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], + const_args[4], const_args[5], false, true); + break; + case INDEX_op_add2_i64: + tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], + const_args[4], const_args[5], false, false); + break; + case INDEX_op_sub2_i32: + tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], + const_args[4], const_args[5], true, true); + break; + case INDEX_op_sub2_i64: + tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], + const_args[4], const_args[5], true, false); + break; + + case INDEX_op_brcond_i32: + case INDEX_op_brcond_i64: + tcg_out_brcond(s, a2, a0, a1, arg_label(args[3])); + break; + case INDEX_op_brcond2_i32: + tcg_out_brcond2(s, args[4], a0, a1, a2, args[3], arg_label(args[5])); + break; + + case INDEX_op_setcond_i32: + case INDEX_op_setcond_i64: + tcg_out_setcond(s, args[3], a0, a1, a2); + break; + case INDEX_op_setcond2_i32: + tcg_out_setcond2(s, args[5], a0, a1, a2, args[3], args[4]); + break; + + case INDEX_op_qemu_ld_i32: + tcg_out_qemu_ld(s, args, false); + break; + case INDEX_op_qemu_ld_i64: + tcg_out_qemu_ld(s, args, true); + break; + case INDEX_op_qemu_st_i32: + tcg_out_qemu_st(s, args, false); + break; + case INDEX_op_qemu_st_i64: + tcg_out_qemu_st(s, args, true); + break; + + case INDEX_op_ext8u_i32: + case INDEX_op_ext8u_i64: + tcg_out_ext8u(s, a0, a1); + break; + + case INDEX_op_ext16u_i32: + case INDEX_op_ext16u_i64: + tcg_out_ext16u(s, a0, a1); + break; + + case INDEX_op_ext32u_i64: + case INDEX_op_extu_i32_i64: + tcg_out_ext32u(s, a0, a1); + break; + + case INDEX_op_ext8s_i32: + case INDEX_op_ext8s_i64: + tcg_out_ext8s(s, a0, a1); + break; + + case INDEX_op_ext16s_i32: + case INDEX_op_ext16s_i64: + tcg_out_ext16s(s, a0, a1); + break; + + case INDEX_op_ext32s_i64: + case INDEX_op_extrl_i64_i32: + case INDEX_op_ext_i32_i64: + tcg_out_ext32s(s, a0, a1); + break; + + case INDEX_op_extrh_i64_i32: + tcg_out_opc_imm(s, OPC_SRAI, a0, a1, 32); + break; + + case INDEX_op_mulsh_i32: + case INDEX_op_mulsh_i64: + tcg_out_opc_reg(s, OPC_MULH, a0, a1, a2); + break; + + case INDEX_op_muluh_i32: + case INDEX_op_muluh_i64: + tcg_out_opc_reg(s, OPC_MULHU, a0, a1, a2); + break; + + case INDEX_op_mb: + tcg_out_mb(s, a0); + break; + + case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ + case INDEX_op_mov_i64: + case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi. */ + case INDEX_op_movi_i64: + case INDEX_op_call: /* Always emitted via tcg_out_call. */ + default: + g_assert_not_reached(); + } +} + +static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) +{ + static const TCGTargetOpDef r + = { .args_ct_str = { "r" } }; + static const TCGTargetOpDef r_r + = { .args_ct_str = { "r", "r" } }; + static const TCGTargetOpDef rZ_r + = { .args_ct_str = { "rZ", "r" } }; + static const TCGTargetOpDef rZ_rZ + = { .args_ct_str = { "rZ", "rZ" } }; + static const TCGTargetOpDef rZ_rZ_rZ_rZ + = { .args_ct_str = { "rZ", "rZ", "rZ", "rZ" } }; + static const TCGTargetOpDef r_r_ri + = { .args_ct_str = { "r", "r", "ri" } }; + static const TCGTargetOpDef r_r_rI + = { .args_ct_str = { "r", "r", "rI" } }; + static const TCGTargetOpDef r_rZ_rN + = { .args_ct_str = { "r", "rZ", "rN" } }; + static const TCGTargetOpDef r_rZ_rZ + = { .args_ct_str = { "r", "rZ", "rZ" } }; + static const TCGTargetOpDef r_rZ_rZ_rZ_rZ + = { .args_ct_str = { "r", "rZ", "rZ", "rZ", "rZ" } }; + static const TCGTargetOpDef r_L + = { .args_ct_str = { "r", "L" } }; + static const TCGTargetOpDef r_r_L + = { .args_ct_str = { "r", "r", "L" } }; + static const TCGTargetOpDef r_L_L + = { .args_ct_str = { "r", "L", "L" } }; + static const TCGTargetOpDef r_r_L_L + = { .args_ct_str = { "r", "r", "L", "L" } }; + static const TCGTargetOpDef LZ_L + = { .args_ct_str = { "LZ", "L" } }; + static const TCGTargetOpDef LZ_L_L + = { .args_ct_str = { "LZ", "L", "L" } }; + static const TCGTargetOpDef LZ_LZ_L + = { .args_ct_str = { "LZ", "LZ", "L" } }; + static const TCGTargetOpDef LZ_LZ_L_L + = { .args_ct_str = { "LZ", "LZ", "L", "L" } }; + static const TCGTargetOpDef r_r_rZ_rZ_rM_rM + = { .args_ct_str = { "r", "r", "rZ", "rZ", "rM", "rM" } }; + + switch (op) { + case INDEX_op_goto_ptr: + return &r; + + case INDEX_op_ld8u_i32: + case INDEX_op_ld8s_i32: + case INDEX_op_ld16u_i32: + case INDEX_op_ld16s_i32: + case INDEX_op_ld_i32: + case INDEX_op_not_i32: + case INDEX_op_neg_i32: + case INDEX_op_ld8u_i64: + case INDEX_op_ld8s_i64: + case INDEX_op_ld16u_i64: + case INDEX_op_ld16s_i64: + case INDEX_op_ld32s_i64: + case INDEX_op_ld32u_i64: + case INDEX_op_ld_i64: + case INDEX_op_not_i64: + case INDEX_op_neg_i64: + case INDEX_op_ext8u_i32: + case INDEX_op_ext8u_i64: + case INDEX_op_ext16u_i32: + case INDEX_op_ext16u_i64: + case INDEX_op_ext32u_i64: + case INDEX_op_extu_i32_i64: + case INDEX_op_ext8s_i32: + case INDEX_op_ext8s_i64: + case INDEX_op_ext16s_i32: + case INDEX_op_ext16s_i64: + case INDEX_op_ext32s_i64: + case INDEX_op_extrl_i64_i32: + case INDEX_op_extrh_i64_i32: + case INDEX_op_ext_i32_i64: + return &r_r; + + case INDEX_op_st8_i32: + case INDEX_op_st16_i32: + case INDEX_op_st_i32: + case INDEX_op_st8_i64: + case INDEX_op_st16_i64: + case INDEX_op_st32_i64: + case INDEX_op_st_i64: + return &rZ_r; + + case INDEX_op_add_i32: + case INDEX_op_and_i32: + case INDEX_op_or_i32: + case INDEX_op_xor_i32: + case INDEX_op_add_i64: + case INDEX_op_and_i64: + case INDEX_op_or_i64: + case INDEX_op_xor_i64: + return &r_r_rI; + + case INDEX_op_sub_i32: + case INDEX_op_sub_i64: + return &r_rZ_rN; + + case INDEX_op_mul_i32: + case INDEX_op_mulsh_i32: + case INDEX_op_muluh_i32: + case INDEX_op_div_i32: + case INDEX_op_divu_i32: + case INDEX_op_rem_i32: + case INDEX_op_remu_i32: + case INDEX_op_setcond_i32: + case INDEX_op_mul_i64: + case INDEX_op_mulsh_i64: + case INDEX_op_muluh_i64: + case INDEX_op_div_i64: + case INDEX_op_divu_i64: + case INDEX_op_rem_i64: + case INDEX_op_remu_i64: + case INDEX_op_setcond_i64: + return &r_rZ_rZ; + + case INDEX_op_shl_i32: + case INDEX_op_shr_i32: + case INDEX_op_sar_i32: + case INDEX_op_shl_i64: + case INDEX_op_shr_i64: + case INDEX_op_sar_i64: + return &r_r_ri; + + case INDEX_op_brcond_i32: + case INDEX_op_brcond_i64: + return &rZ_rZ; + + case INDEX_op_add2_i32: + case INDEX_op_add2_i64: + case INDEX_op_sub2_i32: + case INDEX_op_sub2_i64: + return &r_r_rZ_rZ_rM_rM; + + case INDEX_op_brcond2_i32: + return &rZ_rZ_rZ_rZ; + + case INDEX_op_setcond2_i32: + return &r_rZ_rZ_rZ_rZ; + + case INDEX_op_qemu_ld_i32: + return TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &r_L : &r_L_L; + case INDEX_op_qemu_st_i32: + return TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &LZ_L : &LZ_L_L; + case INDEX_op_qemu_ld_i64: + return TCG_TARGET_REG_BITS == 64 ? &r_L + : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &r_r_L + : &r_r_L_L; + case INDEX_op_qemu_st_i64: + return TCG_TARGET_REG_BITS == 64 ? &LZ_L + : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? &LZ_LZ_L + : &LZ_LZ_L_L; + + default: + return NULL; + } +} From 92c041c59b99fbc35bdf4d5520fcaff80dc69ee0 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:19:39 +0000 Subject: [PATCH 19/42] tcg/riscv: Add the prologue generation and register the JIT Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 111 +++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index 014c5287f5..8a5f1deefc 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -1805,3 +1805,114 @@ static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode op) return NULL; } } + +static const int tcg_target_callee_save_regs[] = { + TCG_REG_S0, /* used for the global env (TCG_AREG0) */ + TCG_REG_S1, + TCG_REG_S2, + TCG_REG_S3, + TCG_REG_S4, + TCG_REG_S5, + TCG_REG_S6, + TCG_REG_S7, + TCG_REG_S8, + TCG_REG_S9, + TCG_REG_S10, + TCG_REG_S11, + TCG_REG_RA, /* should be last for ABI compliance */ +}; + +/* Stack frame parameters. */ +#define REG_SIZE (TCG_TARGET_REG_BITS / 8) +#define SAVE_SIZE ((int)ARRAY_SIZE(tcg_target_callee_save_regs) * REG_SIZE) +#define TEMP_SIZE (CPU_TEMP_BUF_NLONGS * (int)sizeof(long)) +#define FRAME_SIZE ((TCG_STATIC_CALL_ARGS_SIZE + TEMP_SIZE + SAVE_SIZE \ + + TCG_TARGET_STACK_ALIGN - 1) \ + & -TCG_TARGET_STACK_ALIGN) +#define SAVE_OFS (TCG_STATIC_CALL_ARGS_SIZE + TEMP_SIZE) + +/* We're expecting to be able to use an immediate for frame allocation. */ +QEMU_BUILD_BUG_ON(FRAME_SIZE > 0x7ff); + +/* Generate global QEMU prologue and epilogue code */ +static void tcg_target_qemu_prologue(TCGContext *s) +{ + int i; + + tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE, TEMP_SIZE); + + /* TB prologue */ + tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_SP, TCG_REG_SP, -FRAME_SIZE); + for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) { + tcg_out_st(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i], + TCG_REG_SP, SAVE_OFS + i * REG_SIZE); + } + +#if !defined(CONFIG_SOFTMMU) + tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base); + tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG); +#endif + + /* Call generated code */ + tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]); + tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, tcg_target_call_iarg_regs[1], 0); + + /* Return path for goto_ptr. Set return value to 0 */ + s->code_gen_epilogue = s->code_ptr; + tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_A0, TCG_REG_ZERO); + + /* TB epilogue */ + tb_ret_addr = s->code_ptr; + for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) { + tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i], + TCG_REG_SP, SAVE_OFS + i * REG_SIZE); + } + + tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_SP, TCG_REG_SP, FRAME_SIZE); + tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_RA, 0); +} + +typedef struct { + DebugFrameHeader h; + uint8_t fde_def_cfa[4]; + uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2]; +} DebugFrame; + +#define ELF_HOST_MACHINE EM_RISCV + +static const DebugFrame debug_frame = { + .h.cie.len = sizeof(DebugFrameCIE) - 4, /* length after .len member */ + .h.cie.id = -1, + .h.cie.version = 1, + .h.cie.code_align = 1, + .h.cie.data_align = -(TCG_TARGET_REG_BITS / 8) & 0x7f, /* sleb128 */ + .h.cie.return_column = TCG_REG_RA, + + /* Total FDE size does not include the "len" member. */ + .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset), + + .fde_def_cfa = { + 12, TCG_REG_SP, /* DW_CFA_def_cfa sp, ... */ + (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */ + (FRAME_SIZE >> 7) + }, + .fde_reg_ofs = { + 0x80 + 9, 12, /* DW_CFA_offset, s1, -96 */ + 0x80 + 18, 11, /* DW_CFA_offset, s2, -88 */ + 0x80 + 19, 10, /* DW_CFA_offset, s3, -80 */ + 0x80 + 20, 9, /* DW_CFA_offset, s4, -72 */ + 0x80 + 21, 8, /* DW_CFA_offset, s5, -64 */ + 0x80 + 22, 7, /* DW_CFA_offset, s6, -56 */ + 0x80 + 23, 6, /* DW_CFA_offset, s7, -48 */ + 0x80 + 24, 5, /* DW_CFA_offset, s8, -40 */ + 0x80 + 25, 4, /* DW_CFA_offset, s9, -32 */ + 0x80 + 26, 3, /* DW_CFA_offset, s10, -24 */ + 0x80 + 27, 2, /* DW_CFA_offset, s11, -16 */ + 0x80 + 1 , 1, /* DW_CFA_offset, ra, -8 */ + } +}; + +void tcg_register_jit(void *buf, size_t buf_size) +{ + tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); +} From 7a5549f2aea84bf8c71c6eaa438b4bb84e9ad967 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:19:49 +0000 Subject: [PATCH 20/42] tcg/riscv: Add the target init code Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: Signed-off-by: Richard Henderson --- tcg/riscv/tcg-target.inc.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tcg/riscv/tcg-target.inc.c b/tcg/riscv/tcg-target.inc.c index 8a5f1deefc..6cf8de32b5 100644 --- a/tcg/riscv/tcg-target.inc.c +++ b/tcg/riscv/tcg-target.inc.c @@ -1872,6 +1872,37 @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_RA, 0); } +static void tcg_target_init(TCGContext *s) +{ + tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff; + if (TCG_TARGET_REG_BITS == 64) { + tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff; + } + + tcg_target_call_clobber_regs = -1u; + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S0); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S1); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S2); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S3); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S4); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S5); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S6); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S7); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S8); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S9); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S10); + tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S11); + + s->reserved_regs = 0; + tcg_regset_set_reg(s->reserved_regs, TCG_REG_ZERO); + tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP0); + tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP1); + tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP2); + tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP); + tcg_regset_set_reg(s->reserved_regs, TCG_REG_GP); + tcg_regset_set_reg(s->reserved_regs, TCG_REG_TP); +} + typedef struct { DebugFrameHeader h; uint8_t fde_def_cfa[4]; From 464e447a0c4fbda2c5adce9a1b0f96800648a36f Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:19:59 +0000 Subject: [PATCH 21/42] tcg: Add RISC-V cpu signal handler Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: Signed-off-by: Richard Henderson --- accel/tcg/user-exec.c | 75 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index cd75829cf2..941295ea49 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -571,6 +571,81 @@ int cpu_signal_handler(int host_signum, void *pinfo, return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask); } +#elif defined(__riscv) + +int cpu_signal_handler(int host_signum, void *pinfo, + void *puc) +{ + siginfo_t *info = pinfo; + ucontext_t *uc = puc; + greg_t pc = uc->uc_mcontext.__gregs[REG_PC]; + uint32_t insn = *(uint32_t *)pc; + int is_write = 0; + + /* Detect store by reading the instruction at the program + counter. Note: we currently only generate 32-bit + instructions so we thus only detect 32-bit stores */ + switch (((insn >> 0) & 0b11)) { + case 3: + switch (((insn >> 2) & 0b11111)) { + case 8: + switch (((insn >> 12) & 0b111)) { + case 0: /* sb */ + case 1: /* sh */ + case 2: /* sw */ + case 3: /* sd */ + case 4: /* sq */ + is_write = 1; + break; + default: + break; + } + break; + case 9: + switch (((insn >> 12) & 0b111)) { + case 2: /* fsw */ + case 3: /* fsd */ + case 4: /* fsq */ + is_write = 1; + break; + default: + break; + } + break; + default: + break; + } + } + + /* Check for compressed instructions */ + switch (((insn >> 13) & 0b111)) { + case 7: + switch (insn & 0b11) { + case 0: /*c.sd */ + case 2: /* c.sdsp */ + is_write = 1; + break; + default: + break; + } + break; + case 6: + switch (insn & 0b11) { + case 0: /* c.sw */ + case 3: /* c.swsp */ + is_write = 1; + break; + default: + break; + } + break; + default: + break; + } + + return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask); +} + #else #error host CPU specific signal handler needed From 91468b2703ee0e4dcd7439dec8ca60d6cf8270ac Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:20:09 +0000 Subject: [PATCH 22/42] disas: Add RISC-V support Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: Signed-off-by: Richard Henderson --- disas.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/disas.c b/disas.c index f9c517b358..d9aa713a40 100644 --- a/disas.c +++ b/disas.c @@ -522,8 +522,14 @@ void disas(FILE *out, void *code, unsigned long size) # ifdef _ARCH_PPC64 s.info.cap_mode = CS_MODE_64; # endif -#elif defined(__riscv__) - print_insn = print_insn_riscv; +#elif defined(__riscv) && defined(CONFIG_RISCV_DIS) +#if defined(_ILP32) || (__riscv_xlen == 32) + print_insn = print_insn_riscv32; +#elif defined(_LP64) + print_insn = print_insn_riscv64; +#else +#error unsupported RISC-V ABI +#endif #elif defined(__aarch64__) && defined(CONFIG_ARM_A64_DIS) print_insn = print_insn_arm_a64; s.info.cap_arch = CS_ARCH_ARM64; From c4f8054381348385529dc4a5928bbf2226144040 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 19 Dec 2018 19:20:19 +0000 Subject: [PATCH 23/42] configure: Add support for building RISC-V host Signed-off-by: Alistair Francis Signed-off-by: Michael Clark Reviewed-by: Richard Henderson Message-Id: <52160afacecc5b109dc43a412fa3e74ddd6277fb.1545246859.git.alistair.francis@wdc.com> Signed-off-by: Richard Henderson --- configure | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 224d3071ac..79375affc1 100755 --- a/configure +++ b/configure @@ -710,6 +710,12 @@ elif check_define __s390__ ; then else cpu="s390" fi +elif check_define __riscv ; then + if check_define _LP64 ; then + cpu="riscv64" + else + cpu="riscv32" + fi elif check_define __arm__ ; then cpu="arm" elif check_define __aarch64__ ; then @@ -722,7 +728,7 @@ ARCH= # Normalise host CPU name and set ARCH. # Note that this case should only have supported host CPUs, not guests. case "$cpu" in - ppc|ppc64|s390|s390x|sparc64|x32) + ppc|ppc64|s390|s390x|sparc64|x32|riscv32|riscv64) cpu="$cpu" supported_cpu="yes" eval "cross_cc_${cpu}=\$host_cc" @@ -6937,6 +6943,8 @@ elif test "$ARCH" = "x86_64" -o "$ARCH" = "x32" ; then QEMU_INCLUDES="-iquote \$(SRC_PATH)/tcg/i386 $QEMU_INCLUDES" elif test "$ARCH" = "ppc64" ; then QEMU_INCLUDES="-iquote \$(SRC_PATH)/tcg/ppc $QEMU_INCLUDES" +elif test "$ARCH" = "riscv32" -o "$ARCH" = "riscv64" ; then + QEMU_INCLUDES="-I\$(SRC_PATH)/tcg/riscv $QEMU_INCLUDES" else QEMU_INCLUDES="-iquote \$(SRC_PATH)/tcg/\$(ARCH) $QEMU_INCLUDES" fi @@ -7433,7 +7441,7 @@ for i in $ARCH $TARGET_BASE_ARCH ; do ppc*) disas_config "PPC" ;; - riscv) + riscv*) disas_config "RISCV" ;; s390*) From 11cee55f18a02637945f12718237506abb8073b2 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 29 Nov 2018 16:49:54 -0800 Subject: [PATCH 24/42] disas/microblaze: Remove unused REG_SP macro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This causes a build error with debian sid, riscv64 host: disas/microblaze.c:179: error: "REG_SP" redefined [-Werror] #define REG_SP 1 /* stack pointer */ In file included from /usr/include/signal.h:306, from include/qemu/osdep.h:101, from disas/microblaze.c:36: /usr/include/riscv64-linux-gnu/sys/ucontext.h:36: note: this is the location of the previous definition # define REG_SP 2 Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- disas/microblaze.c | 1 - 1 file changed, 1 deletion(-) diff --git a/disas/microblaze.c b/disas/microblaze.c index 598ecbc89d..c23605043a 100644 --- a/disas/microblaze.c +++ b/disas/microblaze.c @@ -176,7 +176,6 @@ enum microblaze_instr_type { #define REG_TLBSX 36869 /* MMU: TLB Search Index reg */ /* alternate names for gen purpose regs */ -#define REG_SP 1 /* stack pointer */ #define REG_ROSDP 2 /* read-only small data pointer */ #define REG_RWSDP 13 /* read-write small data pointer */ From b8c92c6d2969e88b252f1d2351237a87981d1e74 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 20 Dec 2018 12:10:15 -0800 Subject: [PATCH 25/42] linux-user: Add safe_syscall for riscv64 host Reviewed-by: Alistair Francis Signed-off-by: Richard Henderson --- linux-user/host/riscv64/hostdep.h | 23 +++++++ linux-user/host/riscv64/safe-syscall.inc.S | 77 ++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 linux-user/host/riscv64/safe-syscall.inc.S diff --git a/linux-user/host/riscv64/hostdep.h b/linux-user/host/riscv64/hostdep.h index 28467ba00b..865f0fb9ff 100644 --- a/linux-user/host/riscv64/hostdep.h +++ b/linux-user/host/riscv64/hostdep.h @@ -8,4 +8,27 @@ #ifndef RISCV64_HOSTDEP_H #define RISCV64_HOSTDEP_H +/* We have a safe-syscall.inc.S */ +#define HAVE_SAFE_SYSCALL + +#ifndef __ASSEMBLER__ + +/* These are defined by the safe-syscall.inc.S file */ +extern char safe_syscall_start[]; +extern char safe_syscall_end[]; + +/* Adjust the signal context to rewind out of safe-syscall if we're in it */ +static inline void rewind_if_in_safe_syscall(void *puc) +{ + ucontext_t *uc = puc; + unsigned long *pcreg = &uc->uc_mcontext.__gregs[REG_PC]; + + if (*pcreg > (uintptr_t)safe_syscall_start + && *pcreg < (uintptr_t)safe_syscall_end) { + *pcreg = (uintptr_t)safe_syscall_start; + } +} + +#endif /* __ASSEMBLER__ */ + #endif diff --git a/linux-user/host/riscv64/safe-syscall.inc.S b/linux-user/host/riscv64/safe-syscall.inc.S new file mode 100644 index 0000000000..9ca3fbfd1e --- /dev/null +++ b/linux-user/host/riscv64/safe-syscall.inc.S @@ -0,0 +1,77 @@ +/* + * safe-syscall.inc.S : host-specific assembly fragment + * to handle signals occurring at the same time as system calls. + * This is intended to be included by linux-user/safe-syscall.S + * + * Written by Richard Henderson + * Copyright (C) 2018 Linaro, Inc. + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + + .global safe_syscall_base + .global safe_syscall_start + .global safe_syscall_end + .type safe_syscall_base, @function + .type safe_syscall_start, @function + .type safe_syscall_end, @function + + /* + * This is the entry point for making a system call. The calling + * convention here is that of a C varargs function with the + * first argument an 'int *' to the signal_pending flag, the + * second one the system call number (as a 'long'), and all further + * arguments being syscall arguments (also 'long'). + * We return a long which is the syscall's return value, which + * may be negative-errno on failure. Conversion to the + * -1-and-errno-set convention is done by the calling wrapper. + */ +safe_syscall_base: + .cfi_startproc + /* + * The syscall calling convention is nearly the same as C: + * we enter with a0 == *signal_pending + * a1 == syscall number + * a2 ... a7 == syscall arguments + * and return the result in a0 + * and the syscall instruction needs + * a7 == syscall number + * a0 ... a5 == syscall arguments + * and returns the result in a0 + * Shuffle everything around appropriately. + */ + mv t0, a0 /* signal_pending pointer */ + mv t1, a1 /* syscall number */ + mv a0, a2 /* syscall arguments */ + mv a1, a3 + mv a2, a4 + mv a3, a5 + mv a4, a6 + mv a5, a7 + mv a7, t1 + + /* + * This next sequence of code works in conjunction with the + * rewind_if_safe_syscall_function(). If a signal is taken + * and the interrupted PC is anywhere between 'safe_syscall_start' + * and 'safe_syscall_end' then we rewind it to 'safe_syscall_start'. + * The code sequence must therefore be able to cope with this, and + * the syscall instruction must be the final one in the sequence. + */ +safe_syscall_start: + /* If signal_pending is non-zero, don't do the call */ + lw t1, 0(t0) + bnez t1, 0f + scall +safe_syscall_end: + /* code path for having successfully executed the syscall */ + ret + +0: + /* code path when we didn't execute the syscall */ + li a0, -TARGET_ERESTARTSYS + ret + .cfi_endproc + + .size safe_syscall_base, .-safe_syscall_base From 3b50352b05eeafeb95cccd770f7aaba00bbdf6fe Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 26 Nov 2018 10:37:34 -0800 Subject: [PATCH 26/42] tcg: Renumber TCG_CALL_* flags Previously, the low 4 bits were used for TCG_CALL_TYPE_MASK, which was removed in 6a18ae2d2947532d5c26439548afa0481c4529f9. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tcg/tcg.h b/tcg/tcg.h index ade692fdf5..d9b101e2f8 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -462,11 +462,11 @@ typedef TCGv_ptr TCGv_env; /* call flags */ /* Helper does not read globals (either directly or through an exception). It implies TCG_CALL_NO_WRITE_GLOBALS. */ -#define TCG_CALL_NO_READ_GLOBALS 0x0010 +#define TCG_CALL_NO_READ_GLOBALS 0x0001 /* Helper does not write globals */ -#define TCG_CALL_NO_WRITE_GLOBALS 0x0020 +#define TCG_CALL_NO_WRITE_GLOBALS 0x0002 /* Helper can be safely suppressed if the return value is not used. */ -#define TCG_CALL_NO_SIDE_EFFECTS 0x0040 +#define TCG_CALL_NO_SIDE_EFFECTS 0x0004 /* convenience version of most used call flags */ #define TCG_CALL_NO_RWG TCG_CALL_NO_READ_GLOBALS From 15d7409260498505e991e7b9d87118627165e613 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 26 Nov 2018 11:32:38 -0800 Subject: [PATCH 27/42] tcg: Add TCG_CALL_NO_RETURN Remember which helpers have been marked noreturn. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- include/exec/helper-head.h | 13 +++++++++++++ include/exec/helper-tcg.h | 21 ++++++++++++++------- tcg/tcg.h | 2 ++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/include/exec/helper-head.h b/include/exec/helper-head.h index 276dd5afce..ab4f8b6623 100644 --- a/include/exec/helper-head.h +++ b/include/exec/helper-head.h @@ -108,6 +108,19 @@ #define dh_is_signed_env dh_is_signed_ptr #define dh_is_signed(t) dh_is_signed_##t +#define dh_callflag_i32 0 +#define dh_callflag_s32 0 +#define dh_callflag_int 0 +#define dh_callflag_i64 0 +#define dh_callflag_s64 0 +#define dh_callflag_f16 0 +#define dh_callflag_f32 0 +#define dh_callflag_f64 0 +#define dh_callflag_ptr 0 +#define dh_callflag_void 0 +#define dh_callflag_noreturn TCG_CALL_NO_RETURN +#define dh_callflag(t) glue(dh_callflag_, dh_alias(t)) + #define dh_sizemask(t, n) \ ((dh_is_64bit(t) << (n*2)) | (dh_is_signed(t) << (n*2+1))) diff --git a/include/exec/helper-tcg.h b/include/exec/helper-tcg.h index b3bdb0c399..268e0f804b 100644 --- a/include/exec/helper-tcg.h +++ b/include/exec/helper-tcg.h @@ -11,36 +11,43 @@ #define str(s) #s #define DEF_HELPER_FLAGS_0(NAME, FLAGS, ret) \ - { .func = HELPER(NAME), .name = str(NAME), .flags = FLAGS, \ + { .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(ret), \ .sizemask = dh_sizemask(ret, 0) }, #define DEF_HELPER_FLAGS_1(NAME, FLAGS, ret, t1) \ - { .func = HELPER(NAME), .name = str(NAME), .flags = FLAGS, \ + { .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(ret), \ .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) }, #define DEF_HELPER_FLAGS_2(NAME, FLAGS, ret, t1, t2) \ - { .func = HELPER(NAME), .name = str(NAME), .flags = FLAGS, \ + { .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(ret), \ .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \ | dh_sizemask(t2, 2) }, #define DEF_HELPER_FLAGS_3(NAME, FLAGS, ret, t1, t2, t3) \ - { .func = HELPER(NAME), .name = str(NAME), .flags = FLAGS, \ + { .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(ret), \ .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \ | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) }, #define DEF_HELPER_FLAGS_4(NAME, FLAGS, ret, t1, t2, t3, t4) \ - { .func = HELPER(NAME), .name = str(NAME), .flags = FLAGS, \ + { .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(ret), \ .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \ | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) | dh_sizemask(t4, 4) }, #define DEF_HELPER_FLAGS_5(NAME, FLAGS, ret, t1, t2, t3, t4, t5) \ - { .func = HELPER(NAME), .name = str(NAME), .flags = FLAGS, \ + { .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(ret), \ .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \ | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) | dh_sizemask(t4, 4) \ | dh_sizemask(t5, 5) }, #define DEF_HELPER_FLAGS_6(NAME, FLAGS, ret, t1, t2, t3, t4, t5, t6) \ - { .func = HELPER(NAME), .name = str(NAME), .flags = FLAGS, \ + { .func = HELPER(NAME), .name = str(NAME), \ + .flags = FLAGS | dh_callflag(ret), \ .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \ | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) | dh_sizemask(t4, 4) \ | dh_sizemask(t5, 5) | dh_sizemask(t6, 6) }, diff --git a/tcg/tcg.h b/tcg/tcg.h index d9b101e2f8..492d8dcf10 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -467,6 +467,8 @@ typedef TCGv_ptr TCGv_env; #define TCG_CALL_NO_WRITE_GLOBALS 0x0002 /* Helper can be safely suppressed if the return value is not used. */ #define TCG_CALL_NO_SIDE_EFFECTS 0x0004 +/* Helper is QEMU_NORETURN. */ +#define TCG_CALL_NO_RETURN 0x0008 /* convenience version of most used call flags */ #define TCG_CALL_NO_RWG TCG_CALL_NO_READ_GLOBALS From d88a117eaa39b1d0eb1a79fe84c81840a39eb233 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 26 Nov 2018 12:47:28 -0800 Subject: [PATCH 28/42] tcg: Reference count labels Increment when adding branches, and decrement when removing them. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg-op.c | 2 ++ tcg/tcg-op.h | 1 + tcg/tcg.c | 20 ++++++++++++++++++++ tcg/tcg.h | 3 ++- 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index 38652db32c..1bd7ef24af 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -240,6 +240,7 @@ void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) if (cond == TCG_COND_ALWAYS) { tcg_gen_br(l); } else if (cond != TCG_COND_NEVER) { + l->refs++; tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l)); } } @@ -1405,6 +1406,7 @@ void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) if (cond == TCG_COND_ALWAYS) { tcg_gen_br(l); } else if (cond != TCG_COND_NEVER) { + l->refs++; if (TCG_TARGET_REG_BITS == 32) { tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2), diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h index db4e9188f4..7007ec0d4d 100644 --- a/tcg/tcg-op.h +++ b/tcg/tcg-op.h @@ -260,6 +260,7 @@ static inline void gen_set_label(TCGLabel *l) static inline void tcg_gen_br(TCGLabel *l) { + l->refs++; tcg_gen_op1(INDEX_op_br, label_arg(l)); } diff --git a/tcg/tcg.c b/tcg/tcg.c index 963cb37892..99afc65126 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -2171,6 +2171,26 @@ static void process_op_defs(TCGContext *s) void tcg_op_remove(TCGContext *s, TCGOp *op) { + TCGLabel *label; + + switch (op->opc) { + case INDEX_op_br: + label = arg_label(op->args[0]); + label->refs--; + break; + case INDEX_op_brcond_i32: + case INDEX_op_brcond_i64: + label = arg_label(op->args[3]); + label->refs--; + break; + case INDEX_op_brcond2_i32: + label = arg_label(op->args[5]); + label->refs--; + break; + default: + break; + } + QTAILQ_REMOVE(&s->ops, op, link); QTAILQ_INSERT_TAIL(&s->free_ops, op, link); s->nb_ops--; diff --git a/tcg/tcg.h b/tcg/tcg.h index 492d8dcf10..db2d91867f 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -244,7 +244,8 @@ typedef struct TCGRelocation { typedef struct TCGLabel { unsigned has_value : 1; - unsigned id : 31; + unsigned id : 15; + unsigned refs : 16; union { uintptr_t value; tcg_insn_unit *value_ptr; From b4fc67c7afd2c338d6e7c73a7f428dfe05ae0603 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 26 Nov 2018 14:28:28 -0800 Subject: [PATCH 29/42] tcg: Add reachable_code_pass Delete trivially dead code that follows unconditional branches and noreturn helpers. These can occur either via optimization or via the structure of a target's translator following an exception. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg.c | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/tcg/tcg.c b/tcg/tcg.c index 99afc65126..d2be550ab4 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -2239,6 +2239,81 @@ TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc) return new_op; } +/* Reachable analysis : remove unreachable code. */ +static void reachable_code_pass(TCGContext *s) +{ + TCGOp *op, *op_next; + bool dead = false; + + QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { + bool remove = dead; + TCGLabel *label; + int call_flags; + + switch (op->opc) { + case INDEX_op_set_label: + label = arg_label(op->args[0]); + if (label->refs == 0) { + /* + * While there is an occasional backward branch, virtually + * all branches generated by the translators are forward. + * Which means that generally we will have already removed + * all references to the label that will be, and there is + * little to be gained by iterating. + */ + remove = true; + } else { + /* Once we see a label, insns become live again. */ + dead = false; + remove = false; + + /* + * Optimization can fold conditional branches to unconditional. + * If we find a label with one reference which is preceded by + * an unconditional branch to it, remove both. This needed to + * wait until the dead code in between them was removed. + */ + if (label->refs == 1) { + TCGOp *op_prev = QTAILQ_PREV(op, TCGOpHead, link); + if (op_prev->opc == INDEX_op_br && + label == arg_label(op_prev->args[0])) { + tcg_op_remove(s, op_prev); + remove = true; + } + } + } + break; + + case INDEX_op_br: + case INDEX_op_exit_tb: + case INDEX_op_goto_ptr: + /* Unconditional branches; everything following is dead. */ + dead = true; + break; + + case INDEX_op_call: + /* Notice noreturn helper calls, raising exceptions. */ + call_flags = op->args[TCGOP_CALLO(op) + TCGOP_CALLI(op) + 1]; + if (call_flags & TCG_CALL_NO_RETURN) { + dead = true; + } + break; + + case INDEX_op_insn_start: + /* Never remove -- we need to keep these for unwind. */ + remove = false; + break; + + default: + break; + } + + if (remove) { + tcg_op_remove(s, op); + } + } +} + #define TS_DEAD 1 #define TS_MEM 2 @@ -3515,6 +3590,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) atomic_set(&prof->la_time, prof->la_time - profile_getclock()); #endif + reachable_code_pass(s); liveness_pass_1(s); if (s->nb_indirects > 0) { From b016486e7baddb43cfc1e51909b05cde9cf82e0c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 27 Nov 2018 07:16:21 -0800 Subject: [PATCH 30/42] tcg: Add preferred_reg argument to tcg_reg_alloc This new argument will aid register allocation by indicating how the temporary will be used in future. If the preference cannot be satisfied, fall back to the constraints of the current insn. Short circuit the preference when it cannot be satisfied or if it does not further constrain the operation. With an eye toward optimizing function call sequences, optimize for the preferred_reg set containing a single register. For the moment, all users pass 0 for preference. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg.c | 103 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 81 insertions(+), 22 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index d2be550ab4..210bd5c6b9 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -1887,6 +1887,20 @@ static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", }; +static inline bool tcg_regset_single(TCGRegSet d) +{ + return (d & (d - 1)) == 0; +} + +static inline TCGReg tcg_regset_first(TCGRegSet d) +{ + if (TCG_TARGET_NB_REGS <= 32) { + return ctz32(d); + } else { + return ctz64(d); + } +} + void tcg_dump_ops(TCGContext *s) { char buf[128]; @@ -1902,6 +1916,7 @@ void tcg_dump_ops(TCGContext *s) def = &tcg_op_defs[c]; if (c == INDEX_op_insn_start) { + nb_oargs = 0; col += qemu_log("\n ----"); for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { @@ -2902,31 +2917,72 @@ static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) } } -/* Allocate a register belonging to reg1 & ~reg2 */ -static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet desired_regs, - TCGRegSet allocated_regs, bool rev) +/** + * tcg_reg_alloc: + * @required_regs: Set of registers in which we must allocate. + * @allocated_regs: Set of registers which must be avoided. + * @preferred_regs: Set of registers we should prefer. + * @rev: True if we search the registers in "indirect" order. + * + * The allocated register must be in @required_regs & ~@allocated_regs, + * but if we can put it in @preferred_regs we may save a move later. + */ +static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, + TCGRegSet allocated_regs, + TCGRegSet preferred_regs, bool rev) { - int i, n = ARRAY_SIZE(tcg_target_reg_alloc_order); + int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order); + TCGRegSet reg_ct[2]; const int *order; - TCGReg reg; - TCGRegSet reg_ct; - reg_ct = desired_regs & ~allocated_regs; + reg_ct[1] = required_regs & ~allocated_regs; + tcg_debug_assert(reg_ct[1] != 0); + reg_ct[0] = reg_ct[1] & preferred_regs; + + /* Skip the preferred_regs option if it cannot be satisfied, + or if the preference made no difference. */ + f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; + order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; - /* first try free registers */ - for(i = 0; i < n; i++) { - reg = order[i]; - if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == NULL) - return reg; + /* Try free registers, preferences first. */ + for (j = f; j < 2; j++) { + TCGRegSet set = reg_ct[j]; + + if (tcg_regset_single(set)) { + /* One register in the set. */ + TCGReg reg = tcg_regset_first(set); + if (s->reg_to_temp[reg] == NULL) { + return reg; + } + } else { + for (i = 0; i < n; i++) { + TCGReg reg = order[i]; + if (s->reg_to_temp[reg] == NULL && + tcg_regset_test_reg(set, reg)) { + return reg; + } + } + } } - /* XXX: do better spill choice */ - for(i = 0; i < n; i++) { - reg = order[i]; - if (tcg_regset_test_reg(reg_ct, reg)) { + /* We must spill something. */ + for (j = f; j < 2; j++) { + TCGRegSet set = reg_ct[j]; + + if (tcg_regset_single(set)) { + /* One register in the set. */ + TCGReg reg = tcg_regset_first(set); tcg_reg_free(s, reg, allocated_regs); return reg; + } else { + for (i = 0; i < n; i++) { + TCGReg reg = order[i]; + if (tcg_regset_test_reg(set, reg)) { + tcg_reg_free(s, reg, allocated_regs); + return reg; + } + } } } @@ -2944,12 +3000,14 @@ static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, case TEMP_VAL_REG: return; case TEMP_VAL_CONST: - reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base); + reg = tcg_reg_alloc(s, desired_regs, allocated_regs, + 0, ts->indirect_base); tcg_out_movi(s, ts->type, reg, ts->val); ts->mem_coherent = 0; break; case TEMP_VAL_MEM: - reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base); + reg = tcg_reg_alloc(s, desired_regs, allocated_regs, + 0, ts->indirect_base); tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); ts->mem_coherent = 1; break; @@ -3109,7 +3167,8 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) input one. */ tcg_regset_set_reg(allocated_regs, ts->reg); ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype], - allocated_regs, ots->indirect_base); + allocated_regs, 0, + ots->indirect_base); } tcg_out_mov(s, otype, ots->reg, ts->reg); } @@ -3197,7 +3256,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) /* allocate a new register matching the constraint and move the temporary register into it */ reg = tcg_reg_alloc(s, arg_ct->u.regs, i_allocated_regs, - ts->indirect_base); + 0, ts->indirect_base); tcg_out_mov(s, ts->type, reg, ts->reg); } new_args[i] = reg; @@ -3242,7 +3301,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) } else if (arg_ct->ct & TCG_CT_NEWREG) { reg = tcg_reg_alloc(s, arg_ct->u.regs, i_allocated_regs | o_allocated_regs, - ts->indirect_base); + 0, ts->indirect_base); } else { /* if fixed register, we try to use it */ reg = ts->reg; @@ -3251,7 +3310,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) goto oarg_end; } reg = tcg_reg_alloc(s, arg_ct->u.regs, o_allocated_regs, - ts->indirect_base); + 0, ts->indirect_base); } tcg_regset_set_reg(o_allocated_regs, reg); /* if a fixed register is used, then a move will be done afterwards */ From b722452aefb089e003b16946a4d73bad1fd3b79b Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 27 Nov 2018 07:48:06 -0800 Subject: [PATCH 31/42] tcg: Add preferred_reg argument to temp_load Pass this through to tcg_reg_alloc. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 210bd5c6b9..351d302a68 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -2837,7 +2837,7 @@ static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) s->current_frame_offset += sizeof(tcg_target_long); } -static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet); +static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); /* Mark a temporary as free or dead. If 'free_or_dead' is negative, mark it free; otherwise mark it dead. */ @@ -2886,7 +2886,7 @@ static void temp_sync(TCGContext *s, TCGTemp *ts, break; } temp_load(s, ts, tcg_target_available_regs[ts->type], - allocated_regs); + allocated_regs, 0); /* fallthrough */ case TEMP_VAL_REG: @@ -2992,7 +2992,7 @@ static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, /* Make sure the temporary is in a register. If needed, allocate the register from DESIRED while avoiding ALLOCATED. */ static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, - TCGRegSet allocated_regs) + TCGRegSet allocated_regs, TCGRegSet preferred_regs) { TCGReg reg; @@ -3001,13 +3001,13 @@ static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, return; case TEMP_VAL_CONST: reg = tcg_reg_alloc(s, desired_regs, allocated_regs, - 0, ts->indirect_base); + preferred_regs, ts->indirect_base); tcg_out_movi(s, ts->type, reg, ts->val); ts->mem_coherent = 0; break; case TEMP_VAL_MEM: reg = tcg_reg_alloc(s, desired_regs, allocated_regs, - 0, ts->indirect_base); + preferred_regs, ts->indirect_base); tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); ts->mem_coherent = 1; break; @@ -3137,7 +3137,7 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) the SOURCE value into its own register first, that way we don't have to reload SOURCE the next time it is used. */ if (ts->val_type == TEMP_VAL_MEM) { - temp_load(s, ts, tcg_target_available_regs[itype], allocated_regs); + temp_load(s, ts, tcg_target_available_regs[itype], allocated_regs, 0); } tcg_debug_assert(ts->val_type == TEMP_VAL_REG); @@ -3221,7 +3221,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) goto iarg_end; } - temp_load(s, ts, arg_ct->u.regs, i_allocated_regs); + temp_load(s, ts, arg_ct->u.regs, i_allocated_regs, 0); if (arg_ct->ct & TCG_CT_IALIAS) { if (ts->fixed_reg) { @@ -3402,7 +3402,7 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) if (arg != TCG_CALL_DUMMY_ARG) { ts = arg_temp(arg); temp_load(s, ts, tcg_target_available_regs[ts->type], - s->reserved_regs); + s->reserved_regs, 0); tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); } #ifndef TCG_TARGET_STACK_GROWSUP @@ -3427,7 +3427,7 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) TCGRegSet arg_set = 0; tcg_regset_set_reg(arg_set, reg); - temp_load(s, ts, arg_set, allocated_regs); + temp_load(s, ts, arg_set, allocated_regs, 0); } tcg_regset_set_reg(allocated_regs, reg); From 98b4e186c1ccb8f1868c61a33a3be8c2b82654f3 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 27 Nov 2018 15:35:04 -0800 Subject: [PATCH 32/42] tcg: Add preferred_reg argument to temp_sync Pass this through to tcg_reg_alloc. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 351d302a68..fe060c481a 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -2865,8 +2865,8 @@ static inline void temp_dead(TCGContext *s, TCGTemp *ts) registers needs to be allocated to store a constant. If 'free_or_dead' is non-zero, subsequently release the temporary; if it is positive, the temp is dead; if it is negative, the temp is free. */ -static void temp_sync(TCGContext *s, TCGTemp *ts, - TCGRegSet allocated_regs, int free_or_dead) +static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, + TCGRegSet preferred_regs, int free_or_dead) { if (ts->fixed_reg) { return; @@ -2886,7 +2886,7 @@ static void temp_sync(TCGContext *s, TCGTemp *ts, break; } temp_load(s, ts, tcg_target_available_regs[ts->type], - allocated_regs, 0); + allocated_regs, preferred_regs); /* fallthrough */ case TEMP_VAL_REG: @@ -2913,7 +2913,7 @@ static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) { TCGTemp *ts = s->reg_to_temp[reg]; if (ts != NULL) { - temp_sync(s, ts, allocated_regs, -1); + temp_sync(s, ts, allocated_regs, 0, -1); } } @@ -3093,7 +3093,7 @@ static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, ots->val = val; ots->mem_coherent = 0; if (NEED_SYNC_ARG(0)) { - temp_sync(s, ots, s->reserved_regs, IS_DEAD_ARG(0)); + temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); } else if (IS_DEAD_ARG(0)) { temp_dead(s, ots); } @@ -3176,7 +3176,7 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) ots->mem_coherent = 0; s->reg_to_temp[ots->reg] = ots; if (NEED_SYNC_ARG(0)) { - temp_sync(s, ots, allocated_regs, 0); + temp_sync(s, ots, allocated_regs, 0, 0); } } } @@ -3346,7 +3346,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) tcg_out_mov(s, ts->type, ts->reg, reg); } if (NEED_SYNC_ARG(i)) { - temp_sync(s, ts, o_allocated_regs, IS_DEAD_ARG(i)); + temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); } else if (IS_DEAD_ARG(i)) { temp_dead(s, ts); } @@ -3480,7 +3480,7 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) ts->mem_coherent = 0; s->reg_to_temp[reg] = ts; if (NEED_SYNC_ARG(i)) { - temp_sync(s, ts, allocated_regs, IS_DEAD_ARG(i)); + temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i)); } else if (IS_DEAD_ARG(i)) { temp_dead(s, ts); } From ba87719cd267e6f07b17f6cda08246bf483146d4 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 27 Nov 2018 15:39:21 -0800 Subject: [PATCH 33/42] tcg: Add preferred_reg argument to tcg_reg_alloc_do_movi Pass this through to temp_sync. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index fe060c481a..7844158a8a 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -3077,7 +3077,8 @@ static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) } static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, - tcg_target_ulong val, TCGLifeData arg_life) + tcg_target_ulong val, TCGLifeData arg_life, + TCGRegSet preferred_regs) { if (ots->fixed_reg) { /* For fixed registers, we do not do any constant propagation. */ @@ -3093,7 +3094,7 @@ static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, ots->val = val; ots->mem_coherent = 0; if (NEED_SYNC_ARG(0)) { - temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); + temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); } else if (IS_DEAD_ARG(0)) { temp_dead(s, ots); } @@ -3104,7 +3105,7 @@ static void tcg_reg_alloc_movi(TCGContext *s, const TCGOp *op) TCGTemp *ots = arg_temp(op->args[0]); tcg_target_ulong val = op->args[1]; - tcg_reg_alloc_do_movi(s, ots, val, op->life); + tcg_reg_alloc_do_movi(s, ots, val, op->life, 0); } static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) @@ -3128,7 +3129,7 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) if (IS_DEAD_ARG(1)) { temp_dead(s, ts); } - tcg_reg_alloc_do_movi(s, ots, val, arg_life); + tcg_reg_alloc_do_movi(s, ots, val, arg_life, 0); return; } From 69e3706d2b473815e382552e729d12590339e0ac Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 27 Nov 2018 07:44:51 -0800 Subject: [PATCH 34/42] tcg: Add output_pref to TCGOp Allocate storage for, but do not yet fill in, per-opcode preferences for the output operands. Pass it in to the register allocation routines for output operands. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg.c | 18 +++++++++++------- tcg/tcg.h | 3 +++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 7844158a8a..32c66dae7d 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -2591,6 +2591,8 @@ static void liveness_pass_1(TCGContext *s) break; } op->life = arg_life; + op->output_pref[0] = 0; + op->output_pref[1] = 0; } } @@ -3105,17 +3107,18 @@ static void tcg_reg_alloc_movi(TCGContext *s, const TCGOp *op) TCGTemp *ots = arg_temp(op->args[0]); tcg_target_ulong val = op->args[1]; - tcg_reg_alloc_do_movi(s, ots, val, op->life, 0); + tcg_reg_alloc_do_movi(s, ots, val, op->life, op->output_pref[0]); } static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) { const TCGLifeData arg_life = op->life; - TCGRegSet allocated_regs; + TCGRegSet allocated_regs, preferred_regs; TCGTemp *ts, *ots; TCGType otype, itype; allocated_regs = s->reserved_regs; + preferred_regs = op->output_pref[0]; ots = arg_temp(op->args[0]); ts = arg_temp(op->args[1]); @@ -3129,7 +3132,7 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) if (IS_DEAD_ARG(1)) { temp_dead(s, ts); } - tcg_reg_alloc_do_movi(s, ots, val, arg_life, 0); + tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs); return; } @@ -3138,7 +3141,8 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) the SOURCE value into its own register first, that way we don't have to reload SOURCE the next time it is used. */ if (ts->val_type == TEMP_VAL_MEM) { - temp_load(s, ts, tcg_target_available_regs[itype], allocated_regs, 0); + temp_load(s, ts, tcg_target_available_regs[itype], + allocated_regs, preferred_regs); } tcg_debug_assert(ts->val_type == TEMP_VAL_REG); @@ -3168,7 +3172,7 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) input one. */ tcg_regset_set_reg(allocated_regs, ts->reg); ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype], - allocated_regs, 0, + allocated_regs, preferred_regs, ots->indirect_base); } tcg_out_mov(s, otype, ots->reg, ts->reg); @@ -3302,7 +3306,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) } else if (arg_ct->ct & TCG_CT_NEWREG) { reg = tcg_reg_alloc(s, arg_ct->u.regs, i_allocated_regs | o_allocated_regs, - 0, ts->indirect_base); + op->output_pref[k], ts->indirect_base); } else { /* if fixed register, we try to use it */ reg = ts->reg; @@ -3311,7 +3315,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) goto oarg_end; } reg = tcg_reg_alloc(s, arg_ct->u.regs, o_allocated_regs, - 0, ts->indirect_base); + op->output_pref[k], ts->indirect_base); } tcg_regset_set_reg(o_allocated_regs, reg); /* if a fixed register is used, then a move will be done afterwards */ diff --git a/tcg/tcg.h b/tcg/tcg.h index db2d91867f..c6721cc7a7 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -619,6 +619,9 @@ typedef struct TCGOp { /* Arguments for the opcode. */ TCGArg args[MAX_OPC_PARAM]; + + /* Register preferences for the output(s). */ + TCGRegSet output_pref[2]; } TCGOp; #define TCGOP_CALLI(X) (X)->param1 From d62816f2db439b2dd761c674f0256f21d9dd2ed0 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 27 Nov 2018 20:21:31 -0800 Subject: [PATCH 35/42] tcg: Improve register allocation for matching constraints Try harder to honor the output_pref. When we're forced to allocate a second register for the input, it does not need to use the input constraint; that will be honored by the register we allocate for the output and a move is already required. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg.c | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 32c66dae7d..32bac5e792 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -3213,6 +3213,8 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) /* satisfy input constraints */ for (k = 0; k < nb_iargs; k++) { + TCGRegSet i_preferred_regs, o_preferred_regs; + i = def->sorted_args[nb_oargs + k]; arg = op->args[i]; arg_ct = &def->args_ct[i]; @@ -3223,17 +3225,18 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) /* constant is OK for instruction */ const_args[i] = 1; new_args[i] = ts->val; - goto iarg_end; + continue; } - temp_load(s, ts, arg_ct->u.regs, i_allocated_regs, 0); - + i_preferred_regs = o_preferred_regs = 0; if (arg_ct->ct & TCG_CT_IALIAS) { + o_preferred_regs = op->output_pref[arg_ct->alias_index]; if (ts->fixed_reg) { /* if fixed register, we must allocate a new register if the alias is not the same register */ - if (arg != op->args[arg_ct->alias_index]) + if (arg != op->args[arg_ct->alias_index]) { goto allocate_in_reg; + } } else { /* if the input is aliased to an output and if it is not dead after the instruction, we must allocate @@ -3241,33 +3244,42 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) if (!IS_DEAD_ARG(i)) { goto allocate_in_reg; } + /* check if the current register has already been allocated for another input aliased to an output */ - int k2, i2; - for (k2 = 0 ; k2 < k ; k2++) { - i2 = def->sorted_args[nb_oargs + k2]; - if ((def->args_ct[i2].ct & TCG_CT_IALIAS) && - (new_args[i2] == ts->reg)) { - goto allocate_in_reg; + if (ts->val_type == TEMP_VAL_REG) { + int k2, i2; + reg = ts->reg; + for (k2 = 0 ; k2 < k ; k2++) { + i2 = def->sorted_args[nb_oargs + k2]; + if ((def->args_ct[i2].ct & TCG_CT_IALIAS) && + reg == new_args[i2]) { + goto allocate_in_reg; + } } } + i_preferred_regs = o_preferred_regs; } } + + temp_load(s, ts, arg_ct->u.regs, i_allocated_regs, i_preferred_regs); reg = ts->reg; + if (tcg_regset_test_reg(arg_ct->u.regs, reg)) { /* nothing to do : the constraint is satisfied */ } else { allocate_in_reg: /* allocate a new register matching the constraint and move the temporary register into it */ + temp_load(s, ts, tcg_target_available_regs[ts->type], + i_allocated_regs, 0); reg = tcg_reg_alloc(s, arg_ct->u.regs, i_allocated_regs, - 0, ts->indirect_base); + o_preferred_regs, ts->indirect_base); tcg_out_mov(s, ts->type, reg, ts->reg); } new_args[i] = reg; const_args[i] = 0; tcg_regset_set_reg(i_allocated_regs, reg); - iarg_end: ; } /* mark dead temporaries and free the associated registers */ From 1894f69a612b35c2a39b44a824da04d74bfe324a Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 27 Nov 2018 12:46:00 -0800 Subject: [PATCH 36/42] tcg: Dump register preference info with liveness Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg.c | 44 +++++++++++++++++++++++++++++++++++++------- tcg/tcg.h | 3 --- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 32bac5e792..e2ae04d1d0 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -1901,7 +1901,7 @@ static inline TCGReg tcg_regset_first(TCGRegSet d) } } -void tcg_dump_ops(TCGContext *s) +static void tcg_dump_ops(TCGContext *s, bool have_prefs) { char buf[128]; TCGOp *op; @@ -2036,12 +2036,15 @@ void tcg_dump_ops(TCGContext *s) col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]); } } - if (op->life) { - unsigned life = op->life; - for (; col < 48; ++col) { + if (have_prefs || op->life) { + for (; col < 40; ++col) { putc(' ', qemu_logfile); } + } + + if (op->life) { + unsigned life = op->life; if (life & (SYNC_ARG * 3)) { qemu_log(" sync:"); @@ -2061,6 +2064,33 @@ void tcg_dump_ops(TCGContext *s) } } } + + if (have_prefs) { + for (i = 0; i < nb_oargs; ++i) { + TCGRegSet set = op->output_pref[i]; + + if (i == 0) { + qemu_log(" pref="); + } else { + qemu_log(","); + } + if (set == 0) { + qemu_log("none"); + } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) { + qemu_log("all"); +#ifdef CONFIG_DEBUG_TCG + } else if (tcg_regset_single(set)) { + TCGReg reg = tcg_regset_first(set); + qemu_log("%s", tcg_target_reg_names[reg]); +#endif + } else if (TCG_TARGET_NB_REGS <= 32) { + qemu_log("%#x", (uint32_t)set); + } else { + qemu_log("%#" PRIx64, (uint64_t)set); + } + } + } + qemu_log("\n"); } } @@ -3647,7 +3677,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) && qemu_log_in_addr_range(tb->pc))) { qemu_log_lock(); qemu_log("OP:\n"); - tcg_dump_ops(s); + tcg_dump_ops(s, false); qemu_log("\n"); qemu_log_unlock(); } @@ -3675,7 +3705,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) && qemu_log_in_addr_range(tb->pc))) { qemu_log_lock(); qemu_log("OP before indirect lowering:\n"); - tcg_dump_ops(s); + tcg_dump_ops(s, false); qemu_log("\n"); qemu_log_unlock(); } @@ -3696,7 +3726,7 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb) && qemu_log_in_addr_range(tb->pc))) { qemu_log_lock(); qemu_log("OP after optimization and liveness analysis:\n"); - tcg_dump_ops(s); + tcg_dump_ops(s, true); qemu_log("\n"); qemu_log_unlock(); } diff --git a/tcg/tcg.h b/tcg/tcg.h index c6721cc7a7..5e5cf686a3 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -1082,9 +1082,6 @@ TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op, TCGOpcode opc); void tcg_optimize(TCGContext *s); -/* only used for debugging purposes */ -void tcg_dump_ops(TCGContext *s); - TCGv_i32 tcg_const_i32(int32_t val); TCGv_i64 tcg_const_i64(int64_t val); TCGv_i32 tcg_const_local_i32(int32_t val); From 152c35aab43213335e1ae8b865d259c0222dd110 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 27 Nov 2018 13:32:33 -0800 Subject: [PATCH 37/42] tcg: Reindent parts of liveness_pass_1 There are two blocks of the form if (foo) { stuff1; goto bar; } else { baz: stuff2; } which have unnecessary and confusing indentation. Remove the else and unindent stuff2. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg.c | 139 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 71 insertions(+), 68 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index e2ae04d1d0..696a45f7f8 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -2436,47 +2436,46 @@ static void liveness_pass_1(TCGContext *s) } } goto do_remove; - } else { - do_not_remove_call: + } + do_not_remove_call: - /* output args are dead */ - for (i = 0; i < nb_oargs; i++) { - arg_ts = arg_temp(op->args[i]); - if (arg_ts->state & TS_DEAD) { - arg_life |= DEAD_ARG << i; - } - if (arg_ts->state & TS_MEM) { - arg_life |= SYNC_ARG << i; - } - arg_ts->state = TS_DEAD; + /* output args are dead */ + for (i = 0; i < nb_oargs; i++) { + arg_ts = arg_temp(op->args[i]); + if (arg_ts->state & TS_DEAD) { + arg_life |= DEAD_ARG << i; } + if (arg_ts->state & TS_MEM) { + arg_life |= SYNC_ARG << i; + } + arg_ts->state = TS_DEAD; + } - if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | - TCG_CALL_NO_READ_GLOBALS))) { - /* globals should go back to memory */ - for (i = 0; i < nb_globals; i++) { - s->temps[i].state = TS_DEAD | TS_MEM; - } - } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { - /* globals should be synced to memory */ - for (i = 0; i < nb_globals; i++) { - s->temps[i].state |= TS_MEM; - } + if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | + TCG_CALL_NO_READ_GLOBALS))) { + /* globals should go back to memory */ + for (i = 0; i < nb_globals; i++) { + s->temps[i].state = TS_DEAD | TS_MEM; } + } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { + /* globals should be synced to memory */ + for (i = 0; i < nb_globals; i++) { + s->temps[i].state |= TS_MEM; + } + } - /* record arguments that die in this helper */ - for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { - arg_ts = arg_temp(op->args[i]); - if (arg_ts && arg_ts->state & TS_DEAD) { - arg_life |= DEAD_ARG << i; - } + /* record arguments that die in this helper */ + for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { + arg_ts = arg_temp(op->args[i]); + if (arg_ts && arg_ts->state & TS_DEAD) { + arg_life |= DEAD_ARG << i; } - /* input arguments are live for preceding opcodes */ - for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { - arg_ts = arg_temp(op->args[i]); - if (arg_ts) { - arg_ts->state &= ~TS_DEAD; - } + } + /* input arguments are live for preceding opcodes */ + for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { + arg_ts = arg_temp(op->args[i]); + if (arg_ts) { + arg_ts->state &= ~TS_DEAD; } } } @@ -2580,43 +2579,47 @@ static void liveness_pass_1(TCGContext *s) goto do_not_remove; } } - do_remove: - tcg_op_remove(s, op); - } else { - do_not_remove: - /* output args are dead */ - for (i = 0; i < nb_oargs; i++) { - arg_ts = arg_temp(op->args[i]); - if (arg_ts->state & TS_DEAD) { - arg_life |= DEAD_ARG << i; - } - if (arg_ts->state & TS_MEM) { - arg_life |= SYNC_ARG << i; - } - arg_ts->state = TS_DEAD; - } + goto do_remove; + } + goto do_not_remove; - /* if end of basic block, update */ - if (def->flags & TCG_OPF_BB_END) { - tcg_la_bb_end(s); - } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { - /* globals should be synced to memory */ - for (i = 0; i < nb_globals; i++) { - s->temps[i].state |= TS_MEM; - } - } + do_remove: + tcg_op_remove(s, op); + break; - /* record arguments that die in this opcode */ - for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { - arg_ts = arg_temp(op->args[i]); - if (arg_ts->state & TS_DEAD) { - arg_life |= DEAD_ARG << i; - } + do_not_remove: + /* output args are dead */ + for (i = 0; i < nb_oargs; i++) { + arg_ts = arg_temp(op->args[i]); + if (arg_ts->state & TS_DEAD) { + arg_life |= DEAD_ARG << i; } - /* input arguments are live for preceding opcodes */ - for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { - arg_temp(op->args[i])->state &= ~TS_DEAD; + if (arg_ts->state & TS_MEM) { + arg_life |= SYNC_ARG << i; } + arg_ts->state = TS_DEAD; + } + + /* if end of basic block, update */ + if (def->flags & TCG_OPF_BB_END) { + tcg_la_bb_end(s); + } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { + /* globals should be synced to memory */ + for (i = 0; i < nb_globals; i++) { + s->temps[i].state |= TS_MEM; + } + } + + /* record arguments that die in this opcode */ + for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { + arg_ts = arg_temp(op->args[i]); + if (arg_ts->state & TS_DEAD) { + arg_life |= DEAD_ARG << i; + } + } + /* input arguments are live for preceding opcodes */ + for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { + arg_temp(op->args[i])->state &= ~TS_DEAD; } break; } From 2616c8082143373e794b62444bf81754f50dbf6b Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 27 Nov 2018 13:37:24 -0800 Subject: [PATCH 38/42] tcg: Rename and adjust liveness_pass_1 helpers No need for a "tcg_" prefix for a static function; we already have another "la_" prefix for indicating liveness analysis. Pass in nb_globals and nb_temps, as we will already have them in registers for other loops within the parent function. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 696a45f7f8..0afc6ba1e6 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -2367,10 +2367,8 @@ static void reachable_code_pass(TCGContext *s) /* liveness analysis: end of function: all temps are dead, and globals should be in memory. */ -static void tcg_la_func_end(TCGContext *s) +static void la_func_end(TCGContext *s, int ng, int nt) { - int ng = s->nb_globals; - int nt = s->nb_temps; int i; for (i = 0; i < ng; ++i) { @@ -2383,10 +2381,8 @@ static void tcg_la_func_end(TCGContext *s) /* liveness analysis: end of basic block: all temps are dead, globals and local temps should be in memory. */ -static void tcg_la_bb_end(TCGContext *s) +static void la_bb_end(TCGContext *s, int ng, int nt) { - int ng = s->nb_globals; - int nt = s->nb_temps; int i; for (i = 0; i < ng; ++i) { @@ -2405,9 +2401,10 @@ static void tcg_la_bb_end(TCGContext *s) static void liveness_pass_1(TCGContext *s) { int nb_globals = s->nb_globals; + int nb_temps = s->nb_temps; TCGOp *op, *op_prev; - tcg_la_func_end(s); + la_func_end(s, nb_globals, nb_temps); QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, TCGOpHead, link, op_prev) { int i, nb_iargs, nb_oargs; @@ -2602,7 +2599,7 @@ static void liveness_pass_1(TCGContext *s) /* if end of basic block, update */ if (def->flags & TCG_OPF_BB_END) { - tcg_la_bb_end(s); + la_bb_end(s, nb_globals, nb_temps); } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { /* globals should be synced to memory */ for (i = 0; i < nb_globals; i++) { From f65a061c39cc4f9d088201031050e42eb23d5b2a Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 27 Nov 2018 14:00:35 -0800 Subject: [PATCH 39/42] tcg: Split out more subroutines from liveness_pass_1 Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg.c | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 0afc6ba1e6..4f0acf8863 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -2395,6 +2395,26 @@ static void la_bb_end(TCGContext *s, int ng, int nt) } } +/* liveness analysis: sync globals back to memory. */ +static void la_global_sync(TCGContext *s, int ng) +{ + int i; + + for (i = 0; i < ng; ++i) { + s->temps[i].state |= TS_MEM; + } +} + +/* liveness analysis: sync globals back to memory and kill. */ +static void la_global_kill(TCGContext *s, int ng) +{ + int i; + + for (i = 0; i < ng; i++) { + s->temps[i].state = TS_DEAD | TS_MEM; + } +} + /* Liveness analysis : update the opc_arg_life array to tell if a given input arguments is dead. Instructions updating dead temporaries are removed. */ @@ -2450,15 +2470,9 @@ static void liveness_pass_1(TCGContext *s) if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | TCG_CALL_NO_READ_GLOBALS))) { - /* globals should go back to memory */ - for (i = 0; i < nb_globals; i++) { - s->temps[i].state = TS_DEAD | TS_MEM; - } + la_global_kill(s, nb_globals); } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { - /* globals should be synced to memory */ - for (i = 0; i < nb_globals; i++) { - s->temps[i].state |= TS_MEM; - } + la_global_sync(s, nb_globals); } /* record arguments that die in this helper */ @@ -2601,10 +2615,7 @@ static void liveness_pass_1(TCGContext *s) if (def->flags & TCG_OPF_BB_END) { la_bb_end(s, nb_globals, nb_temps); } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { - /* globals should be synced to memory */ - for (i = 0; i < nb_globals; i++) { - s->temps[i].state |= TS_MEM; - } + la_global_sync(s, nb_globals); } /* record arguments that die in this opcode */ From ae36a246ed1a0e96c6c4f478f03d047dfa3a8898 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 27 Nov 2018 13:45:08 -0800 Subject: [PATCH 40/42] tcg: Add TCG_OPF_BB_EXIT Use this to notice the opcodes that exit the TB, which implies that local temps are really dead and need not be synced. Previously we so marked the true end of the TB, but that was immediately overwritten by the la_bb_end invoked by any TCG_OPF_BB_END opcode, like exit_tb. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg-opc.h | 7 ++++--- tcg/tcg.c | 5 ++++- tcg/tcg.h | 14 ++++++++------ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/tcg/tcg-opc.h b/tcg/tcg-opc.h index e3a43aabb6..7a8a3edb5b 100644 --- a/tcg/tcg-opc.h +++ b/tcg/tcg-opc.h @@ -191,9 +191,10 @@ DEF(mulsh_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_mulsh_i64)) /* QEMU specific */ DEF(insn_start, 0, 0, TLADDR_ARGS * TARGET_INSN_START_WORDS, TCG_OPF_NOT_PRESENT) -DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_END) -DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_END) -DEF(goto_ptr, 0, 1, 0, TCG_OPF_BB_END | IMPL(TCG_TARGET_HAS_goto_ptr)) +DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_EXIT | TCG_OPF_BB_END) +DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_EXIT | TCG_OPF_BB_END) +DEF(goto_ptr, 0, 1, 0, + TCG_OPF_BB_EXIT | TCG_OPF_BB_END | IMPL(TCG_TARGET_HAS_goto_ptr)) DEF(qemu_ld_i32, 1, TLADDR_ARGS, 1, TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS) diff --git a/tcg/tcg.c b/tcg/tcg.c index 4f0acf8863..d40edb4f57 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -2424,6 +2424,7 @@ static void liveness_pass_1(TCGContext *s) int nb_temps = s->nb_temps; TCGOp *op, *op_prev; + /* ??? Should be redundant with the exit_tb that ends the TB. */ la_func_end(s, nb_globals, nb_temps); QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, TCGOpHead, link, op_prev) { @@ -2612,7 +2613,9 @@ static void liveness_pass_1(TCGContext *s) } /* if end of basic block, update */ - if (def->flags & TCG_OPF_BB_END) { + if (def->flags & TCG_OPF_BB_EXIT) { + la_func_end(s, nb_globals, nb_temps); + } else if (def->flags & TCG_OPF_BB_END) { la_bb_end(s, nb_globals, nb_temps); } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { la_global_sync(s, nb_globals); diff --git a/tcg/tcg.h b/tcg/tcg.h index 5e5cf686a3..3a629991ca 100644 --- a/tcg/tcg.h +++ b/tcg/tcg.h @@ -1030,20 +1030,22 @@ typedef struct TCGArgConstraint { /* Bits for TCGOpDef->flags, 8 bits available. */ enum { + /* Instruction exits the translation block. */ + TCG_OPF_BB_EXIT = 0x01, /* Instruction defines the end of a basic block. */ - TCG_OPF_BB_END = 0x01, + TCG_OPF_BB_END = 0x02, /* Instruction clobbers call registers and potentially update globals. */ - TCG_OPF_CALL_CLOBBER = 0x02, + TCG_OPF_CALL_CLOBBER = 0x04, /* Instruction has side effects: it cannot be removed if its outputs are not used, and might trigger exceptions. */ - TCG_OPF_SIDE_EFFECTS = 0x04, + TCG_OPF_SIDE_EFFECTS = 0x08, /* Instruction operands are 64-bits (otherwise 32-bits). */ - TCG_OPF_64BIT = 0x08, + TCG_OPF_64BIT = 0x10, /* Instruction is optional and not implemented by the host, or insn is generic and should not be implemened by the host. */ - TCG_OPF_NOT_PRESENT = 0x10, + TCG_OPF_NOT_PRESENT = 0x20, /* Instruction operands are vectors. */ - TCG_OPF_VECTOR = 0x20, + TCG_OPF_VECTOR = 0x40, }; typedef struct TCGOpDef { From 25f49c5f1508ddf081ce89fa6bbfd87a51eea37b Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 27 Nov 2018 12:45:26 -0800 Subject: [PATCH 41/42] tcg: Record register preferences during liveness With these preferences, we can arrange for function call arguments to be computed into the proper registers instead of requiring extra moves. Reviewed-by: Emilio G. Cota Signed-off-by: Richard Henderson --- tcg/tcg.c | 197 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 165 insertions(+), 32 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index d40edb4f57..5f8c09b0b4 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -2365,6 +2365,21 @@ static void reachable_code_pass(TCGContext *s) #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) +/* For liveness_pass_1, the register preferences for a given temp. */ +static inline TCGRegSet *la_temp_pref(TCGTemp *ts) +{ + return ts->state_ptr; +} + +/* For liveness_pass_1, reset the preferences for a given temp to the + * maximal regset for its type. + */ +static inline void la_reset_pref(TCGTemp *ts) +{ + *la_temp_pref(ts) + = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); +} + /* liveness analysis: end of function: all temps are dead, and globals should be in memory. */ static void la_func_end(TCGContext *s, int ng, int nt) @@ -2373,9 +2388,11 @@ static void la_func_end(TCGContext *s, int ng, int nt) for (i = 0; i < ng; ++i) { s->temps[i].state = TS_DEAD | TS_MEM; + la_reset_pref(&s->temps[i]); } for (i = ng; i < nt; ++i) { s->temps[i].state = TS_DEAD; + la_reset_pref(&s->temps[i]); } } @@ -2387,11 +2404,13 @@ static void la_bb_end(TCGContext *s, int ng, int nt) for (i = 0; i < ng; ++i) { s->temps[i].state = TS_DEAD | TS_MEM; + la_reset_pref(&s->temps[i]); } for (i = ng; i < nt; ++i) { s->temps[i].state = (s->temps[i].temp_local ? TS_DEAD | TS_MEM : TS_DEAD); + la_reset_pref(&s->temps[i]); } } @@ -2401,7 +2420,12 @@ static void la_global_sync(TCGContext *s, int ng) int i; for (i = 0; i < ng; ++i) { - s->temps[i].state |= TS_MEM; + int state = s->temps[i].state; + s->temps[i].state = state | TS_MEM; + if (state == TS_DEAD) { + /* If the global was previously dead, reset prefs. */ + la_reset_pref(&s->temps[i]); + } } } @@ -2412,6 +2436,29 @@ static void la_global_kill(TCGContext *s, int ng) for (i = 0; i < ng; i++) { s->temps[i].state = TS_DEAD | TS_MEM; + la_reset_pref(&s->temps[i]); + } +} + +/* liveness analysis: note live globals crossing calls. */ +static void la_cross_call(TCGContext *s, int nt) +{ + TCGRegSet mask = ~tcg_target_call_clobber_regs; + int i; + + for (i = 0; i < nt; i++) { + TCGTemp *ts = &s->temps[i]; + if (!(ts->state & TS_DEAD)) { + TCGRegSet *pset = la_temp_pref(ts); + TCGRegSet set = *pset; + + set &= mask; + /* If the combination is not possible, restart. */ + if (set == 0) { + set = tcg_target_available_regs[ts->type] & mask; + } + *pset = set; + } } } @@ -2423,16 +2470,23 @@ static void liveness_pass_1(TCGContext *s) int nb_globals = s->nb_globals; int nb_temps = s->nb_temps; TCGOp *op, *op_prev; + TCGRegSet *prefs; + int i; + + prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); + for (i = 0; i < nb_temps; ++i) { + s->temps[i].state_ptr = prefs + i; + } /* ??? Should be redundant with the exit_tb that ends the TB. */ la_func_end(s, nb_globals, nb_temps); QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, TCGOpHead, link, op_prev) { - int i, nb_iargs, nb_oargs; + int nb_iargs, nb_oargs; TCGOpcode opc_new, opc_new2; bool have_opc_new2; TCGLifeData arg_life = 0; - TCGTemp *arg_ts; + TCGTemp *ts; TCGOpcode opc = op->opc; const TCGOpDef *def = &tcg_op_defs[opc]; @@ -2440,6 +2494,7 @@ static void liveness_pass_1(TCGContext *s) case INDEX_op_call: { int call_flags; + int nb_call_regs; nb_oargs = TCGOP_CALLO(op); nb_iargs = TCGOP_CALLI(op); @@ -2448,8 +2503,8 @@ static void liveness_pass_1(TCGContext *s) /* pure functions can be removed if their result is unused */ if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { for (i = 0; i < nb_oargs; i++) { - arg_ts = arg_temp(op->args[i]); - if (arg_ts->state != TS_DEAD) { + ts = arg_temp(op->args[i]); + if (ts->state != TS_DEAD) { goto do_not_remove_call; } } @@ -2457,16 +2512,20 @@ static void liveness_pass_1(TCGContext *s) } do_not_remove_call: - /* output args are dead */ + /* Output args are dead. */ for (i = 0; i < nb_oargs; i++) { - arg_ts = arg_temp(op->args[i]); - if (arg_ts->state & TS_DEAD) { + ts = arg_temp(op->args[i]); + if (ts->state & TS_DEAD) { arg_life |= DEAD_ARG << i; } - if (arg_ts->state & TS_MEM) { + if (ts->state & TS_MEM) { arg_life |= SYNC_ARG << i; } - arg_ts->state = TS_DEAD; + ts->state = TS_DEAD; + la_reset_pref(ts); + + /* Not used -- it will be tcg_target_call_oarg_regs[i]. */ + op->output_pref[i] = 0; } if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | @@ -2476,18 +2535,42 @@ static void liveness_pass_1(TCGContext *s) la_global_sync(s, nb_globals); } - /* record arguments that die in this helper */ + /* Record arguments that die in this helper. */ for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { - arg_ts = arg_temp(op->args[i]); - if (arg_ts && arg_ts->state & TS_DEAD) { + ts = arg_temp(op->args[i]); + if (ts && ts->state & TS_DEAD) { arg_life |= DEAD_ARG << i; } } - /* input arguments are live for preceding opcodes */ - for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { - arg_ts = arg_temp(op->args[i]); - if (arg_ts) { - arg_ts->state &= ~TS_DEAD; + + /* For all live registers, remove call-clobbered prefs. */ + la_cross_call(s, nb_temps); + + nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); + + /* Input arguments are live for preceding opcodes. */ + for (i = 0; i < nb_iargs; i++) { + ts = arg_temp(op->args[i + nb_oargs]); + if (ts && ts->state & TS_DEAD) { + /* For those arguments that die, and will be allocated + * in registers, clear the register set for that arg, + * to be filled in below. For args that will be on + * the stack, reset to any available reg. + */ + *la_temp_pref(ts) + = (i < nb_call_regs ? 0 : + tcg_target_available_regs[ts->type]); + ts->state &= ~TS_DEAD; + } + } + + /* For each input argument, add its input register to prefs. + If a temp is used once, this produces a single set bit. */ + for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) { + ts = arg_temp(op->args[i + nb_oargs]); + if (ts) { + tcg_regset_set_reg(*la_temp_pref(ts), + tcg_target_call_iarg_regs[i]); } } } @@ -2496,7 +2579,9 @@ static void liveness_pass_1(TCGContext *s) break; case INDEX_op_discard: /* mark the temporary as dead */ - arg_temp(op->args[0])->state = TS_DEAD; + ts = arg_temp(op->args[0]); + ts->state = TS_DEAD; + la_reset_pref(ts); break; case INDEX_op_add2_i32: @@ -2600,43 +2685,91 @@ static void liveness_pass_1(TCGContext *s) break; do_not_remove: - /* output args are dead */ for (i = 0; i < nb_oargs; i++) { - arg_ts = arg_temp(op->args[i]); - if (arg_ts->state & TS_DEAD) { + ts = arg_temp(op->args[i]); + + /* Remember the preference of the uses that followed. */ + op->output_pref[i] = *la_temp_pref(ts); + + /* Output args are dead. */ + if (ts->state & TS_DEAD) { arg_life |= DEAD_ARG << i; } - if (arg_ts->state & TS_MEM) { + if (ts->state & TS_MEM) { arg_life |= SYNC_ARG << i; } - arg_ts->state = TS_DEAD; + ts->state = TS_DEAD; + la_reset_pref(ts); } - /* if end of basic block, update */ + /* If end of basic block, update. */ if (def->flags & TCG_OPF_BB_EXIT) { la_func_end(s, nb_globals, nb_temps); } else if (def->flags & TCG_OPF_BB_END) { la_bb_end(s, nb_globals, nb_temps); } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { la_global_sync(s, nb_globals); + if (def->flags & TCG_OPF_CALL_CLOBBER) { + la_cross_call(s, nb_temps); + } } - /* record arguments that die in this opcode */ + /* Record arguments that die in this opcode. */ for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { - arg_ts = arg_temp(op->args[i]); - if (arg_ts->state & TS_DEAD) { + ts = arg_temp(op->args[i]); + if (ts->state & TS_DEAD) { arg_life |= DEAD_ARG << i; } } - /* input arguments are live for preceding opcodes */ + + /* Input arguments are live for preceding opcodes. */ for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { - arg_temp(op->args[i])->state &= ~TS_DEAD; + ts = arg_temp(op->args[i]); + if (ts->state & TS_DEAD) { + /* For operands that were dead, initially allow + all regs for the type. */ + *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; + ts->state &= ~TS_DEAD; + } + } + + /* Incorporate constraints for this operand. */ + switch (opc) { + case INDEX_op_mov_i32: + case INDEX_op_mov_i64: + /* Note that these are TCG_OPF_NOT_PRESENT and do not + have proper constraints. That said, special case + moves to propagate preferences backward. */ + if (IS_DEAD_ARG(1)) { + *la_temp_pref(arg_temp(op->args[0])) + = *la_temp_pref(arg_temp(op->args[1])); + } + break; + + default: + for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { + const TCGArgConstraint *ct = &def->args_ct[i]; + TCGRegSet set, *pset; + + ts = arg_temp(op->args[i]); + pset = la_temp_pref(ts); + set = *pset; + + set &= ct->u.regs; + if (ct->ct & TCG_CT_IALIAS) { + set &= op->output_pref[ct->alias_index]; + } + /* If the combination is not possible, restart. */ + if (set == 0) { + set = ct->u.regs; + } + *pset = set; + } + break; } break; } op->life = arg_life; - op->output_pref[0] = 0; - op->output_pref[1] = 0; } } From 4250da10923347c9ee907f8d72bd93dfa5ee8742 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 11 Dec 2018 10:25:02 -0600 Subject: [PATCH 42/42] tcg: Improve call argument loading Free the argument register only after we have verified that the temporary is not already in that register. This case is likely now that we are back propagating the preferred register. Signed-off-by: Richard Henderson --- tcg/tcg.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 5f8c09b0b4..c54b119020 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -3611,15 +3611,16 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) if (arg != TCG_CALL_DUMMY_ARG) { ts = arg_temp(arg); reg = tcg_target_call_iarg_regs[i]; - tcg_reg_free(s, reg, allocated_regs); if (ts->val_type == TEMP_VAL_REG) { if (ts->reg != reg) { + tcg_reg_free(s, reg, allocated_regs); tcg_out_mov(s, ts->type, reg, ts->reg); } } else { TCGRegSet arg_set = 0; + tcg_reg_free(s, reg, allocated_regs); tcg_regset_set_reg(arg_set, reg); temp_load(s, ts, arg_set, allocated_regs, 0); }