target/arm: Use tcg_gen_atomic_cmpxchg_i128 for STXP

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
Message-Id: <20221112042555.2622152-2-richard.henderson@linaro.org>
This commit is contained in:
Richard Henderson 2022-11-12 14:25:54 +10:00
parent d1beee4da1
commit 546789c7df
3 changed files with 35 additions and 135 deletions

View File

@ -505,110 +505,6 @@ uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes)
return crc32c(acc, buf, bytes) ^ 0xffffffff;
}
uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr,
uint64_t new_lo, uint64_t new_hi)
{
Int128 cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
Int128 newv = int128_make128(new_lo, new_hi);
Int128 oldv;
uintptr_t ra = GETPC();
uint64_t o0, o1;
bool success;
int mem_idx = cpu_mmu_index(env, false);
MemOpIdx oi0 = make_memop_idx(MO_LEUQ | MO_ALIGN_16, mem_idx);
MemOpIdx oi1 = make_memop_idx(MO_LEUQ, mem_idx);
o0 = cpu_ldq_le_mmu(env, addr + 0, oi0, ra);
o1 = cpu_ldq_le_mmu(env, addr + 8, oi1, ra);
oldv = int128_make128(o0, o1);
success = int128_eq(oldv, cmpv);
if (success) {
cpu_stq_le_mmu(env, addr + 0, int128_getlo(newv), oi1, ra);
cpu_stq_le_mmu(env, addr + 8, int128_gethi(newv), oi1, ra);
}
return !success;
}
uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr,
uint64_t new_lo, uint64_t new_hi)
{
Int128 oldv, cmpv, newv;
uintptr_t ra = GETPC();
bool success;
int mem_idx;
MemOpIdx oi;
assert(HAVE_CMPXCHG128);
mem_idx = cpu_mmu_index(env, false);
oi = make_memop_idx(MO_LE | MO_128 | MO_ALIGN, mem_idx);
cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
newv = int128_make128(new_lo, new_hi);
oldv = cpu_atomic_cmpxchgo_le_mmu(env, addr, cmpv, newv, oi, ra);
success = int128_eq(oldv, cmpv);
return !success;
}
uint64_t HELPER(paired_cmpxchg64_be)(CPUARMState *env, uint64_t addr,
uint64_t new_lo, uint64_t new_hi)
{
/*
* High and low need to be switched here because this is not actually a
* 128bit store but two doublewords stored consecutively
*/
Int128 cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
Int128 newv = int128_make128(new_hi, new_lo);
Int128 oldv;
uintptr_t ra = GETPC();
uint64_t o0, o1;
bool success;
int mem_idx = cpu_mmu_index(env, false);
MemOpIdx oi0 = make_memop_idx(MO_BEUQ | MO_ALIGN_16, mem_idx);
MemOpIdx oi1 = make_memop_idx(MO_BEUQ, mem_idx);
o1 = cpu_ldq_be_mmu(env, addr + 0, oi0, ra);
o0 = cpu_ldq_be_mmu(env, addr + 8, oi1, ra);
oldv = int128_make128(o0, o1);
success = int128_eq(oldv, cmpv);
if (success) {
cpu_stq_be_mmu(env, addr + 0, int128_gethi(newv), oi1, ra);
cpu_stq_be_mmu(env, addr + 8, int128_getlo(newv), oi1, ra);
}
return !success;
}
uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr,
uint64_t new_lo, uint64_t new_hi)
{
Int128 oldv, cmpv, newv;
uintptr_t ra = GETPC();
bool success;
int mem_idx;
MemOpIdx oi;
assert(HAVE_CMPXCHG128);
mem_idx = cpu_mmu_index(env, false);
oi = make_memop_idx(MO_BE | MO_128 | MO_ALIGN, mem_idx);
/*
* High and low need to be switched here because this is not actually a
* 128bit store but two doublewords stored consecutively
*/
cmpv = int128_make128(env->exclusive_high, env->exclusive_val);
newv = int128_make128(new_hi, new_lo);
oldv = cpu_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
success = int128_eq(oldv, cmpv);
return !success;
}
/* Writes back the old data into Rs. */
void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
uint64_t new_lo, uint64_t new_hi)

View File

@ -50,12 +50,6 @@ DEF_HELPER_FLAGS_2(frecpx_f16, TCG_CALL_NO_RWG, f16, f16, ptr)
DEF_HELPER_FLAGS_2(fcvtx_f64_to_f32, TCG_CALL_NO_RWG, f32, f64, env)
DEF_HELPER_FLAGS_3(crc32_64, TCG_CALL_NO_RWG_SE, i64, i64, i64, i32)
DEF_HELPER_FLAGS_3(crc32c_64, TCG_CALL_NO_RWG_SE, i64, i64, i64, i32)
DEF_HELPER_FLAGS_4(paired_cmpxchg64_le, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
DEF_HELPER_FLAGS_4(paired_cmpxchg64_le_parallel, TCG_CALL_NO_WG,
i64, env, i64, i64, i64)
DEF_HELPER_FLAGS_4(paired_cmpxchg64_be, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
DEF_HELPER_FLAGS_4(paired_cmpxchg64_be_parallel, TCG_CALL_NO_WG,
i64, env, i64, i64, i64)
DEF_HELPER_5(casp_le_parallel, void, env, i32, i64, i64, i64)
DEF_HELPER_5(casp_be_parallel, void, env, i32, i64, i64, i64)
DEF_HELPER_FLAGS_3(advsimd_maxh, TCG_CALL_NO_RWG, f16, f16, f16, ptr)

View File

@ -2601,32 +2601,42 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
get_mem_index(s),
MO_64 | MO_ALIGN | s->be_data);
tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
} else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
if (!HAVE_CMPXCHG128) {
gen_helper_exit_atomic(cpu_env);
/*
* Produce a result so we have a well-formed opcode
* stream when the following (dead) code uses 'tmp'.
* TCG will remove the dead ops for us.
*/
tcg_gen_movi_i64(tmp, 0);
} else if (s->be_data == MO_LE) {
gen_helper_paired_cmpxchg64_le_parallel(tmp, cpu_env,
cpu_exclusive_addr,
cpu_reg(s, rt),
cpu_reg(s, rt2));
} else {
gen_helper_paired_cmpxchg64_be_parallel(tmp, cpu_env,
cpu_exclusive_addr,
cpu_reg(s, rt),
cpu_reg(s, rt2));
TCGv_i128 t16 = tcg_temp_new_i128();
TCGv_i128 c16 = tcg_temp_new_i128();
TCGv_i64 a, b;
if (s->be_data == MO_LE) {
tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt), cpu_reg(s, rt2));
tcg_gen_concat_i64_i128(c16, cpu_exclusive_val,
cpu_exclusive_high);
} else {
tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt2), cpu_reg(s, rt));
tcg_gen_concat_i64_i128(c16, cpu_exclusive_high,
cpu_exclusive_val);
}
} else if (s->be_data == MO_LE) {
gen_helper_paired_cmpxchg64_le(tmp, cpu_env, cpu_exclusive_addr,
cpu_reg(s, rt), cpu_reg(s, rt2));
tcg_gen_atomic_cmpxchg_i128(t16, cpu_exclusive_addr, c16, t16,
get_mem_index(s),
MO_128 | MO_ALIGN | s->be_data);
tcg_temp_free_i128(c16);
a = tcg_temp_new_i64();
b = tcg_temp_new_i64();
if (s->be_data == MO_LE) {
tcg_gen_extr_i128_i64(a, b, t16);
} else {
gen_helper_paired_cmpxchg64_be(tmp, cpu_env, cpu_exclusive_addr,
cpu_reg(s, rt), cpu_reg(s, rt2));
tcg_gen_extr_i128_i64(b, a, t16);
}
tcg_gen_xor_i64(a, a, cpu_exclusive_val);
tcg_gen_xor_i64(b, b, cpu_exclusive_high);
tcg_gen_or_i64(tmp, a, b);
tcg_temp_free_i64(a);
tcg_temp_free_i64(b);
tcg_temp_free_i128(t16);
tcg_gen_setcondi_i64(TCG_COND_NE, tmp, tmp, 0);
}
} else {
tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val,