mirrors/qemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
ee36a772c0
qemu/target/arm/tcg/helper-a64.c
Richard Henderson 64678fc45d target/arm: Fix BTI versus CF_PCREL 
With pcrel, we cannot check the guarded page bit at translation
time, as different mappings of the same physical page may or may
not have the GP bit set.

Instead, add a couple of helpers to check the page at runtime,
after all other filters that might obviate the need for the check.

The set_btype_for_br call must be moved after the gen_a64_set_pc
call to ensure the current pc can still be computed.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
Message-id: 20240802003028.795476-1-richard.henderson@linaro.org
Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
2024-08-09 17:37:54 +01:00

1919 lines
58 KiB
C
Raw Blame History

/*
* AArch64 specific helpers
*
* Copyright (c) 2013 Alexander Graf <agraf@suse.de>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
#include "qemu/osdep.h"
#include "qemu/units.h"
#include "cpu.h"
#include "gdbstub/helpers.h"
#include "exec/helper-proto.h"
#include "qemu/host-utils.h"
#include "qemu/log.h"
#include "qemu/main-loop.h"
#include "qemu/bitops.h"
#include "internals.h"
#include "qemu/crc32c.h"
#include "exec/exec-all.h"
#include "exec/cpu_ldst.h"
#include "qemu/int128.h"
#include "qemu/atomic128.h"
#include "fpu/softfloat.h"
#include <zlib.h> /* For crc32 */
/* C2.4.7 Multiply and divide */
/* special cases for 0 and LLONG_MIN are mandated by the standard */
uint64_t HELPER(udiv64)(uint64_t num, uint64_t den)
{
if (den == 0) {
return 0;
}
return num / den;
}
int64_t HELPER(sdiv64)(int64_t num, int64_t den)
{
if (den == 0) {
return 0;
}
if (num == LLONG_MIN && den == -1) {
return LLONG_MIN;
}
return num / den;
}
uint64_t HELPER(rbit64)(uint64_t x)
{
return revbit64(x);
}
void HELPER(msr_i_spsel)(CPUARMState *env, uint32_t imm)
{
update_spsel(env, imm);
}
void HELPER(msr_set_allint_el1)(CPUARMState *env)
{
/* ALLINT update to PSTATE. */
if (arm_hcrx_el2_eff(env) & HCRX_TALLINT) {
raise_exception_ra(env, EXCP_UDEF,
syn_aa64_sysregtrap(0, 1, 0, 4, 1, 0x1f, 0), 2,
GETPC());
}
env->pstate |= PSTATE_ALLINT;
}
static void daif_check(CPUARMState *env, uint32_t op,
uint32_t imm, uintptr_t ra)
{
/* DAIF update to PSTATE. This is OK from EL0 only if UMA is set. */
if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) {
raise_exception_ra(env, EXCP_UDEF,
syn_aa64_sysregtrap(0, extract32(op, 0, 3),
extract32(op, 3, 3), 4,
imm, 0x1f, 0),
exception_target_el(env), ra);
}
}
void HELPER(msr_i_daifset)(CPUARMState *env, uint32_t imm)
{
daif_check(env, 0x1e, imm, GETPC());
env->daif |= (imm << 6) & PSTATE_DAIF;
arm_rebuild_hflags(env);
}
void HELPER(msr_i_daifclear)(CPUARMState *env, uint32_t imm)
{
daif_check(env, 0x1f, imm, GETPC());
env->daif &= ~((imm << 6) & PSTATE_DAIF);
arm_rebuild_hflags(env);
}
/* Convert a softfloat float_relation_ (as returned by
* the float*_compare functions) to the correct ARM
* NZCV flag state.
*/
static inline uint32_t float_rel_to_flags(int res)
{
uint64_t flags;
switch (res) {
case float_relation_equal:
flags = PSTATE_Z | PSTATE_C;
break;
case float_relation_less:
flags = PSTATE_N;
break;
case float_relation_greater:
flags = PSTATE_C;
break;
case float_relation_unordered:
default:
flags = PSTATE_C | PSTATE_V;
break;
}
return flags;
}
uint64_t HELPER(vfp_cmph_a64)(uint32_t x, uint32_t y, void *fp_status)
{
return float_rel_to_flags(float16_compare_quiet(x, y, fp_status));
}
uint64_t HELPER(vfp_cmpeh_a64)(uint32_t x, uint32_t y, void *fp_status)
{
return float_rel_to_flags(float16_compare(x, y, fp_status));
}
uint64_t HELPER(vfp_cmps_a64)(float32 x, float32 y, void *fp_status)
{
return float_rel_to_flags(float32_compare_quiet(x, y, fp_status));
}
uint64_t HELPER(vfp_cmpes_a64)(float32 x, float32 y, void *fp_status)
{
return float_rel_to_flags(float32_compare(x, y, fp_status));
}
uint64_t HELPER(vfp_cmpd_a64)(float64 x, float64 y, void *fp_status)
{
return float_rel_to_flags(float64_compare_quiet(x, y, fp_status));
}
uint64_t HELPER(vfp_cmped_a64)(float64 x, float64 y, void *fp_status)
{
return float_rel_to_flags(float64_compare(x, y, fp_status));
}
float32 HELPER(vfp_mulxs)(float32 a, float32 b, void *fpstp)
{
float_status *fpst = fpstp;
a = float32_squash_input_denormal(a, fpst);
b = float32_squash_input_denormal(b, fpst);
if ((float32_is_zero(a) && float32_is_infinity(b)) ||
(float32_is_infinity(a) && float32_is_zero(b))) {
/* 2.0 with the sign bit set to sign(A) XOR sign(B) */
return make_float32((1U << 30) |
((float32_val(a) ^ float32_val(b)) & (1U << 31)));
}
return float32_mul(a, b, fpst);
}
float64 HELPER(vfp_mulxd)(float64 a, float64 b, void *fpstp)
{
float_status *fpst = fpstp;
a = float64_squash_input_denormal(a, fpst);
b = float64_squash_input_denormal(b, fpst);
if ((float64_is_zero(a) && float64_is_infinity(b)) ||
(float64_is_infinity(a) && float64_is_zero(b))) {
/* 2.0 with the sign bit set to sign(A) XOR sign(B) */
return make_float64((1ULL << 62) |
((float64_val(a) ^ float64_val(b)) & (1ULL << 63)));
}
return float64_mul(a, b, fpst);
}
/* 64bit/double versions of the neon float compare functions */
uint64_t HELPER(neon_ceq_f64)(float64 a, float64 b, void *fpstp)
{
float_status *fpst = fpstp;
return -float64_eq_quiet(a, b, fpst);
}
uint64_t HELPER(neon_cge_f64)(float64 a, float64 b, void *fpstp)
{
float_status *fpst = fpstp;
return -float64_le(b, a, fpst);
}
uint64_t HELPER(neon_cgt_f64)(float64 a, float64 b, void *fpstp)
{
float_status *fpst = fpstp;
return -float64_lt(b, a, fpst);
}
/* Reciprocal step and sqrt step. Note that unlike the A32/T32
* versions, these do a fully fused multiply-add or
* multiply-add-and-halve.
*/
uint32_t HELPER(recpsf_f16)(uint32_t a, uint32_t b, void *fpstp)
{
float_status *fpst = fpstp;
a = float16_squash_input_denormal(a, fpst);
b = float16_squash_input_denormal(b, fpst);
a = float16_chs(a);
if ((float16_is_infinity(a) && float16_is_zero(b)) ||
(float16_is_infinity(b) && float16_is_zero(a))) {
return float16_two;
}
return float16_muladd(a, b, float16_two, 0, fpst);
}
float32 HELPER(recpsf_f32)(float32 a, float32 b, void *fpstp)
{
float_status *fpst = fpstp;
a = float32_squash_input_denormal(a, fpst);
b = float32_squash_input_denormal(b, fpst);
a = float32_chs(a);
if ((float32_is_infinity(a) && float32_is_zero(b)) ||
(float32_is_infinity(b) && float32_is_zero(a))) {
return float32_two;
}
return float32_muladd(a, b, float32_two, 0, fpst);
}
float64 HELPER(recpsf_f64)(float64 a, float64 b, void *fpstp)
{
float_status *fpst = fpstp;
a = float64_squash_input_denormal(a, fpst);
b = float64_squash_input_denormal(b, fpst);
a = float64_chs(a);
if ((float64_is_infinity(a) && float64_is_zero(b)) ||
(float64_is_infinity(b) && float64_is_zero(a))) {
return float64_two;
}
return float64_muladd(a, b, float64_two, 0, fpst);
}
uint32_t HELPER(rsqrtsf_f16)(uint32_t a, uint32_t b, void *fpstp)
{
float_status *fpst = fpstp;
a = float16_squash_input_denormal(a, fpst);
b = float16_squash_input_denormal(b, fpst);
a = float16_chs(a);
if ((float16_is_infinity(a) && float16_is_zero(b)) ||
(float16_is_infinity(b) && float16_is_zero(a))) {
return float16_one_point_five;
}
return float16_muladd(a, b, float16_three, float_muladd_halve_result, fpst);
}
float32 HELPER(rsqrtsf_f32)(float32 a, float32 b, void *fpstp)
{
float_status *fpst = fpstp;
a = float32_squash_input_denormal(a, fpst);
b = float32_squash_input_denormal(b, fpst);
a = float32_chs(a);
if ((float32_is_infinity(a) && float32_is_zero(b)) ||
(float32_is_infinity(b) && float32_is_zero(a))) {
return float32_one_point_five;
}
return float32_muladd(a, b, float32_three, float_muladd_halve_result, fpst);
}
float64 HELPER(rsqrtsf_f64)(float64 a, float64 b, void *fpstp)
{
float_status *fpst = fpstp;
a = float64_squash_input_denormal(a, fpst);
b = float64_squash_input_denormal(b, fpst);
a = float64_chs(a);
if ((float64_is_infinity(a) && float64_is_zero(b)) ||
(float64_is_infinity(b) && float64_is_zero(a))) {
return float64_one_point_five;
}
return float64_muladd(a, b, float64_three, float_muladd_halve_result, fpst);
}
/* Pairwise long add: add pairs of adjacent elements into
* double-width elements in the result (eg _s8 is an 8x8->16 op)
*/
uint64_t HELPER(neon_addlp_s8)(uint64_t a)
{
uint64_t nsignmask = 0x0080008000800080ULL;
uint64_t wsignmask = 0x8000800080008000ULL;
uint64_t elementmask = 0x00ff00ff00ff00ffULL;
uint64_t tmp1, tmp2;
uint64_t res, signres;
/* Extract odd elements, sign extend each to a 16 bit field */
tmp1 = a & elementmask;
tmp1 ^= nsignmask;
tmp1 |= wsignmask;
tmp1 = (tmp1 - nsignmask) ^ wsignmask;
/* Ditto for the even elements */
tmp2 = (a >> 8) & elementmask;
tmp2 ^= nsignmask;
tmp2 |= wsignmask;
tmp2 = (tmp2 - nsignmask) ^ wsignmask;
/* calculate the result by summing bits 0..14, 16..22, etc,
* and then adjusting the sign bits 15, 23, etc manually.
* This ensures the addition can't overflow the 16 bit field.
*/
signres = (tmp1 ^ tmp2) & wsignmask;
res = (tmp1 & ~wsignmask) + (tmp2 & ~wsignmask);
res ^= signres;
return res;
}
uint64_t HELPER(neon_addlp_u8)(uint64_t a)
{
uint64_t tmp;
tmp = a & 0x00ff00ff00ff00ffULL;
tmp += (a >> 8) & 0x00ff00ff00ff00ffULL;
return tmp;
}
uint64_t HELPER(neon_addlp_s16)(uint64_t a)
{
int32_t reslo, reshi;
reslo = (int32_t)(int16_t)a + (int32_t)(int16_t)(a >> 16);
reshi = (int32_t)(int16_t)(a >> 32) + (int32_t)(int16_t)(a >> 48);
return (uint32_t)reslo | (((uint64_t)reshi) << 32);
}
uint64_t HELPER(neon_addlp_u16)(uint64_t a)
{
uint64_t tmp;
tmp = a & 0x0000ffff0000ffffULL;
tmp += (a >> 16) & 0x0000ffff0000ffffULL;
return tmp;
}
/* Floating-point reciprocal exponent - see FPRecpX in ARM ARM */
uint32_t HELPER(frecpx_f16)(uint32_t a, void *fpstp)
{
float_status *fpst = fpstp;
uint16_t val16, sbit;
int16_t exp;
if (float16_is_any_nan(a)) {
float16 nan = a;
if (float16_is_signaling_nan(a, fpst)) {
float_raise(float_flag_invalid, fpst);
if (!fpst->default_nan_mode) {
nan = float16_silence_nan(a, fpst);
}
}
if (fpst->default_nan_mode) {
nan = float16_default_nan(fpst);
}
return nan;
}
a = float16_squash_input_denormal(a, fpst);
val16 = float16_val(a);
sbit = 0x8000 & val16;
exp = extract32(val16, 10, 5);
if (exp == 0) {
return make_float16(deposit32(sbit, 10, 5, 0x1e));
} else {
return make_float16(deposit32(sbit, 10, 5, ~exp));
}
}
float32 HELPER(frecpx_f32)(float32 a, void *fpstp)
{
float_status *fpst = fpstp;
uint32_t val32, sbit;
int32_t exp;
if (float32_is_any_nan(a)) {
float32 nan = a;
if (float32_is_signaling_nan(a, fpst)) {
float_raise(float_flag_invalid, fpst);
if (!fpst->default_nan_mode) {
nan = float32_silence_nan(a, fpst);
}
}
if (fpst->default_nan_mode) {
nan = float32_default_nan(fpst);
}
return nan;
}
a = float32_squash_input_denormal(a, fpst);
val32 = float32_val(a);
sbit = 0x80000000ULL & val32;
exp = extract32(val32, 23, 8);
if (exp == 0) {
return make_float32(sbit | (0xfe << 23));
} else {
return make_float32(sbit | (~exp & 0xff) << 23);
}
}
float64 HELPER(frecpx_f64)(float64 a, void *fpstp)
{
float_status *fpst = fpstp;
uint64_t val64, sbit;
int64_t exp;
if (float64_is_any_nan(a)) {
float64 nan = a;
if (float64_is_signaling_nan(a, fpst)) {
float_raise(float_flag_invalid, fpst);
if (!fpst->default_nan_mode) {
nan = float64_silence_nan(a, fpst);
}
}
if (fpst->default_nan_mode) {
nan = float64_default_nan(fpst);
}
return nan;
}
a = float64_squash_input_denormal(a, fpst);
val64 = float64_val(a);
sbit = 0x8000000000000000ULL & val64;
exp = extract64(float64_val(a), 52, 11);
if (exp == 0) {
return make_float64(sbit | (0x7feULL << 52));
} else {
return make_float64(sbit | (~exp & 0x7ffULL) << 52);
}
}
float32 HELPER(fcvtx_f64_to_f32)(float64 a, CPUARMState *env)
{
/* Von Neumann rounding is implemented by using round-to-zero
* and then setting the LSB of the result if Inexact was raised.
*/
float32 r;
float_status *fpst = &env->vfp.fp_status;
float_status tstat = *fpst;
int exflags;
set_float_rounding_mode(float_round_to_zero, &tstat);
set_float_exception_flags(0, &tstat);
r = float64_to_float32(a, &tstat);
exflags = get_float_exception_flags(&tstat);
if (exflags & float_flag_inexact) {
r = make_float32(float32_val(r) | 1);
}
exflags |= get_float_exception_flags(fpst);
set_float_exception_flags(exflags, fpst);
return r;
}
/* 64-bit versions of the CRC helpers. Note that although the operation
* (and the prototypes of crc32c() and crc32() mean that only the bottom
* 32 bits of the accumulator and result are used, we pass and return
* uint64_t for convenience of the generated code. Unlike the 32-bit
* instruction set versions, val may genuinely have 64 bits of data in it.
* The upper bytes of val (above the number specified by 'bytes') must have
* been zeroed out by the caller.
*/
uint64_t HELPER(crc32_64)(uint64_t acc, uint64_t val, uint32_t bytes)
{
uint8_t buf[8];
stq_le_p(buf, val);
/* zlib crc32 converts the accumulator and output to one's complement. */
return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff;
}
uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes)
{
uint8_t buf[8];
stq_le_p(buf, val);
/* Linux crc32c converts the output to one's complement. */
return crc32c(acc, buf, bytes) ^ 0xffffffff;
}
/*
* AdvSIMD half-precision
*/
#define ADVSIMD_HELPER(name, suffix) HELPER(glue(glue(advsimd_, name), suffix))
#define ADVSIMD_HALFOP(name) \
uint32_t ADVSIMD_HELPER(name, h)(uint32_t a, uint32_t b, void *fpstp) \
{ \
float_status *fpst = fpstp; \
return float16_ ## name(a, b, fpst); \
}
ADVSIMD_HALFOP(add)
ADVSIMD_HALFOP(sub)
ADVSIMD_HALFOP(mul)
ADVSIMD_HALFOP(div)
ADVSIMD_HALFOP(min)
ADVSIMD_HALFOP(max)
ADVSIMD_HALFOP(minnum)
ADVSIMD_HALFOP(maxnum)
#define ADVSIMD_TWOHALFOP(name) \
uint32_t ADVSIMD_HELPER(name, 2h)(uint32_t two_a, uint32_t two_b, void *fpstp) \
{ \
float16 a1, a2, b1, b2; \
uint32_t r1, r2; \
float_status *fpst = fpstp; \
a1 = extract32(two_a, 0, 16); \
a2 = extract32(two_a, 16, 16); \
b1 = extract32(two_b, 0, 16); \
b2 = extract32(two_b, 16, 16); \
r1 = float16_ ## name(a1, b1, fpst); \
r2 = float16_ ## name(a2, b2, fpst); \
return deposit32(r1, 16, 16, r2); \
}
ADVSIMD_TWOHALFOP(add)
ADVSIMD_TWOHALFOP(sub)
ADVSIMD_TWOHALFOP(mul)
ADVSIMD_TWOHALFOP(div)
ADVSIMD_TWOHALFOP(min)
ADVSIMD_TWOHALFOP(max)
ADVSIMD_TWOHALFOP(minnum)
ADVSIMD_TWOHALFOP(maxnum)
/* Data processing - scalar floating-point and advanced SIMD */
static float16 float16_mulx(float16 a, float16 b, void *fpstp)
{
float_status *fpst = fpstp;
a = float16_squash_input_denormal(a, fpst);
b = float16_squash_input_denormal(b, fpst);
if ((float16_is_zero(a) && float16_is_infinity(b)) ||
(float16_is_infinity(a) && float16_is_zero(b))) {
/* 2.0 with the sign bit set to sign(A) XOR sign(B) */
return make_float16((1U << 14) |
((float16_val(a) ^ float16_val(b)) & (1U << 15)));
}
return float16_mul(a, b, fpst);
}
ADVSIMD_HALFOP(mulx)
ADVSIMD_TWOHALFOP(mulx)
/* fused multiply-accumulate */
uint32_t HELPER(advsimd_muladdh)(uint32_t a, uint32_t b, uint32_t c,
void *fpstp)
{
float_status *fpst = fpstp;
return float16_muladd(a, b, c, 0, fpst);
}
uint32_t HELPER(advsimd_muladd2h)(uint32_t two_a, uint32_t two_b,
uint32_t two_c, void *fpstp)
{
float_status *fpst = fpstp;
float16 a1, a2, b1, b2, c1, c2;
uint32_t r1, r2;
a1 = extract32(two_a, 0, 16);
a2 = extract32(two_a, 16, 16);
b1 = extract32(two_b, 0, 16);
b2 = extract32(two_b, 16, 16);
c1 = extract32(two_c, 0, 16);
c2 = extract32(two_c, 16, 16);
r1 = float16_muladd(a1, b1, c1, 0, fpst);
r2 = float16_muladd(a2, b2, c2, 0, fpst);
return deposit32(r1, 16, 16, r2);
}
/*
* Floating point comparisons produce an integer result. Softfloat
* routines return float_relation types which we convert to the 0/-1
* Neon requires.
*/
#define ADVSIMD_CMPRES(test) (test) ? 0xffff : 0
uint32_t HELPER(advsimd_ceq_f16)(uint32_t a, uint32_t b, void *fpstp)
{
float_status *fpst = fpstp;
int compare = float16_compare_quiet(a, b, fpst);
return ADVSIMD_CMPRES(compare == float_relation_equal);
}
uint32_t HELPER(advsimd_cge_f16)(uint32_t a, uint32_t b, void *fpstp)
{
float_status *fpst = fpstp;
int compare = float16_compare(a, b, fpst);
return ADVSIMD_CMPRES(compare == float_relation_greater ||
compare == float_relation_equal);
}
uint32_t HELPER(advsimd_cgt_f16)(uint32_t a, uint32_t b, void *fpstp)
{
float_status *fpst = fpstp;
int compare = float16_compare(a, b, fpst);
return ADVSIMD_CMPRES(compare == float_relation_greater);
}
uint32_t HELPER(advsimd_acge_f16)(uint32_t a, uint32_t b, void *fpstp)
{
float_status *fpst = fpstp;
float16 f0 = float16_abs(a);
float16 f1 = float16_abs(b);
int compare = float16_compare(f0, f1, fpst);
return ADVSIMD_CMPRES(compare == float_relation_greater ||
compare == float_relation_equal);
}
uint32_t HELPER(advsimd_acgt_f16)(uint32_t a, uint32_t b, void *fpstp)
{
float_status *fpst = fpstp;
float16 f0 = float16_abs(a);
float16 f1 = float16_abs(b);
int compare = float16_compare(f0, f1, fpst);
return ADVSIMD_CMPRES(compare == float_relation_greater);
}
/* round to integral */
uint32_t HELPER(advsimd_rinth_exact)(uint32_t x, void *fp_status)
{
return float16_round_to_int(x, fp_status);
}
uint32_t HELPER(advsimd_rinth)(uint32_t x, void *fp_status)
{
int old_flags = get_float_exception_flags(fp_status), new_flags;
float16 ret;
ret = float16_round_to_int(x, fp_status);
/* Suppress any inexact exceptions the conversion produced */
if (!(old_flags & float_flag_inexact)) {
new_flags = get_float_exception_flags(fp_status);
set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
}
return ret;
}
/*
* Half-precision floating point conversion functions
*
* There are a multitude of conversion functions with various
* different rounding modes. This is dealt with by the calling code
* setting the mode appropriately before calling the helper.
*/
uint32_t HELPER(advsimd_f16tosinth)(uint32_t a, void *fpstp)
{
float_status *fpst = fpstp;
/* Invalid if we are passed a NaN */
if (float16_is_any_nan(a)) {
float_raise(float_flag_invalid, fpst);
return 0;
}
return float16_to_int16(a, fpst);
}
uint32_t HELPER(advsimd_f16touinth)(uint32_t a, void *fpstp)
{
float_status *fpst = fpstp;
/* Invalid if we are passed a NaN */
if (float16_is_any_nan(a)) {
float_raise(float_flag_invalid, fpst);
return 0;
}
return float16_to_uint16(a, fpst);
}
static int el_from_spsr(uint32_t spsr)
{
/* Return the exception level that this SPSR is requesting a return to,
* or -1 if it is invalid (an illegal return)
*/
if (spsr & PSTATE_nRW) {
switch (spsr & CPSR_M) {
case ARM_CPU_MODE_USR:
return 0;
case ARM_CPU_MODE_HYP:
return 2;
case ARM_CPU_MODE_FIQ:
case ARM_CPU_MODE_IRQ:
case ARM_CPU_MODE_SVC:
case ARM_CPU_MODE_ABT:
case ARM_CPU_MODE_UND:
case ARM_CPU_MODE_SYS:
return 1;
case ARM_CPU_MODE_MON:
/* Returning to Mon from AArch64 is never possible,
* so this is an illegal return.
*/
default:
return -1;
}
} else {
if (extract32(spsr, 1, 1)) {
/* Return with reserved M[1] bit set */
return -1;
}
if (extract32(spsr, 0, 4) == 1) {
/* return to EL0 with M[0] bit set */
return -1;
}
return extract32(spsr, 2, 2);
}
}
static void cpsr_write_from_spsr_elx(CPUARMState *env,
uint32_t val)
{
uint32_t mask;
/* Save SPSR_ELx.SS into PSTATE. */
env->pstate = (env->pstate & ~PSTATE_SS) | (val & PSTATE_SS);
val &= ~PSTATE_SS;
/* Move DIT to the correct location for CPSR */
if (val & PSTATE_DIT) {
val &= ~PSTATE_DIT;
val |= CPSR_DIT;
}
mask = aarch32_cpsr_valid_mask(env->features, \
&env_archcpu(env)->isar);
cpsr_write(env, val, mask, CPSRWriteRaw);
}
void HELPER(exception_return)(CPUARMState *env, uint64_t new_pc)
{
int cur_el = arm_current_el(env);
unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el);
uint32_t spsr = env->banked_spsr[spsr_idx];
int new_el;
bool return_to_aa64 = (spsr & PSTATE_nRW) == 0;
aarch64_save_sp(env, cur_el);
arm_clear_exclusive(env);
/* We must squash the PSTATE.SS bit to zero unless both of the
* following hold:
* 1. debug exceptions are currently disabled
* 2. singlestep will be active in the EL we return to
* We check 1 here and 2 after we've done the pstate/cpsr write() to
* transition to the EL we're going to.
*/
if (arm_generate_debug_exceptions(env)) {
spsr &= ~PSTATE_SS;
}
/*
* FEAT_RME forbids return from EL3 with an invalid security state.
* We don't need an explicit check for FEAT_RME here because we enforce
* in scr_write() that you can't set the NSE bit without it.
*/
if (cur_el == 3 && (env->cp15.scr_el3 & (SCR_NS | SCR_NSE)) == SCR_NSE) {
goto illegal_return;
}
new_el = el_from_spsr(spsr);
if (new_el == -1) {
goto illegal_return;
}
if (new_el > cur_el || (new_el == 2 && !arm_is_el2_enabled(env))) {
/* Disallow return to an EL which is unimplemented or higher
* than the current one.
*/
goto illegal_return;
}
if (new_el != 0 && arm_el_is_aa64(env, new_el) != return_to_aa64) {
/* Return to an EL which is configured for a different register width */
goto illegal_return;
}
if (new_el == 1 && (arm_hcr_el2_eff(env) & HCR_TGE)) {
goto illegal_return;
}
bql_lock();
arm_call_pre_el_change_hook(env_archcpu(env));
bql_unlock();
if (!return_to_aa64) {
env->aarch64 = false;
/* We do a raw CPSR write because aarch64_sync_64_to_32()
* will sort the register banks out for us, and we've already
* caught all the bad-mode cases in el_from_spsr().
*/
cpsr_write_from_spsr_elx(env, spsr);
if (!arm_singlestep_active(env)) {
env->pstate &= ~PSTATE_SS;
}
aarch64_sync_64_to_32(env);
if (spsr & CPSR_T) {
env->regs[15] = new_pc & ~0x1;
} else {
env->regs[15] = new_pc & ~0x3;
}
helper_rebuild_hflags_a32(env, new_el);
qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to "
"AArch32 EL%d PC 0x%" PRIx32 "\n",
cur_el, new_el, env->regs[15]);
} else {
int tbii;
env->aarch64 = true;
spsr &= aarch64_pstate_valid_mask(&env_archcpu(env)->isar);
pstate_write(env, spsr);
if (!arm_singlestep_active(env)) {
env->pstate &= ~PSTATE_SS;
}
aarch64_restore_sp(env, new_el);
helper_rebuild_hflags_a64(env, new_el);
/*
* Apply TBI to the exception return address. We had to delay this
* until after we selected the new EL, so that we could select the
* correct TBI+TBID bits. This is made easier by waiting until after
* the hflags rebuild, since we can pull the composite TBII field
* from there.
*/
tbii = EX_TBFLAG_A64(env->hflags, TBII);
if ((tbii >> extract64(new_pc, 55, 1)) & 1) {
/* TBI is enabled. */
int core_mmu_idx = arm_env_mmu_index(env);
if (regime_has_2_ranges(core_to_aa64_mmu_idx(core_mmu_idx))) {
new_pc = sextract64(new_pc, 0, 56);
} else {
new_pc = extract64(new_pc, 0, 56);
}
}
env->pc = new_pc;
qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to "
"AArch64 EL%d PC 0x%" PRIx64 "\n",
cur_el, new_el, env->pc);
}
/*
* Note that cur_el can never be 0. If new_el is 0, then
* el0_a64 is return_to_aa64, else el0_a64 is ignored.
*/
aarch64_sve_change_el(env, cur_el, new_el, return_to_aa64);
bql_lock();
arm_call_el_change_hook(env_archcpu(env));
bql_unlock();
return;
illegal_return:
/* Illegal return events of various kinds have architecturally
* mandated behaviour:
* restore NZCV and DAIF from SPSR_ELx
* set PSTATE.IL
* restore PC from ELR_ELx
* no change to exception level, execution state or stack pointer
*/
env->pstate |= PSTATE_IL;
env->pc = new_pc;
spsr &= PSTATE_NZCV | PSTATE_DAIF | PSTATE_ALLINT;
spsr |= pstate_read(env) & ~(PSTATE_NZCV | PSTATE_DAIF | PSTATE_ALLINT);
pstate_write(env, spsr);
if (!arm_singlestep_active(env)) {
env->pstate &= ~PSTATE_SS;
}
helper_rebuild_hflags_a64(env, cur_el);
qemu_log_mask(LOG_GUEST_ERROR, "Illegal exception return at EL%d: "
"resuming execution at 0x%" PRIx64 "\n", cur_el, env->pc);
}
/*
* Square Root and Reciprocal square root
*/
uint32_t HELPER(sqrt_f16)(uint32_t a, void *fpstp)
{
float_status *s = fpstp;
return float16_sqrt(a, s);
}
void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
{
uintptr_t ra = GETPC();
/*
* Implement DC ZVA, which zeroes a fixed-length block of memory.
* Note that we do not implement the (architecturally mandated)
* alignment fault for attempts to use this on Device memory
* (which matches the usual QEMU behaviour of not implementing either
* alignment faults or any memory attribute handling).
*/
int blocklen = 4 << env_archcpu(env)->dcz_blocksize;
uint64_t vaddr = vaddr_in & ~(blocklen - 1);
int mmu_idx = arm_env_mmu_index(env);
void *mem;
/*
* Trapless lookup. In addition to actual invalid page, may
* return NULL for I/O, watchpoints, clean pages, etc.
*/
mem = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx);
#ifndef CONFIG_USER_ONLY
if (unlikely(!mem)) {
/*
* Trap if accessing an invalid page. DC_ZVA requires that we supply
* the original pointer for an invalid page. But watchpoints require
* that we probe the actual space. So do both.
*/
(void) probe_write(env, vaddr_in, 1, mmu_idx, ra);
mem = probe_write(env, vaddr, blocklen, mmu_idx, ra);
if (unlikely(!mem)) {
/*
* The only remaining reason for mem == NULL is I/O.
* Just do a series of byte writes as the architecture demands.
*/
for (int i = 0; i < blocklen; i++) {
cpu_stb_mmuidx_ra(env, vaddr + i, 0, mmu_idx, ra);
}
return;
}
}
#endif
set_helper_retaddr(ra);
memset(mem, 0, blocklen);
clear_helper_retaddr();
}
void HELPER(unaligned_access)(CPUARMState *env, uint64_t addr,
uint32_t access_type, uint32_t mmu_idx)
{
arm_cpu_do_unaligned_access(env_cpu(env), addr, access_type,
mmu_idx, GETPC());
}
/* Memory operations (memset, memmove, memcpy) */
/*
* Return true if the CPY* and SET* insns can execute; compare
* pseudocode CheckMOPSEnabled(), though we refactor it a little.
*/
static bool mops_enabled(CPUARMState *env)
{
int el = arm_current_el(env);
if (el < 2 &&
(arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE) &&
!(arm_hcrx_el2_eff(env) & HCRX_MSCEN)) {
return false;
}
if (el == 0) {
if (!el_is_in_host(env, 0)) {
return env->cp15.sctlr_el[1] & SCTLR_MSCEN;
} else {
return env->cp15.sctlr_el[2] & SCTLR_MSCEN;
}
}
return true;
}
static void check_mops_enabled(CPUARMState *env, uintptr_t ra)
{
if (!mops_enabled(env)) {
raise_exception_ra(env, EXCP_UDEF, syn_uncategorized(),
exception_target_el(env), ra);
}
}
/*
* Return the target exception level for an exception due
* to mismatched arguments in a FEAT_MOPS copy or set.
* Compare pseudocode MismatchedCpySetTargetEL()
*/
static int mops_mismatch_exception_target_el(CPUARMState *env)
{
int el = arm_current_el(env);
if (el > 1) {
return el;
}
if (el == 0 && (arm_hcr_el2_eff(env) & HCR_TGE)) {
return 2;
}
if (el == 1 && (arm_hcrx_el2_eff(env) & HCRX_MCE2)) {
return 2;
}
return 1;
}
/*
* Check whether an M or E instruction was executed with a CF value
* indicating the wrong option for this implementation.
* Assumes we are always Option A.
*/
static void check_mops_wrong_option(CPUARMState *env, uint32_t syndrome,
uintptr_t ra)
{
if (env->CF != 0) {
syndrome |= 1 << 17; /* Set the wrong-option bit */
raise_exception_ra(env, EXCP_UDEF, syndrome,
mops_mismatch_exception_target_el(env), ra);
}
}
/*
* Return the maximum number of bytes we can transfer starting at addr
* without crossing a page boundary.
*/
static uint64_t page_limit(uint64_t addr)
{
return TARGET_PAGE_ALIGN(addr + 1) - addr;
}
/*
* Return the number of bytes we can copy starting from addr and working
* backwards without crossing a page boundary.
*/
static uint64_t page_limit_rev(uint64_t addr)
{
return (addr & ~TARGET_PAGE_MASK) + 1;
}
/*
* Perform part of a memory set on an area of guest memory starting at
* toaddr (a dirty address) and extending for setsize bytes.
*
* Returns the number of bytes actually set, which might be less than
* setsize; the caller should loop until the whole set has been done.
* The caller should ensure that the guest registers are correct
* for the possibility that the first byte of the set encounters
* an exception or watchpoint. We guarantee not to take any faults
* for bytes other than the first.
*/
static uint64_t set_step(CPUARMState *env, uint64_t toaddr,
uint64_t setsize, uint32_t data, int memidx,
uint32_t *mtedesc, uintptr_t ra)
{
void *mem;
setsize = MIN(setsize, page_limit(toaddr));
if (*mtedesc) {
uint64_t mtesize = mte_mops_probe(env, toaddr, setsize, *mtedesc);
if (mtesize == 0) {
/* Trap, or not. All CPU state is up to date */
mte_check_fail(env, *mtedesc, toaddr, ra);
/* Continue, with no further MTE checks required */
*mtedesc = 0;
} else {
/* Advance to the end, or to the tag mismatch */
setsize = MIN(setsize, mtesize);
}
}
toaddr = useronly_clean_ptr(toaddr);
/*
* Trapless lookup: returns NULL for invalid page, I/O,
* watchpoints, clean pages, etc.
*/
mem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, memidx);
#ifndef CONFIG_USER_ONLY
if (unlikely(!mem)) {
/*
* Slow-path: just do one byte write. This will handle the
* watchpoint, invalid page, etc handling correctly.
* For clean code pages, the next iteration will see
* the page dirty and will use the fast path.
*/
cpu_stb_mmuidx_ra(env, toaddr, data, memidx, ra);
return 1;
}
#endif
/* Easy case: just memset the host memory */
set_helper_retaddr(ra);
memset(mem, data, setsize);
clear_helper_retaddr();
return setsize;
}
/*
* Similar, but setting tags. The architecture requires us to do this
* in 16-byte chunks. SETP accesses are not tag checked; they set
* the tags.
*/
static uint64_t set_step_tags(CPUARMState *env, uint64_t toaddr,
uint64_t setsize, uint32_t data, int memidx,
uint32_t *mtedesc, uintptr_t ra)
{
void *mem;
uint64_t cleanaddr;
setsize = MIN(setsize, page_limit(toaddr));
cleanaddr = useronly_clean_ptr(toaddr);
/*
* Trapless lookup: returns NULL for invalid page, I/O,
* watchpoints, clean pages, etc.
*/
mem = tlb_vaddr_to_host(env, cleanaddr, MMU_DATA_STORE, memidx);
#ifndef CONFIG_USER_ONLY
if (unlikely(!mem)) {
/*
* Slow-path: just do one write. This will handle the
* watchpoint, invalid page, etc handling correctly.
* The architecture requires that we do 16 bytes at a time,
* and we know both ptr and size are 16 byte aligned.
* For clean code pages, the next iteration will see
* the page dirty and will use the fast path.
*/
uint64_t repldata = data * 0x0101010101010101ULL;
MemOpIdx oi16 = make_memop_idx(MO_TE | MO_128, memidx);
cpu_st16_mmu(env, toaddr, int128_make128(repldata, repldata), oi16, ra);
mte_mops_set_tags(env, toaddr, 16, *mtedesc);
return 16;
}
#endif
/* Easy case: just memset the host memory */
set_helper_retaddr(ra);
memset(mem, data, setsize);
clear_helper_retaddr();
mte_mops_set_tags(env, toaddr, setsize, *mtedesc);
return setsize;
}
typedef uint64_t StepFn(CPUARMState *env, uint64_t toaddr,
uint64_t setsize, uint32_t data,
int memidx, uint32_t *mtedesc, uintptr_t ra);
/* Extract register numbers from a MOPS exception syndrome value */
static int mops_destreg(uint32_t syndrome)
{
return extract32(syndrome, 10, 5);
}
static int mops_srcreg(uint32_t syndrome)
{
return extract32(syndrome, 5, 5);
}
static int mops_sizereg(uint32_t syndrome)
{
return extract32(syndrome, 0, 5);
}
/*
* Return true if TCMA and TBI bits mean we need to do MTE checks.
* We only need to do this once per MOPS insn, not for every page.
*/
static bool mte_checks_needed(uint64_t ptr, uint32_t desc)
{
int bit55 = extract64(ptr, 55, 1);
/*
* Note that tbi_check() returns true for "access checked" but
* tcma_check() returns true for "access unchecked".
*/
if (!tbi_check(desc, bit55)) {
return false;
}
return !tcma_check(desc, bit55, allocation_tag_from_addr(ptr));
}
/* Take an exception if the SETG addr/size are not granule aligned */
static void check_setg_alignment(CPUARMState *env, uint64_t ptr, uint64_t size,
uint32_t memidx, uintptr_t ra)
{
if ((size != 0 && !QEMU_IS_ALIGNED(ptr, TAG_GRANULE)) ||
!QEMU_IS_ALIGNED(size, TAG_GRANULE)) {
arm_cpu_do_unaligned_access(env_cpu(env), ptr, MMU_DATA_STORE,
memidx, ra);
}
}
static uint64_t arm_reg_or_xzr(CPUARMState *env, int reg)
{
/*
* Runtime equivalent of cpu_reg() -- return the CPU register value,
* for contexts when index 31 means XZR (not SP).
*/
return reg == 31 ? 0 : env->xregs[reg];
}
/*
* For the Memory Set operation, our implementation chooses
* always to use "option A", where we update Xd to the final
* address in the SETP insn, and set Xn to be -(bytes remaining).
* On SETM and SETE insns we only need update Xn.
*
* @env: CPU
* @syndrome: syndrome value for mismatch exceptions
* (also contains the register numbers we need to use)
* @mtedesc: MTE descriptor word
* @stepfn: function which does a single part of the set operation
* @is_setg: true if this is the tag-setting SETG variant
*/
static void do_setp(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc,
StepFn *stepfn, bool is_setg, uintptr_t ra)
{
/* Prologue: we choose to do up to the next page boundary */
int rd = mops_destreg(syndrome);
int rs = mops_srcreg(syndrome);
int rn = mops_sizereg(syndrome);
uint8_t data = arm_reg_or_xzr(env, rs);
uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX);
uint64_t toaddr = env->xregs[rd];
uint64_t setsize = env->xregs[rn];
uint64_t stagesetsize, step;
check_mops_enabled(env, ra);
if (setsize > INT64_MAX) {
setsize = INT64_MAX;
if (is_setg) {
setsize &= ~0xf;
}
}
if (unlikely(is_setg)) {
check_setg_alignment(env, toaddr, setsize, memidx, ra);
} else if (!mte_checks_needed(toaddr, mtedesc)) {
mtedesc = 0;
}
stagesetsize = MIN(setsize, page_limit(toaddr));
while (stagesetsize) {
env->xregs[rd] = toaddr;
env->xregs[rn] = setsize;
step = stepfn(env, toaddr, stagesetsize, data, memidx, &mtedesc, ra);
toaddr += step;
setsize -= step;
stagesetsize -= step;
}
/* Insn completed, so update registers to the Option A format */
env->xregs[rd] = toaddr + setsize;
env->xregs[rn] = -setsize;
/* Set NZCV = 0000 to indicate we are an Option A implementation */
env->NF = 0;
env->ZF = 1; /* our env->ZF encoding is inverted */
env->CF = 0;
env->VF = 0;
return;
}
void HELPER(setp)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc)
{
do_setp(env, syndrome, mtedesc, set_step, false, GETPC());
}
void HELPER(setgp)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc)
{
do_setp(env, syndrome, mtedesc, set_step_tags, true, GETPC());
}
static void do_setm(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc,
StepFn *stepfn, bool is_setg, uintptr_t ra)
{
/* Main: we choose to do all the full-page chunks */
CPUState *cs = env_cpu(env);
int rd = mops_destreg(syndrome);
int rs = mops_srcreg(syndrome);
int rn = mops_sizereg(syndrome);
uint8_t data = arm_reg_or_xzr(env, rs);
uint64_t toaddr = env->xregs[rd] + env->xregs[rn];
uint64_t setsize = -env->xregs[rn];
uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX);
uint64_t step, stagesetsize;
check_mops_enabled(env, ra);
/*
* We're allowed to NOP out "no data to copy" before the consistency
* checks; we choose to do so.
*/
if (env->xregs[rn] == 0) {
return;
}
check_mops_wrong_option(env, syndrome, ra);
/*
* Our implementation will work fine even if we have an unaligned
* destination address, and because we update Xn every time around
* the loop below and the return value from stepfn() may be less
* than requested, we might find toaddr is unaligned. So we don't
* have an IMPDEF check for alignment here.
*/
if (unlikely(is_setg)) {
check_setg_alignment(env, toaddr, setsize, memidx, ra);
} else if (!mte_checks_needed(toaddr, mtedesc)) {
mtedesc = 0;
}
/* Do the actual memset: we leave the last partial page to SETE */
stagesetsize = setsize & TARGET_PAGE_MASK;
while (stagesetsize > 0) {
step = stepfn(env, toaddr, setsize, data, memidx, &mtedesc, ra);
toaddr += step;
setsize -= step;
stagesetsize -= step;
env->xregs[rn] = -setsize;
if (stagesetsize > 0 && unlikely(cpu_loop_exit_requested(cs))) {
cpu_loop_exit_restore(cs, ra);
}
}
}
void HELPER(setm)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc)
{
do_setm(env, syndrome, mtedesc, set_step, false, GETPC());
}
void HELPER(setgm)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc)
{
do_setm(env, syndrome, mtedesc, set_step_tags, true, GETPC());
}
static void do_sete(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc,
StepFn *stepfn, bool is_setg, uintptr_t ra)
{
/* Epilogue: do the last partial page */
int rd = mops_destreg(syndrome);
int rs = mops_srcreg(syndrome);
int rn = mops_sizereg(syndrome);
uint8_t data = arm_reg_or_xzr(env, rs);
uint64_t toaddr = env->xregs[rd] + env->xregs[rn];
uint64_t setsize = -env->xregs[rn];
uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX);
uint64_t step;
check_mops_enabled(env, ra);
/*
* We're allowed to NOP out "no data to copy" before the consistency
* checks; we choose to do so.
*/
if (setsize == 0) {
return;
}
check_mops_wrong_option(env, syndrome, ra);
/*
* Our implementation has no address alignment requirements, but
* we do want to enforce the "less than a page" size requirement,
* so we don't need to have the "check for interrupts" here.
*/
if (setsize >= TARGET_PAGE_SIZE) {
raise_exception_ra(env, EXCP_UDEF, syndrome,
mops_mismatch_exception_target_el(env), ra);
}
if (unlikely(is_setg)) {
check_setg_alignment(env, toaddr, setsize, memidx, ra);
} else if (!mte_checks_needed(toaddr, mtedesc)) {
mtedesc = 0;
}
/* Do the actual memset */
while (setsize > 0) {
step = stepfn(env, toaddr, setsize, data, memidx, &mtedesc, ra);
toaddr += step;
setsize -= step;
env->xregs[rn] = -setsize;
}
}
void HELPER(sete)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc)
{
do_sete(env, syndrome, mtedesc, set_step, false, GETPC());
}
void HELPER(setge)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc)
{
do_sete(env, syndrome, mtedesc, set_step_tags, true, GETPC());
}
/*
* Perform part of a memory copy from the guest memory at fromaddr
* and extending for copysize bytes, to the guest memory at
* toaddr. Both addresses are dirty.
*
* Returns the number of bytes actually set, which might be less than
* copysize; the caller should loop until the whole copy has been done.
* The caller should ensure that the guest registers are correct
* for the possibility that the first byte of the copy encounters
* an exception or watchpoint. We guarantee not to take any faults
* for bytes other than the first.
*/
static uint64_t copy_step(CPUARMState *env, uint64_t toaddr, uint64_t fromaddr,
uint64_t copysize, int wmemidx, int rmemidx,
uint32_t *wdesc, uint32_t *rdesc, uintptr_t ra)
{
void *rmem;
void *wmem;
/* Don't cross a page boundary on either source or destination */
copysize = MIN(copysize, page_limit(toaddr));
copysize = MIN(copysize, page_limit(fromaddr));
/*
* Handle MTE tag checks: either handle the tag mismatch for byte 0,
* or else copy up to but not including the byte with the mismatch.
*/
if (*rdesc) {
uint64_t mtesize = mte_mops_probe(env, fromaddr, copysize, *rdesc);
if (mtesize == 0) {
mte_check_fail(env, *rdesc, fromaddr, ra);
*rdesc = 0;
} else {
copysize = MIN(copysize, mtesize);
}
}
if (*wdesc) {
uint64_t mtesize = mte_mops_probe(env, toaddr, copysize, *wdesc);
if (mtesize == 0) {
mte_check_fail(env, *wdesc, toaddr, ra);
*wdesc = 0;
} else {
copysize = MIN(copysize, mtesize);
}
}
toaddr = useronly_clean_ptr(toaddr);
fromaddr = useronly_clean_ptr(fromaddr);
/* Trapless lookup of whether we can get a host memory pointer */
wmem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, wmemidx);
rmem = tlb_vaddr_to_host(env, fromaddr, MMU_DATA_LOAD, rmemidx);
#ifndef CONFIG_USER_ONLY
/*
* If we don't have host memory for both source and dest then just
* do a single byte copy. This will handle watchpoints, invalid pages,
* etc correctly. For clean code pages, the next iteration will see
* the page dirty and will use the fast path.
*/
if (unlikely(!rmem || !wmem)) {
uint8_t byte;
if (rmem) {
byte = *(uint8_t *)rmem;
} else {
byte = cpu_ldub_mmuidx_ra(env, fromaddr, rmemidx, ra);
}
if (wmem) {
*(uint8_t *)wmem = byte;
} else {
cpu_stb_mmuidx_ra(env, toaddr, byte, wmemidx, ra);
}
return 1;
}
#endif
/* Easy case: just memmove the host memory */
set_helper_retaddr(ra);
memmove(wmem, rmem, copysize);
clear_helper_retaddr();
return copysize;
}
/*
* Do part of a backwards memory copy. Here toaddr and fromaddr point
* to the *last* byte to be copied.
*/
static uint64_t copy_step_rev(CPUARMState *env, uint64_t toaddr,
uint64_t fromaddr,
uint64_t copysize, int wmemidx, int rmemidx,
uint32_t *wdesc, uint32_t *rdesc, uintptr_t ra)
{
void *rmem;
void *wmem;
/* Don't cross a page boundary on either source or destination */
copysize = MIN(copysize, page_limit_rev(toaddr));
copysize = MIN(copysize, page_limit_rev(fromaddr));
/*
* Handle MTE tag checks: either handle the tag mismatch for byte 0,
* or else copy up to but not including the byte with the mismatch.
*/
if (*rdesc) {
uint64_t mtesize = mte_mops_probe_rev(env, fromaddr, copysize, *rdesc);
if (mtesize == 0) {
mte_check_fail(env, *rdesc, fromaddr, ra);
*rdesc = 0;
} else {
copysize = MIN(copysize, mtesize);
}
}
if (*wdesc) {
uint64_t mtesize = mte_mops_probe_rev(env, toaddr, copysize, *wdesc);
if (mtesize == 0) {
mte_check_fail(env, *wdesc, toaddr, ra);
*wdesc = 0;
} else {
copysize = MIN(copysize, mtesize);
}
}
toaddr = useronly_clean_ptr(toaddr);
fromaddr = useronly_clean_ptr(fromaddr);
/* Trapless lookup of whether we can get a host memory pointer */
wmem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, wmemidx);
rmem = tlb_vaddr_to_host(env, fromaddr, MMU_DATA_LOAD, rmemidx);
#ifndef CONFIG_USER_ONLY
/*
* If we don't have host memory for both source and dest then just
* do a single byte copy. This will handle watchpoints, invalid pages,
* etc correctly. For clean code pages, the next iteration will see
* the page dirty and will use the fast path.
*/
if (unlikely(!rmem || !wmem)) {
uint8_t byte;
if (rmem) {
byte = *(uint8_t *)rmem;
} else {
byte = cpu_ldub_mmuidx_ra(env, fromaddr, rmemidx, ra);
}
if (wmem) {
*(uint8_t *)wmem = byte;
} else {
cpu_stb_mmuidx_ra(env, toaddr, byte, wmemidx, ra);
}
return 1;
}
#endif
/*
* Easy case: just memmove the host memory. Note that wmem and
* rmem here point to the *last* byte to copy.
*/
set_helper_retaddr(ra);
memmove(wmem - (copysize - 1), rmem - (copysize - 1), copysize);
clear_helper_retaddr();
return copysize;
}
/*
* for the Memory Copy operation, our implementation chooses always
* to use "option A", where we update Xd and Xs to the final addresses
* in the CPYP insn, and then in CPYM and CPYE only need to update Xn.
*
* @env: CPU
* @syndrome: syndrome value for mismatch exceptions
* (also contains the register numbers we need to use)
* @wdesc: MTE descriptor for the writes (destination)
* @rdesc: MTE descriptor for the reads (source)
* @move: true if this is CPY (memmove), false for CPYF (memcpy forwards)
*/
static void do_cpyp(CPUARMState *env, uint32_t syndrome, uint32_t wdesc,
uint32_t rdesc, uint32_t move, uintptr_t ra)
{
int rd = mops_destreg(syndrome);
int rs = mops_srcreg(syndrome);
int rn = mops_sizereg(syndrome);
uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX);
uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX);
bool forwards = true;
uint64_t toaddr = env->xregs[rd];
uint64_t fromaddr = env->xregs[rs];
uint64_t copysize = env->xregs[rn];
uint64_t stagecopysize, step;
check_mops_enabled(env, ra);
if (move) {
/*
* Copy backwards if necessary. The direction for a non-overlapping
* copy is IMPDEF; we choose forwards.
*/
if (copysize > 0x007FFFFFFFFFFFFFULL) {
copysize = 0x007FFFFFFFFFFFFFULL;
}
uint64_t fs = extract64(fromaddr, 0, 56);
uint64_t ts = extract64(toaddr, 0, 56);
uint64_t fe = extract64(fromaddr + copysize, 0, 56);
if (fs < ts && fe > ts) {
forwards = false;
}
} else {
if (copysize > INT64_MAX) {
copysize = INT64_MAX;
}
}
if (!mte_checks_needed(fromaddr, rdesc)) {
rdesc = 0;
}
if (!mte_checks_needed(toaddr, wdesc)) {
wdesc = 0;
}
if (forwards) {
stagecopysize = MIN(copysize, page_limit(toaddr));
stagecopysize = MIN(stagecopysize, page_limit(fromaddr));
while (stagecopysize) {
env->xregs[rd] = toaddr;
env->xregs[rs] = fromaddr;
env->xregs[rn] = copysize;
step = copy_step(env, toaddr, fromaddr, stagecopysize,
wmemidx, rmemidx, &wdesc, &rdesc, ra);
toaddr += step;
fromaddr += step;
copysize -= step;
stagecopysize -= step;
}
/* Insn completed, so update registers to the Option A format */
env->xregs[rd] = toaddr + copysize;
env->xregs[rs] = fromaddr + copysize;
env->xregs[rn] = -copysize;
} else {
/*
* In a reverse copy the to and from addrs in Xs and Xd are the start
* of the range, but it's more convenient for us to work with pointers
* to the last byte being copied.
*/
toaddr += copysize - 1;
fromaddr += copysize - 1;
stagecopysize = MIN(copysize, page_limit_rev(toaddr));
stagecopysize = MIN(stagecopysize, page_limit_rev(fromaddr));
while (stagecopysize) {
env->xregs[rn] = copysize;
step = copy_step_rev(env, toaddr, fromaddr, stagecopysize,
wmemidx, rmemidx, &wdesc, &rdesc, ra);
copysize -= step;
stagecopysize -= step;
toaddr -= step;
fromaddr -= step;
}
/*
* Insn completed, so update registers to the Option A format.
* For a reverse copy this is no different to the CPYP input format.
*/
env->xregs[rn] = copysize;
}
/* Set NZCV = 0000 to indicate we are an Option A implementation */
env->NF = 0;
env->ZF = 1; /* our env->ZF encoding is inverted */
env->CF = 0;
env->VF = 0;
return;
}
void HELPER(cpyp)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc,
uint32_t rdesc)
{
do_cpyp(env, syndrome, wdesc, rdesc, true, GETPC());
}
void HELPER(cpyfp)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc,
uint32_t rdesc)
{
do_cpyp(env, syndrome, wdesc, rdesc, false, GETPC());
}
static void do_cpym(CPUARMState *env, uint32_t syndrome, uint32_t wdesc,
uint32_t rdesc, uint32_t move, uintptr_t ra)
{
/* Main: we choose to copy until less than a page remaining */
CPUState *cs = env_cpu(env);
int rd = mops_destreg(syndrome);
int rs = mops_srcreg(syndrome);
int rn = mops_sizereg(syndrome);
uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX);
uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX);
bool forwards = true;
uint64_t toaddr, fromaddr, copysize, step;
check_mops_enabled(env, ra);
/* We choose to NOP out "no data to copy" before consistency checks */
if (env->xregs[rn] == 0) {
return;
}
check_mops_wrong_option(env, syndrome, ra);
if (move) {
forwards = (int64_t)env->xregs[rn] < 0;
}
if (forwards) {
toaddr = env->xregs[rd] + env->xregs[rn];
fromaddr = env->xregs[rs] + env->xregs[rn];
copysize = -env->xregs[rn];
} else {
copysize = env->xregs[rn];
/* This toaddr and fromaddr point to the *last* byte to copy */
toaddr = env->xregs[rd] + copysize - 1;
fromaddr = env->xregs[rs] + copysize - 1;
}
if (!mte_checks_needed(fromaddr, rdesc)) {
rdesc = 0;
}
if (!mte_checks_needed(toaddr, wdesc)) {
wdesc = 0;
}
/* Our implementation has no particular parameter requirements for CPYM */
/* Do the actual memmove */
if (forwards) {
while (copysize >= TARGET_PAGE_SIZE) {
step = copy_step(env, toaddr, fromaddr, copysize,
wmemidx, rmemidx, &wdesc, &rdesc, ra);
toaddr += step;
fromaddr += step;
copysize -= step;
env->xregs[rn] = -copysize;
if (copysize >= TARGET_PAGE_SIZE &&
unlikely(cpu_loop_exit_requested(cs))) {
cpu_loop_exit_restore(cs, ra);
}
}
} else {
while (copysize >= TARGET_PAGE_SIZE) {
step = copy_step_rev(env, toaddr, fromaddr, copysize,
wmemidx, rmemidx, &wdesc, &rdesc, ra);
toaddr -= step;
fromaddr -= step;
copysize -= step;
env->xregs[rn] = copysize;
if (copysize >= TARGET_PAGE_SIZE &&
unlikely(cpu_loop_exit_requested(cs))) {
cpu_loop_exit_restore(cs, ra);
}
}
}
}
void HELPER(cpym)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc,
uint32_t rdesc)
{
do_cpym(env, syndrome, wdesc, rdesc, true, GETPC());
}
void HELPER(cpyfm)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc,
uint32_t rdesc)
{
do_cpym(env, syndrome, wdesc, rdesc, false, GETPC());
}
static void do_cpye(CPUARMState *env, uint32_t syndrome, uint32_t wdesc,
uint32_t rdesc, uint32_t move, uintptr_t ra)
{
/* Epilogue: do the last partial page */
int rd = mops_destreg(syndrome);
int rs = mops_srcreg(syndrome);
int rn = mops_sizereg(syndrome);
uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX);
uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX);
bool forwards = true;
uint64_t toaddr, fromaddr, copysize, step;
check_mops_enabled(env, ra);
/* We choose to NOP out "no data to copy" before consistency checks */
if (env->xregs[rn] == 0) {
return;
}
check_mops_wrong_option(env, syndrome, ra);
if (move) {
forwards = (int64_t)env->xregs[rn] < 0;
}
if (forwards) {
toaddr = env->xregs[rd] + env->xregs[rn];
fromaddr = env->xregs[rs] + env->xregs[rn];
copysize = -env->xregs[rn];
} else {
copysize = env->xregs[rn];
/* This toaddr and fromaddr point to the *last* byte to copy */
toaddr = env->xregs[rd] + copysize - 1;
fromaddr = env->xregs[rs] + copysize - 1;
}
if (!mte_checks_needed(fromaddr, rdesc)) {
rdesc = 0;
}
if (!mte_checks_needed(toaddr, wdesc)) {
wdesc = 0;
}
/* Check the size; we don't want to have do a check-for-interrupts */
if (copysize >= TARGET_PAGE_SIZE) {
raise_exception_ra(env, EXCP_UDEF, syndrome,
mops_mismatch_exception_target_el(env), ra);
}
/* Do the actual memmove */
if (forwards) {
while (copysize > 0) {
step = copy_step(env, toaddr, fromaddr, copysize,
wmemidx, rmemidx, &wdesc, &rdesc, ra);
toaddr += step;
fromaddr += step;
copysize -= step;
env->xregs[rn] = -copysize;
}
} else {
while (copysize > 0) {
step = copy_step_rev(env, toaddr, fromaddr, copysize,
wmemidx, rmemidx, &wdesc, &rdesc, ra);
toaddr -= step;
fromaddr -= step;
copysize -= step;
env->xregs[rn] = copysize;
}
}
}
void HELPER(cpye)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc,
uint32_t rdesc)
{
do_cpye(env, syndrome, wdesc, rdesc, true, GETPC());
}
void HELPER(cpyfe)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc,
uint32_t rdesc)
{
do_cpye(env, syndrome, wdesc, rdesc, false, GETPC());
}
static bool is_guarded_page(CPUARMState *env, target_ulong addr, uintptr_t ra)
{
#ifdef CONFIG_USER_ONLY
return page_get_flags(addr) & PAGE_BTI;
#else
CPUTLBEntryFull *full;
void *host;
int mmu_idx = cpu_mmu_index(env_cpu(env), true);
int flags = probe_access_full(env, addr, 0, MMU_INST_FETCH, mmu_idx,
false, &host, &full, ra);
assert(!(flags & TLB_INVALID_MASK));
return full->extra.arm.guarded;
#endif
}
void HELPER(guarded_page_check)(CPUARMState *env)
{
/*
* We have already verified that bti is enabled, and that the
* instruction at PC is not ok for BTYPE. This is always at
* the beginning of a block, so PC is always up-to-date and
* no unwind is required.
*/
if (is_guarded_page(env, env->pc, 0)) {
raise_exception(env, EXCP_UDEF, syn_btitrap(env->btype),
exception_target_el(env));
}
}
void HELPER(guarded_page_br)(CPUARMState *env, target_ulong pc)
{
/*
* We have already checked for branch via x16 and x17.
* What remains for choosing BTYPE is checking for a guarded page.
*/
env->btype = is_guarded_page(env, pc, GETPC()) ? 3 : 1;
}
Reference in New Issue View Git Blame Copy Permalink