mirrors/qemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

target-arm:

Browse Source 
* Implement SME emulation, for both system and linux-user
 -----BEGIN PGP SIGNATURE-----
 
 iQJNBAABCAA3FiEE4aXFk81BneKOgxXPPCUl7RQ2DN4FAmLMLC8ZHHBldGVyLm1h
 eWRlbGxAbGluYXJvLm9yZwAKCRA8JSXtFDYM3sR6D/wN7+rQ86FnPEpeqqh37Chx
 dyCwmAjad7cSJWY4d6RlhNBll35D2nPIdo7MQrUi7ViNL+mDRGr4xSeYOU3IBXd7
 hxRY2rAjwlVtpL6WWJkHeZTr7n+gHtds/Pda+f7d8E7RmsXtD4uRSnbGG82HVGy0
 suG6MJwDYncSoa4AlX/J6sBBYha0cusTguTbkGkEfRonKgvQ6PAogYU8zAmI4EEz
 BsdhXOjH3FGO6aUUVjZKJd1CuISNYe32sZY1OcfuiB1KbBQaRHt2LbxfKSQGMKwd
 0GzXZK9NJ0xIteAX/ESHVakudennqaBY7kdbBdLwb485pQ25r7P2jGyPM4J4fota
 fXScOQKGGVfNySPwTOPR0QLzshcckWw7+Y+AOBg/rpbepSNT41bwI6Ldjs8oNtcI
 O44wNgbN4uvbVul4dj0rOrc5azMvfH0cU+SSrZAInahEHpCN9o/NQz1D2fju7j9W
 MujJCzRVBQSCvUwD1jX6+YDTJU3y84HeGYB/whXt8tg67Ump/5IgUInACgQzwwEZ
 ZP+vxFmhuMlIkbfaiDgYdx2CLjTVSJ7YHjGx/rkqlpLVx6DgGA0klzgHV4L8fbWh
 RFY8fejQve5QzsdiiDdeRWigFY2LnDEnphwOAy7tzktRpai8FnK3aMZyIj/5WODL
 uu+gSmYoToLhC35Uan5otg==
 =D1/T
 -----END PGP SIGNATURE-----

Merge tag 'pull-target-arm-20220711' of https://git.linaro.org/people/pmaydell/qemu-arm into staging

target-arm:
 * Implement SME emulation, for both system and linux-user

# -----BEGIN PGP SIGNATURE-----
#
# iQJNBAABCAA3FiEE4aXFk81BneKOgxXPPCUl7RQ2DN4FAmLMLC8ZHHBldGVyLm1h
# eWRlbGxAbGluYXJvLm9yZwAKCRA8JSXtFDYM3sR6D/wN7+rQ86FnPEpeqqh37Chx
# dyCwmAjad7cSJWY4d6RlhNBll35D2nPIdo7MQrUi7ViNL+mDRGr4xSeYOU3IBXd7
# hxRY2rAjwlVtpL6WWJkHeZTr7n+gHtds/Pda+f7d8E7RmsXtD4uRSnbGG82HVGy0
# suG6MJwDYncSoa4AlX/J6sBBYha0cusTguTbkGkEfRonKgvQ6PAogYU8zAmI4EEz
# BsdhXOjH3FGO6aUUVjZKJd1CuISNYe32sZY1OcfuiB1KbBQaRHt2LbxfKSQGMKwd
# 0GzXZK9NJ0xIteAX/ESHVakudennqaBY7kdbBdLwb485pQ25r7P2jGyPM4J4fota
# fXScOQKGGVfNySPwTOPR0QLzshcckWw7+Y+AOBg/rpbepSNT41bwI6Ldjs8oNtcI
# O44wNgbN4uvbVul4dj0rOrc5azMvfH0cU+SSrZAInahEHpCN9o/NQz1D2fju7j9W
# MujJCzRVBQSCvUwD1jX6+YDTJU3y84HeGYB/whXt8tg67Ump/5IgUInACgQzwwEZ
# ZP+vxFmhuMlIkbfaiDgYdx2CLjTVSJ7YHjGx/rkqlpLVx6DgGA0klzgHV4L8fbWh
# RFY8fejQve5QzsdiiDdeRWigFY2LnDEnphwOAy7tzktRpai8FnK3aMZyIj/5WODL
# uu+gSmYoToLhC35Uan5otg==
# =D1/T
# -----END PGP SIGNATURE-----
# gpg: Signature made Mon 11 Jul 2022 07:27:03 PM +0530
# gpg:                using RSA key E1A5C593CD419DE28E8315CF3C2525ED14360CDE
# gpg:                issuer "peter.maydell@linaro.org"
# gpg: Good signature from "Peter Maydell <peter.maydell@linaro.org>" [full]
# gpg:                 aka "Peter Maydell <pmaydell@gmail.com>" [full]
# gpg:                 aka "Peter Maydell <pmaydell@chiark.greenend.org.uk>" [full]

* tag 'pull-target-arm-20220711' of https://git.linaro.org/people/pmaydell/qemu-arm: (45 commits)
  linux-user/aarch64: Add SME related hwcap entries
  target/arm: Enable SME for user-only
  target/arm: Only set ZEN in reset if SVE present
  linux-user/aarch64: Implement PR_SME_GET_VL, PR_SME_SET_VL
  linux-user: Rename sve prctls
  linux-user/aarch64: Implement SME signal handling
  linux-user/aarch64: Move sve record checks into restore
  linux-user/aarch64: Verify extra record lock succeeded
  linux-user/aarch64: Do not allow duplicate or short sve records
  linux-user/aarch64: Tidy target_restore_sigframe error return
  linux-user/aarch64: Add SM bit to SVE signal context
  linux-user/aarch64: Reset PSTATE.SM on syscalls
  linux-user/aarch64: Clear tpidr2_el0 if CLONE_SETTLS
  target/arm: Enable SME for -cpu max
  target/arm: Reset streaming sve state on exception boundaries
  target/arm: Implement SCLAMP, UCLAMP
  target/arm: Implement REVD
  target/arm: Implement PSEL
  target/arm: Implement SME integer outer product
  target/arm: Implement FMOPA, FMOPS (widening)
  ...

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
This commit is contained in:
Richard Henderson 2022-07-11 22:02:09 +05:30
commit 9fed1bca6b
28 changed files with 2821 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/system/arm/emulation.rst
View File

@ -65,6 +65,10 @@ the following architecture extensions:
- FEAT_SHA512 (Advanced SIMD SHA512 instructions)
- FEAT_SM3 (Advanced SIMD SM3 instructions)
- FEAT_SM4 (Advanced SIMD SM4 instructions)
- FEAT_SME (Scalable Matrix Extension)
- FEAT_SME_FA64 (Full A64 instruction set in Streaming SVE mode)
- FEAT_SME_F64F64 (Double-precision floating-point outer product instructions)
- FEAT_SME_I16I64 (16-bit to 64-bit integer widening outer product instructions)
- FEAT_SPECRES (Speculation restriction instructions)
- FEAT_SSBS (Speculative Store Bypass Safe)
- FEAT_TLBIOS (TLB invalidate instructions in Outer Shareable domain)

9
linux-user/aarch64/cpu_loop.c
View File

@ -89,6 +89,15 @@ void cpu_loop(CPUARMState *env)
switch (trapnr) {
case EXCP_SWI:
/*
* On syscall, PSTATE.ZA is preserved, along with the ZA matrix.
* PSTATE.SM is cleared, per SMSTOP, which does ResetSVEState.
*/
if (FIELD_EX64(env->svcr, SVCR, SM)) {
env->svcr = FIELD_DP64(env->svcr, SVCR, SM, 0);
arm_rebuild_hflags(env);
arm_reset_sve_state(env);
}
ret = do_syscall(env,
env->xregs[8],
env->xregs[0],

243
linux-user/aarch64/signal.c
View File

@ -78,7 +78,8 @@ struct target_extra_context {
struct target_sve_context {
struct target_aarch64_ctx head;
uint16_t vl;
uint16_t reserved[3];
uint16_t flags;
uint16_t reserved[2];
/* The actual SVE data immediately follows. It is laid out
* according to TARGET_SVE_SIG_{Z,P}REG_OFFSET, based off of
* the original struct pointer.
@ -101,6 +102,24 @@ struct target_sve_context {
#define TARGET_SVE_SIG_CONTEXT_SIZE(VQ) \
(TARGET_SVE_SIG_PREG_OFFSET(VQ, 17))
#define TARGET_SVE_SIG_FLAG_SM 1
#define TARGET_ZA_MAGIC 0x54366345
struct target_za_context {
struct target_aarch64_ctx head;
uint16_t vl;
uint16_t reserved[3];
/* The actual ZA data immediately follows. */
};
#define TARGET_ZA_SIG_REGS_OFFSET \
QEMU_ALIGN_UP(sizeof(struct target_za_context), TARGET_SVE_VQ_BYTES)
#define TARGET_ZA_SIG_ZAV_OFFSET(VQ, N) \
(TARGET_ZA_SIG_REGS_OFFSET + (VQ) * TARGET_SVE_VQ_BYTES * (N))
#define TARGET_ZA_SIG_CONTEXT_SIZE(VQ) \
TARGET_ZA_SIG_ZAV_OFFSET(VQ, VQ * TARGET_SVE_VQ_BYTES)
struct target_rt_sigframe {
struct target_siginfo info;
struct target_ucontext uc;
@ -173,13 +192,17 @@ static void target_setup_end_record(struct target_aarch64_ctx *end)
}
static void target_setup_sve_record(struct target_sve_context *sve,
CPUARMState *env, int vq, int size)
CPUARMState *env, int size)
{
int i, j;
int i, j, vq = sve_vq(env);
memset(sve, 0, sizeof(*sve));
__put_user(TARGET_SVE_MAGIC, &sve->head.magic);
__put_user(size, &sve->head.size);
__put_user(vq * TARGET_SVE_VQ_BYTES, &sve->vl);
if (FIELD_EX64(env->svcr, SVCR, SM)) {
__put_user(TARGET_SVE_SIG_FLAG_SM, &sve->flags);
}
/* Note that SVE regs are stored as a byte stream, with each byte element
* at a subsequent address. This corresponds to a little-endian store
@ -200,6 +223,35 @@ static void target_setup_sve_record(struct target_sve_context *sve,
}
}
static void target_setup_za_record(struct target_za_context *za,
CPUARMState *env, int size)
{
int vq = sme_vq(env);
int vl = vq * TARGET_SVE_VQ_BYTES;
int i, j;
memset(za, 0, sizeof(*za));
__put_user(TARGET_ZA_MAGIC, &za->head.magic);
__put_user(size, &za->head.size);
__put_user(vl, &za->vl);
if (size == TARGET_ZA_SIG_CONTEXT_SIZE(0)) {
return;
}
assert(size == TARGET_ZA_SIG_CONTEXT_SIZE(vq));
/*
* Note that ZA vectors are stored as a byte stream,
* with each byte element at a subsequent address.
*/
for (i = 0; i < vl; ++i) {
uint64_t *z = (void *)za + TARGET_ZA_SIG_ZAV_OFFSET(vq, i);
for (j = 0; j < vq * 2; ++j) {
__put_user_e(env->zarray[i].d[j], z + j, le);
}
}
}
static void target_restore_general_frame(CPUARMState *env,
struct target_rt_sigframe *sf)
{
@ -243,12 +295,50 @@ static void target_restore_fpsimd_record(CPUARMState *env,
}
}
static void target_restore_sve_record(CPUARMState *env,
struct target_sve_context *sve, int vq)
static bool target_restore_sve_record(CPUARMState *env,
struct target_sve_context *sve,
int size, int *svcr)
{
int i, j;
int i, j, vl, vq, flags;
bool sm;
/* Note that SVE regs are stored as a byte stream, with each byte element
__get_user(vl, &sve->vl);
__get_user(flags, &sve->flags);
sm = flags & TARGET_SVE_SIG_FLAG_SM;
/* The cpu must support Streaming or Non-streaming SVE. */
if (sm
? !cpu_isar_feature(aa64_sme, env_archcpu(env))
: !cpu_isar_feature(aa64_sve, env_archcpu(env))) {
return false;
}
/*
* Note that we cannot use sve_vq() because that depends on the
* current setting of PSTATE.SM, not the state to be restored.
*/
vq = sve_vqm1_for_el_sm(env, 0, sm) + 1;
/* Reject mismatched VL. */
if (vl != vq * TARGET_SVE_VQ_BYTES) {
return false;
}
/* Accept empty record -- used to clear PSTATE.SM. */
if (size <= sizeof(*sve)) {
return true;
}
/* Reject non-empty but incomplete record. */
if (size < TARGET_SVE_SIG_CONTEXT_SIZE(vq)) {
return false;
}
*svcr = FIELD_DP64(*svcr, SVCR, SM, sm);
/*
* Note that SVE regs are stored as a byte stream, with each byte element
* at a subsequent address. This corresponds to a little-endian load
* of our 64-bit hunks.
*/
@ -270,6 +360,46 @@ static void target_restore_sve_record(CPUARMState *env,
}
}
}
return true;
}
static bool target_restore_za_record(CPUARMState *env,
struct target_za_context *za,
int size, int *svcr)
{
int i, j, vl, vq;
if (!cpu_isar_feature(aa64_sme, env_archcpu(env))) {
return false;
}
__get_user(vl, &za->vl);
vq = sme_vq(env);
/* Reject mismatched VL. */
if (vl != vq * TARGET_SVE_VQ_BYTES) {
return false;
}
/* Accept empty record -- used to clear PSTATE.ZA. */
if (size <= TARGET_ZA_SIG_CONTEXT_SIZE(0)) {
return true;
}
/* Reject non-empty but incomplete record. */
if (size < TARGET_ZA_SIG_CONTEXT_SIZE(vq)) {
return false;
}
*svcr = FIELD_DP64(*svcr, SVCR, ZA, 1);
for (i = 0; i < vl; ++i) {
uint64_t *z = (void *)za + TARGET_ZA_SIG_ZAV_OFFSET(vq, i);
for (j = 0; j < vq * 2; ++j) {
__get_user_e(env->zarray[i].d[j], z + j, le);
}
}
return true;
}
static int target_restore_sigframe(CPUARMState *env,
@ -278,10 +408,12 @@ static int target_restore_sigframe(CPUARMState *env,
struct target_aarch64_ctx *ctx, *extra = NULL;
struct target_fpsimd_context *fpsimd = NULL;
struct target_sve_context *sve = NULL;
struct target_za_context *za = NULL;
uint64_t extra_datap = 0;
bool used_extra = false;
bool err = false;
int vq = 0, sve_size = 0;
int sve_size = 0;
int za_size = 0;
int svcr = 0;
target_restore_general_frame(env, sf);
@ -294,8 +426,7 @@ static int target_restore_sigframe(CPUARMState *env,
switch (magic) {
case 0:
if (size != 0) {
err = true;
goto exit;
goto err;
}
if (used_extra) {
ctx = NULL;
@ -307,42 +438,46 @@ static int target_restore_sigframe(CPUARMState *env,
case TARGET_FPSIMD_MAGIC:
if (fpsimd || size != sizeof(struct target_fpsimd_context)) {
err = true;
goto exit;
goto err;
}
fpsimd = (struct target_fpsimd_context *)ctx;
break;
case TARGET_SVE_MAGIC:
if (cpu_isar_feature(aa64_sve, env_archcpu(env))) {
vq = sve_vq(env);
sve_size = QEMU_ALIGN_UP(TARGET_SVE_SIG_CONTEXT_SIZE(vq), 16);
if (!sve && size == sve_size) {
sve = (struct target_sve_context *)ctx;
break;
}
if (sve || size < sizeof(struct target_sve_context)) {
goto err;
}
err = true;
goto exit;
sve = (struct target_sve_context *)ctx;
sve_size = size;
break;
case TARGET_ZA_MAGIC:
if (za || size < sizeof(struct target_za_context)) {
goto err;
}
za = (struct target_za_context *)ctx;
za_size = size;
break;
case TARGET_EXTRA_MAGIC:
if (extra || size != sizeof(struct target_extra_context)) {
err = true;
goto exit;
goto err;
}
__get_user(extra_datap,
&((struct target_extra_context *)ctx)->datap);
__get_user(extra_size,
&((struct target_extra_context *)ctx)->size);
extra = lock_user(VERIFY_READ, extra_datap, extra_size, 0);
if (!extra) {
return 1;
}
break;
default:
/* Unknown record -- we certainly didn't generate it.
* Did we in fact get out of sync?
*/
err = true;
goto exit;
goto err;
}
ctx = (void *)ctx + size;
}
@ -351,17 +486,26 @@ static int target_restore_sigframe(CPUARMState *env,
if (fpsimd) {
target_restore_fpsimd_record(env, fpsimd);
} else {
err = true;
goto err;
}
/* SVE data, if present, overwrites FPSIMD data. */
if (sve) {
target_restore_sve_record(env, sve, vq);
if (sve && !target_restore_sve_record(env, sve, sve_size, &svcr)) {
goto err;
}
if (za && !target_restore_za_record(env, za, za_size, &svcr)) {
goto err;
}
if (env->svcr != svcr) {
env->svcr = svcr;
arm_rebuild_hflags(env);
}
exit:
unlock_user(extra, extra_datap, 0);
return err;
return 0;
err:
unlock_user(extra, extra_datap, 0);
return 1;
}
static abi_ulong get_sigframe(struct target_sigaction *ka,
@ -423,7 +567,8 @@ static void target_setup_frame(int usig, struct target_sigaction *ka,
.total_size = offsetof(struct target_rt_sigframe,
uc.tuc_mcontext.__reserved),
};
int fpsimd_ofs, fr_ofs, sve_ofs = 0, vq = 0, sve_size = 0;
int fpsimd_ofs, fr_ofs, sve_ofs = 0, za_ofs = 0;
int sve_size = 0, za_size = 0;
struct target_rt_sigframe *frame;
struct target_rt_frame_record *fr;
abi_ulong frame_addr, return_addr;
@ -433,11 +578,20 @@ static void target_setup_frame(int usig, struct target_sigaction *ka,
&layout);
/* SVE state needs saving only if it exists. */
if (cpu_isar_feature(aa64_sve, env_archcpu(env))) {
vq = sve_vq(env);
sve_size = QEMU_ALIGN_UP(TARGET_SVE_SIG_CONTEXT_SIZE(vq), 16);
if (cpu_isar_feature(aa64_sve, env_archcpu(env)) ||
cpu_isar_feature(aa64_sme, env_archcpu(env))) {
sve_size = QEMU_ALIGN_UP(TARGET_SVE_SIG_CONTEXT_SIZE(sve_vq(env)), 16);
sve_ofs = alloc_sigframe_space(sve_size, &layout);
}
if (cpu_isar_feature(aa64_sme, env_archcpu(env))) {
/* ZA state needs saving only if it is enabled. */
if (FIELD_EX64(env->svcr, SVCR, ZA)) {
za_size = TARGET_ZA_SIG_CONTEXT_SIZE(sme_vq(env));
} else {
za_size = TARGET_ZA_SIG_CONTEXT_SIZE(0);
}
za_ofs = alloc_sigframe_space(za_size, &layout);
}
if (layout.extra_ofs) {
/* Reserve space for the extra end marker. The standard end marker
@ -484,7 +638,10 @@ static void target_setup_frame(int usig, struct target_sigaction *ka,
target_setup_end_record((void *)frame + layout.extra_end_ofs);
}
if (sve_ofs) {
target_setup_sve_record((void *)frame + sve_ofs, env, vq, sve_size);
target_setup_sve_record((void *)frame + sve_ofs, env, sve_size);
}
if (za_ofs) {
target_setup_za_record((void *)frame + za_ofs, env, za_size);
}
/* Set up the stack frame for unwinding. */
@ -508,6 +665,18 @@ static void target_setup_frame(int usig, struct target_sigaction *ka,
env->btype = 2;
}
/*
* Invoke the signal handler with both SM and ZA disabled.
* When clearing SM, ResetSVEState, per SMSTOP.
*/
if (FIELD_EX64(env->svcr, SVCR, SM)) {
arm_reset_sve_state(env);
}
if (env->svcr) {
env->svcr = 0;
arm_rebuild_hflags(env);
}
if (info) {
tswap_siginfo(&frame->info, info);
env->xregs[1] = frame_addr + offsetof(struct target_rt_sigframe, info);

5
linux-user/aarch64/target_cpu.h
View File

@ -34,10 +34,13 @@ static inline void cpu_clone_regs_parent(CPUARMState *env, unsigned flags)
static inline void cpu_set_tls(CPUARMState *env, target_ulong newtls)
{
/* Note that AArch64 Linux keeps the TLS pointer in TPIDR; this is
/*
* Note that AArch64 Linux keeps the TLS pointer in TPIDR; this is
* different from AArch32 Linux, which uses TPIDRRO.
*/
env->cp15.tpidr_el[0] = newtls;
/* TPIDR2_EL0 is cleared with CLONE_SETTLS. */
env->cp15.tpidr2_el0 = 0;
}
static inline abi_ulong get_sp_from_cpustate(CPUARMState *state)

62
linux-user/aarch64/target_prctl.h
View File

@ -6,17 +6,18 @@
#ifndef AARCH64_TARGET_PRCTL_H
#define AARCH64_TARGET_PRCTL_H
static abi_long do_prctl_get_vl(CPUArchState *env)
static abi_long do_prctl_sve_get_vl(CPUArchState *env)
{
ARMCPU *cpu = env_archcpu(env);
if (cpu_isar_feature(aa64_sve, cpu)) {
/* PSTATE.SM is always unset on syscall entry. */
return sve_vq(env) * 16;
}
return -TARGET_EINVAL;
}
#define do_prctl_get_vl do_prctl_get_vl
#define do_prctl_sve_get_vl do_prctl_sve_get_vl
static abi_long do_prctl_set_vl(CPUArchState *env, abi_long arg2)
static abi_long do_prctl_sve_set_vl(CPUArchState *env, abi_long arg2)
{
/*
* We cannot support either PR_SVE_SET_VL_ONEXEC or PR_SVE_VL_INHERIT.
@ -27,6 +28,7 @@ static abi_long do_prctl_set_vl(CPUArchState *env, abi_long arg2)
&& arg2 >= 0 && arg2 <= 512 * 16 && !(arg2 & 15)) {
uint32_t vq, old_vq;
/* PSTATE.SM is always unset on syscall entry. */
old_vq = sve_vq(env);
/*
@ -47,7 +49,59 @@ static abi_long do_prctl_set_vl(CPUArchState *env, abi_long arg2)
}
return -TARGET_EINVAL;
}
#define do_prctl_set_vl do_prctl_set_vl
#define do_prctl_sve_set_vl do_prctl_sve_set_vl
static abi_long do_prctl_sme_get_vl(CPUArchState *env)
{
ARMCPU *cpu = env_archcpu(env);
if (cpu_isar_feature(aa64_sme, cpu)) {
return sme_vq(env) * 16;
}
return -TARGET_EINVAL;
}
#define do_prctl_sme_get_vl do_prctl_sme_get_vl
static abi_long do_prctl_sme_set_vl(CPUArchState *env, abi_long arg2)
{
/*
* We cannot support either PR_SME_SET_VL_ONEXEC or PR_SME_VL_INHERIT.
* Note the kernel definition of sve_vl_valid allows for VQ=512,
* i.e. VL=8192, even though the architectural maximum is VQ=16.
*/
if (cpu_isar_feature(aa64_sme, env_archcpu(env))
&& arg2 >= 0 && arg2 <= 512 * 16 && !(arg2 & 15)) {
int vq, old_vq;
old_vq = sme_vq(env);
/*
* Bound the value of vq, so that we know that it fits into
* the 4-bit field in SMCR_EL1. Because PSTATE.SM is cleared
* on syscall entry, we are not modifying the current SVE
* vector length.
*/
vq = MAX(arg2 / 16, 1);
vq = MIN(vq, 16);
env->vfp.smcr_el[1] =
FIELD_DP64(env->vfp.smcr_el[1], SMCR, LEN, vq - 1);
/* Delay rebuilding hflags until we know if ZA must change. */
vq = sve_vqm1_for_el_sm(env, 0, true) + 1;
if (vq != old_vq) {
/*
* PSTATE.ZA state is cleared on any change to SVL.
* We need not call arm_rebuild_hflags because PSTATE.SM was
* cleared on syscall entry, so this hasn't changed VL.
*/
env->svcr = FIELD_DP64(env->svcr, SVCR, ZA, 0);
arm_rebuild_hflags(env);
}
return vq * 16;
}
return -TARGET_EINVAL;
}
#define do_prctl_sme_set_vl do_prctl_sme_set_vl
static abi_long do_prctl_reset_keys(CPUArchState *env, abi_long arg2)
{

20
linux-user/elfload.c
View File

@ -605,6 +605,18 @@ enum {
ARM_HWCAP2_A64_RNG = 1 << 16,
ARM_HWCAP2_A64_BTI = 1 << 17,
ARM_HWCAP2_A64_MTE = 1 << 18,
ARM_HWCAP2_A64_ECV = 1 << 19,
ARM_HWCAP2_A64_AFP = 1 << 20,
ARM_HWCAP2_A64_RPRES = 1 << 21,
ARM_HWCAP2_A64_MTE3 = 1 << 22,
ARM_HWCAP2_A64_SME = 1 << 23,
ARM_HWCAP2_A64_SME_I16I64 = 1 << 24,
ARM_HWCAP2_A64_SME_F64F64 = 1 << 25,
ARM_HWCAP2_A64_SME_I8I32 = 1 << 26,
ARM_HWCAP2_A64_SME_F16F32 = 1 << 27,
ARM_HWCAP2_A64_SME_B16F32 = 1 << 28,
ARM_HWCAP2_A64_SME_F32F32 = 1 << 29,
ARM_HWCAP2_A64_SME_FA64 = 1 << 30,
};
#define ELF_HWCAP get_elf_hwcap()
@ -674,6 +686,14 @@ static uint32_t get_elf_hwcap2(void)
GET_FEATURE_ID(aa64_rndr, ARM_HWCAP2_A64_RNG);
GET_FEATURE_ID(aa64_bti, ARM_HWCAP2_A64_BTI);
GET_FEATURE_ID(aa64_mte, ARM_HWCAP2_A64_MTE);
GET_FEATURE_ID(aa64_sme, (ARM_HWCAP2_A64_SME |
ARM_HWCAP2_A64_SME_F32F32 |
ARM_HWCAP2_A64_SME_B16F32 |
ARM_HWCAP2_A64_SME_F16F32 |
ARM_HWCAP2_A64_SME_I8I32));
GET_FEATURE_ID(aa64_sme_f64f64, ARM_HWCAP2_A64_SME_F64F64);
GET_FEATURE_ID(aa64_sme_i16i64, ARM_HWCAP2_A64_SME_I16I64);
GET_FEATURE_ID(aa64_sme_fa64, ARM_HWCAP2_A64_SME_FA64);
return hwcaps;
}

28
linux-user/syscall.c
View File

@ -6343,6 +6343,12 @@ abi_long do_arch_prctl(CPUX86State *env, int code, abi_ulong addr)
#ifndef PR_SET_SYSCALL_USER_DISPATCH
# define PR_SET_SYSCALL_USER_DISPATCH 59
#endif
#ifndef PR_SME_SET_VL
# define PR_SME_SET_VL 63
# define PR_SME_GET_VL 64
# define PR_SME_VL_LEN_MASK 0xffff
# define PR_SME_VL_INHERIT (1 << 17)
#endif
#include "target_prctl.h"
@ -6362,11 +6368,11 @@ static abi_long do_prctl_inval1(CPUArchState *env, abi_long arg2)
#ifndef do_prctl_set_fp_mode
#define do_prctl_set_fp_mode do_prctl_inval1
#endif
#ifndef do_prctl_get_vl
#define do_prctl_get_vl do_prctl_inval0
#ifndef do_prctl_sve_get_vl
#define do_prctl_sve_get_vl do_prctl_inval0
#endif
#ifndef do_prctl_set_vl
#define do_prctl_set_vl do_prctl_inval1
#ifndef do_prctl_sve_set_vl
#define do_prctl_sve_set_vl do_prctl_inval1
#endif
#ifndef do_prctl_reset_keys
#define do_prctl_reset_keys do_prctl_inval1
@ -6383,6 +6389,12 @@ static abi_long do_prctl_inval1(CPUArchState *env, abi_long arg2)
#ifndef do_prctl_set_unalign
#define do_prctl_set_unalign do_prctl_inval1
#endif
#ifndef do_prctl_sme_get_vl
#define do_prctl_sme_get_vl do_prctl_inval0
#endif
#ifndef do_prctl_sme_set_vl
#define do_prctl_sme_set_vl do_prctl_inval1
#endif
static abi_long do_prctl(CPUArchState *env, abi_long option, abi_long arg2,
abi_long arg3, abi_long arg4, abi_long arg5)
@ -6431,9 +6443,13 @@ static abi_long do_prctl(CPUArchState *env, abi_long option, abi_long arg2,
case PR_SET_FP_MODE:
return do_prctl_set_fp_mode(env, arg2);
case PR_SVE_GET_VL:
return do_prctl_get_vl(env);
return do_prctl_sve_get_vl(env);
case PR_SVE_SET_VL:
return do_prctl_set_vl(env, arg2);
return do_prctl_sve_set_vl(env, arg2);
case PR_SME_GET_VL:
return do_prctl_sme_get_vl(env);
case PR_SME_SET_VL:
return do_prctl_sme_set_vl(env, arg2);
case PR_PAC_RESET_KEYS:
if (arg3 || arg4 || arg5) {
return -TARGET_EINVAL;

35
target/arm/cpu.c
View File

@ -204,13 +204,23 @@ static void arm_cpu_reset(DeviceState *dev)
/* and to the FP/Neon instructions */
env->cp15.cpacr_el1 = FIELD_DP64(env->cp15.cpacr_el1,
CPACR_EL1, FPEN, 3);
/* and to the SVE instructions */
env->cp15.cpacr_el1 = FIELD_DP64(env->cp15.cpacr_el1,
CPACR_EL1, ZEN, 3);
/* with reasonable vector length */
/* and to the SVE instructions, with default vector length */
if (cpu_isar_feature(aa64_sve, cpu)) {
env->cp15.cpacr_el1 = FIELD_DP64(env->cp15.cpacr_el1,
CPACR_EL1, ZEN, 3);
env->vfp.zcr_el[1] = cpu->sve_default_vq - 1;
}
/* and for SME instructions, with default vector length, and TPIDR2 */
if (cpu_isar_feature(aa64_sme, cpu)) {
env->cp15.sctlr_el[1] |= SCTLR_EnTP2;
env->cp15.cpacr_el1 = FIELD_DP64(env->cp15.cpacr_el1,
CPACR_EL1, SMEN, 3);
env->vfp.smcr_el[1] = cpu->sme_default_vq - 1;
if (cpu_isar_feature(aa64_sme_fa64, cpu)) {
env->vfp.smcr_el[1] = FIELD_DP64(env->vfp.smcr_el[1],
SMCR, FA64, 1);
}
}
/*
* Enable 48-bit address space (TODO: take reserved_va into account).
* Enable TBI0 but not TBI1.
@ -878,6 +888,7 @@ static void aarch64_cpu_dump_state(CPUState *cs, FILE *f, int flags)
int i;
int el = arm_current_el(env);
const char *ns_status;
bool sve;
qemu_fprintf(f, " PC=%016" PRIx64 " ", env->pc);
for (i = 0; i < 32; i++) {
@ -904,6 +915,12 @@ static void aarch64_cpu_dump_state(CPUState *cs, FILE *f, int flags)
el,
psr & PSTATE_SP ? 'h' : 't');
if (cpu_isar_feature(aa64_sme, cpu)) {
qemu_fprintf(f, " SVCR=%08" PRIx64 " %c%c",
env->svcr,
(FIELD_EX64(env->svcr, SVCR, ZA) ? 'Z' : '-'),
(FIELD_EX64(env->svcr, SVCR, SM) ? 'S' : '-'));
}
if (cpu_isar_feature(aa64_bti, cpu)) {
qemu_fprintf(f, " BTYPE=%d", (psr & PSTATE_BTYPE) >> 10);
}
@ -918,7 +935,15 @@ static void aarch64_cpu_dump_state(CPUState *cs, FILE *f, int flags)
qemu_fprintf(f, " FPCR=%08x FPSR=%08x\n",
vfp_get_fpcr(env), vfp_get_fpsr(env));
if (cpu_isar_feature(aa64_sve, cpu) && sve_exception_el(env, el) == 0) {
if (cpu_isar_feature(aa64_sme, cpu) && FIELD_EX64(env->svcr, SVCR, SM)) {
sve = sme_exception_el(env, el) == 0;
} else if (cpu_isar_feature(aa64_sve, cpu)) {
sve = sve_exception_el(env, el) == 0;
} else {
sve = false;
}
if (sve) {
int j, zcr_len = sve_vqm1_for_el(env, el);
for (i = 0; i <= FFR_PRED_NUM; i++) {

7
target/arm/cpu.h
View File

@ -3158,6 +3158,11 @@ FIELD(TBFLAG_A32, HSTR_ACTIVE, 9, 1)
* the same thing as the current security state of the processor!
*/
FIELD(TBFLAG_A32, NS, 10, 1)
/*
* Indicates that SME Streaming mode is active, and SMCR_ELx.FA64 is not.
* This requires an SME trap from AArch32 mode when using NEON.
*/
FIELD(TBFLAG_A32, SME_TRAP_NONSTREAMING, 11, 1)
/*
* Bit usage when in AArch32 state, for M-profile only.
@ -3195,6 +3200,8 @@ FIELD(TBFLAG_A64, SMEEXC_EL, 20, 2)
FIELD(TBFLAG_A64, PSTATE_SM, 22, 1)
FIELD(TBFLAG_A64, PSTATE_ZA, 23, 1)
FIELD(TBFLAG_A64, SVL, 24, 4)
/* Indicates that SME Streaming mode is active, and SMCR_ELx.FA64 is not. */
FIELD(TBFLAG_A64, SME_TRAP_NONSTREAMING, 28, 1)
/*
* Helpers for using the above.

11
target/arm/cpu64.c
View File

@ -1024,6 +1024,7 @@ static void aarch64_max_initfn(Object *obj)
*/
t = FIELD_DP64(t, ID_AA64PFR1, MTE, 3); /* FEAT_MTE3 */
t = FIELD_DP64(t, ID_AA64PFR1, RAS_FRAC, 0); /* FEAT_RASv1p1 + FEAT_DoubleFault */
t = FIELD_DP64(t, ID_AA64PFR1, SME, 1); /* FEAT_SME */
t = FIELD_DP64(t, ID_AA64PFR1, CSV2_FRAC, 0); /* FEAT_CSV2_2 */
cpu->isar.id_aa64pfr1 = t;
@ -1074,6 +1075,16 @@ static void aarch64_max_initfn(Object *obj)
t = FIELD_DP64(t, ID_AA64DFR0, PMUVER, 5); /* FEAT_PMUv3p4 */
cpu->isar.id_aa64dfr0 = t;
t = cpu->isar.id_aa64smfr0;
t = FIELD_DP64(t, ID_AA64SMFR0, F32F32, 1); /* FEAT_SME */
t = FIELD_DP64(t, ID_AA64SMFR0, B16F32, 1); /* FEAT_SME */
t = FIELD_DP64(t, ID_AA64SMFR0, F16F32, 1); /* FEAT_SME */
t = FIELD_DP64(t, ID_AA64SMFR0, I8I32, 0xf); /* FEAT_SME */
t = FIELD_DP64(t, ID_AA64SMFR0, F64F64, 1); /* FEAT_SME_F64F64 */
t = FIELD_DP64(t, ID_AA64SMFR0, I16I64, 0xf); /* FEAT_SME_I16I64 */
t = FIELD_DP64(t, ID_AA64SMFR0, FA64, 1); /* FEAT_SME_FA64 */
cpu->isar.id_aa64smfr0 = t;
/* Replicate the same data to the 32-bit id registers. */
aa32_max_features(cpu);

126
target/arm/helper-sme.h
View File

@ -19,3 +19,129 @@
DEF_HELPER_FLAGS_2(set_pstate_sm, TCG_CALL_NO_RWG, void, env, i32)
DEF_HELPER_FLAGS_2(set_pstate_za, TCG_CALL_NO_RWG, void, env, i32)
DEF_HELPER_FLAGS_3(sme_zero, TCG_CALL_NO_RWG, void, env, i32, i32)
/* Move to/from vertical array slices, i.e. columns, so 'c'. */
DEF_HELPER_FLAGS_4(sme_mova_cz_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_zc_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_cz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_zc_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_cz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_zc_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_cz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_zc_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_cz_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_zc_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sme_ld1b_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1b_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1b_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1b_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1h_be_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1h_le_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1h_be_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1h_le_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1h_be_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1h_le_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1h_be_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1h_le_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1s_be_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1s_le_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1s_be_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1s_le_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1s_be_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1s_le_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1s_be_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1s_le_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1d_be_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1d_le_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1d_be_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1d_le_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1d_be_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1d_le_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1d_be_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1d_le_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1q_be_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1q_le_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1q_be_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1q_le_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1q_be_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1q_le_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1q_be_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_ld1q_le_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1b_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1b_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1b_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1b_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1h_be_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1h_le_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1h_be_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1h_le_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1h_be_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1h_le_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1h_be_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1h_le_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1s_be_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1s_le_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1s_be_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1s_le_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1s_be_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1s_le_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1s_be_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1s_le_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1d_be_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1d_le_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1d_be_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1d_le_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1d_be_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1d_le_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1d_be_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1d_le_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1q_be_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1q_le_h, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1q_be_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1q_le_v, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1q_be_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1q_le_h_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1q_be_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_st1q_le_v_mte, TCG_CALL_NO_WG, void, env, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_5(sme_addha_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sme_addva_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sme_addha_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sme_addva_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_7(sme_fmopa_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_7(sme_fmopa_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_7(sme_fmopa_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sme_bfmopa, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sme_smopa_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sme_umopa_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sme_sumopa_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sme_usmopa_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sme_smopa_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sme_umopa_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sme_sumopa_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sme_usmopa_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)

4
target/arm/helper-sve.h
View File

@ -325,6 +325,8 @@ DEF_HELPER_FLAGS_5(sve_sel_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_sel_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_sel_zpzz_q, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_addp_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
@ -717,6 +719,8 @@ DEF_HELPER_FLAGS_4(sve_revh_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_revw_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_revd_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_rbit_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_rbit_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sve_rbit_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)

56
target/arm/helper.c
View File

@ -6098,6 +6098,32 @@ int sme_exception_el(CPUARMState *env, int el)
return 0;
}
/* This corresponds to the ARM pseudocode function IsFullA64Enabled(). */
static bool sme_fa64(CPUARMState *env, int el)
{
if (!cpu_isar_feature(aa64_sme_fa64, env_archcpu(env))) {
return false;
}
if (el <= 1 && !el_is_in_host(env, el)) {
if (!FIELD_EX64(env->vfp.smcr_el[1], SMCR, FA64)) {
return false;
}
}
if (el <= 2 && arm_is_el2_enabled(env)) {
if (!FIELD_EX64(env->vfp.smcr_el[2], SMCR, FA64)) {
return false;
}
}
if (arm_feature(env, ARM_FEATURE_EL3)) {
if (!FIELD_EX64(env->vfp.smcr_el[3], SMCR, FA64)) {
return false;
}
}
return true;
}
/*
* Given that SVE is enabled, return the vector length for EL.
*/
@ -10801,6 +10827,20 @@ static CPUARMTBFlags rebuild_hflags_a32(CPUARMState *env, int fp_el,
DP_TBFLAG_ANY(flags, PSTATE__IL, 1);
}
/*
* The SME exception we are testing for is raised via
* AArch64.CheckFPAdvSIMDEnabled(), as called from
* AArch32.CheckAdvSIMDOrFPEnabled().
*/
if (el == 0
&& FIELD_EX64(env->svcr, SVCR, SM)
&& (!arm_is_el2_enabled(env)
|| (arm_el_is_aa64(env, 2) && !(env->cp15.hcr_el2 & HCR_TGE)))
&& arm_el_is_aa64(env, 1)
&& !sme_fa64(env, el)) {
DP_TBFLAG_A32(flags, SME_TRAP_NONSTREAMING, 1);
}
return rebuild_hflags_common_32(env, fp_el, mmu_idx, flags);
}
@ -10850,6 +10890,7 @@ static CPUARMTBFlags rebuild_hflags_a64(CPUARMState *env, int el, int fp_el,
}
if (FIELD_EX64(env->svcr, SVCR, SM)) {
DP_TBFLAG_A64(flags, PSTATE_SM, 1);
DP_TBFLAG_A64(flags, SME_TRAP_NONSTREAMING, !sme_fa64(env, el));
}
DP_TBFLAG_A64(flags, PSTATE_ZA, FIELD_EX64(env->svcr, SVCR, ZA));
}
@ -11201,6 +11242,19 @@ void aarch64_sve_change_el(CPUARMState *env, int old_el,
return;
}
old_a64 = old_el ? arm_el_is_aa64(env, old_el) : el0_a64;
new_a64 = new_el ? arm_el_is_aa64(env, new_el) : el0_a64;
/*
* Both AArch64.TakeException and AArch64.ExceptionReturn
* invoke ResetSVEState when taking an exception from, or
* returning to, AArch32 state when PSTATE.SM is enabled.
*/
if (old_a64 != new_a64 && FIELD_EX64(env->svcr, SVCR, SM)) {
arm_reset_sve_state(env);
return;
}
/*
* DDI0584A.d sec 3.2: "If SVE instructions are disabled or trapped
* at ELx, or not available because the EL is in AArch32 state, then
@ -11213,10 +11267,8 @@ void aarch64_sve_change_el(CPUARMState *env, int old_el,
* we already have the correct register contents when encountering the
* vq0->vq0 transition between EL0->EL1.
*/
old_a64 = old_el ? arm_el_is_aa64(env, old_el) : el0_a64;
old_len = (old_a64 && !sve_exception_el(env, old_el)
? sve_vqm1_for_el(env, old_el) : 0);
new_a64 = new_el ? arm_el_is_aa64(env, new_el) : el0_a64;
new_len = (new_a64 && !sve_exception_el(env, new_el)
? sve_vqm1_for_el(env, new_el) : 0);

18
target/arm/helper.h
View File

@ -1019,6 +1019,24 @@ DEF_HELPER_FLAGS_6(gvec_bfmlal, TCG_CALL_NO_RWG,
DEF_HELPER_FLAGS_6(gvec_bfmlal_idx, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_sclamp_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_sclamp_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_sclamp_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_sclamp_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_uclamp_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_uclamp_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_uclamp_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(gvec_uclamp_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
#ifdef TARGET_AARCH64
#include "helper-a64.h"
#include "helper-sve.h"

3
target/arm/meson.build
View File

@ -1,5 +1,7 @@
gen = [
decodetree.process('sve.decode', extra_args: '--decode=disas_sve'),
decodetree.process('sme.decode', extra_args: '--decode=disas_sme'),
decodetree.process('sme-fa64.decode', extra_args: '--static-decode=disas_sme_fa64'),
decodetree.process('neon-shared.decode', extra_args: '--decode=disas_neon_shared'),
decodetree.process('neon-dp.decode', extra_args: '--decode=disas_neon_dp'),
decodetree.process('neon-ls.decode', extra_args: '--decode=disas_neon_ls'),
@ -50,6 +52,7 @@ arm_ss.add(when: 'TARGET_AARCH64', if_true: files(
'sme_helper.c',
'translate-a64.c',
'translate-sve.c',
'translate-sme.c',
))
arm_softmmu_ss = ss.source_set()

60
target/arm/sme-fa64.decode Normal file
View File

@ -0,0 +1,60 @@
# AArch64 SME allowed instruction decoding
#
# Copyright (c) 2022 Linaro, Ltd
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, see <http://www.gnu.org/licenses/>.
#
# This file is processed by scripts/decodetree.py
#
# These patterns are taken from Appendix E1.1 of DDI0616 A.a,
# Arm Architecture Reference Manual Supplement,
# The Scalable Matrix Extension (SME), for Armv9-A
{
[
OK 0-00 1110 0000 0001 0010 11-- ---- ---- # SMOV W|Xd,Vn.B[0]
OK 0-00 1110 0000 0010 0010 11-- ---- ---- # SMOV W|Xd,Vn.H[0]
OK 0100 1110 0000 0100 0010 11-- ---- ---- # SMOV Xd,Vn.S[0]
OK 0000 1110 0000 0001 0011 11-- ---- ---- # UMOV Wd,Vn.B[0]
OK 0000 1110 0000 0010 0011 11-- ---- ---- # UMOV Wd,Vn.H[0]
OK 0000 1110 0000 0100 0011 11-- ---- ---- # UMOV Wd,Vn.S[0]
OK 0100 1110 0000 1000 0011 11-- ---- ---- # UMOV Xd,Vn.D[0]
]
FAIL 0--0 111- ---- ---- ---- ---- ---- ---- # Advanced SIMD vector operations
}
{
[
OK 0101 1110 --1- ---- 11-1 11-- ---- ---- # FMULX/FRECPS/FRSQRTS (scalar)
OK 0101 1110 -10- ---- 00-1 11-- ---- ---- # FMULX/FRECPS/FRSQRTS (scalar, FP16)
OK 01-1 1110 1-10 0001 11-1 10-- ---- ---- # FRECPE/FRSQRTE/FRECPX (scalar)
OK 01-1 1110 1111 1001 11-1 10-- ---- ---- # FRECPE/FRSQRTE/FRECPX (scalar, FP16)
]
FAIL 01-1 111- ---- ---- ---- ---- ---- ---- # Advanced SIMD single-element operations
}
FAIL 0-00 110- ---- ---- ---- ---- ---- ---- # Advanced SIMD structure load/store
FAIL 1100 1110 ---- ---- ---- ---- ---- ---- # Advanced SIMD cryptography extensions
FAIL 0001 1110 0111 1110 0000 00-- ---- ---- # FJCVTZS
# These are the "avoidance of doubt" final table of Illegal Advanced SIMD instructions
# We don't actually need to include these, as the default is OK.
# -001 111- ---- ---- ---- ---- ---- ---- # Scalar floating-point operations
# --10 110- ---- ---- ---- ---- ---- ---- # Load/store pair of FP registers
# --01 1100 ---- ---- ---- ---- ---- ---- # Load FP register (PC-relative literal)
# --11 1100 --0- ---- ---- ---- ---- ---- # Load/store FP register (unscaled imm)
# --11 1100 --1- ---- ---- ---- ---- --10 # Load/store FP register (register offset)
# --11 1101 ---- ---- ---- ---- ---- ---- # Load/store FP register (scaled imm)

88
target/arm/sme.decode Normal file
View File

@ -0,0 +1,88 @@
# AArch64 SME instruction descriptions
#
# Copyright (c) 2022 Linaro, Ltd
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, see <http://www.gnu.org/licenses/>.
#
# This file is processed by scripts/decodetree.py
#
### SME Misc
ZERO 11000000 00 001 00000000000 imm:8
### SME Move into/from Array
%mova_rs 13:2 !function=plus_12
&mova esz rs pg zr za_imm v:bool to_vec:bool
MOVA 11000000 esz:2 00000 0 v:1 .. pg:3 zr:5 0 za_imm:4 \
&mova to_vec=0 rs=%mova_rs
MOVA 11000000 11 00000 1 v:1 .. pg:3 zr:5 0 za_imm:4 \
&mova to_vec=0 rs=%mova_rs esz=4
MOVA 11000000 esz:2 00001 0 v:1 .. pg:3 0 za_imm:4 zr:5 \
&mova to_vec=1 rs=%mova_rs
MOVA 11000000 11 00001 1 v:1 .. pg:3 0 za_imm:4 zr:5 \
&mova to_vec=1 rs=%mova_rs esz=4
### SME Memory
&ldst esz rs pg rn rm za_imm v:bool st:bool
LDST1 1110000 0 esz:2 st:1 rm:5 v:1 .. pg:3 rn:5 0 za_imm:4 \
&ldst rs=%mova_rs
LDST1 1110000 111 st:1 rm:5 v:1 .. pg:3 rn:5 0 za_imm:4 \
&ldst esz=4 rs=%mova_rs
&ldstr rv rn imm
@ldstr ....... ... . ...... .. ... rn:5 . imm:4 \
&ldstr rv=%mova_rs
LDR 1110000 100 0 000000 .. 000 ..... 0 .... @ldstr
STR 1110000 100 1 000000 .. 000 ..... 0 .... @ldstr
### SME Add Vector to Array
&adda zad zn pm pn
@adda_32 ........ .. ..... . pm:3 pn:3 zn:5 ... zad:2 &adda
@adda_64 ........ .. ..... . pm:3 pn:3 zn:5 .. zad:3 &adda
ADDHA_s 11000000 10 01000 0 ... ... ..... 000 .. @adda_32
ADDVA_s 11000000 10 01000 1 ... ... ..... 000 .. @adda_32
ADDHA_d 11000000 11 01000 0 ... ... ..... 00 ... @adda_64
ADDVA_d 11000000 11 01000 1 ... ... ..... 00 ... @adda_64
### SME Outer Product
&op zad zn zm pm pn sub:bool
@op_32 ........ ... zm:5 pm:3 pn:3 zn:5 sub:1 .. zad:2 &op
@op_64 ........ ... zm:5 pm:3 pn:3 zn:5 sub:1 . zad:3 &op
FMOPA_s 10000000 100 ..... ... ... ..... . 00 .. @op_32
FMOPA_d 10000000 110 ..... ... ... ..... . 0 ... @op_64
BFMOPA 10000001 100 ..... ... ... ..... . 00 .. @op_32
FMOPA_h 10000001 101 ..... ... ... ..... . 00 .. @op_32
SMOPA_s 1010000 0 10 0 ..... ... ... ..... . 00 .. @op_32
SUMOPA_s 1010000 0 10 1 ..... ... ... ..... . 00 .. @op_32
USMOPA_s 1010000 1 10 0 ..... ... ... ..... . 00 .. @op_32
UMOPA_s 1010000 1 10 1 ..... ... ... ..... . 00 .. @op_32
SMOPA_d 1010000 0 11 0 ..... ... ... ..... . 0 ... @op_64
SUMOPA_d 1010000 0 11 1 ..... ... ... ..... . 0 ... @op_64
USMOPA_d 1010000 1 11 0 ..... ... ... ..... . 0 ... @op_64
UMOPA_d 1010000 1 11 1 ..... ... ... ..... . 0 ... @op_64

1140
target/arm/sme_helper.c
View File

File diff suppressed because it is too large Load Diff

41
target/arm/sve.decode
View File

@ -449,14 +449,17 @@ INDEX_ri 00000100 esz:2 1 imm:s5 010001 rn:5 rd:5
# SVE index generation (register start, register increment)
INDEX_rr 00000100 .. 1 ..... 010011 ..... ..... @rd_rn_rm
### SVE Stack Allocation Group
### SVE / Streaming SVE Stack Allocation Group
# SVE stack frame adjustment
ADDVL 00000100 001 ..... 01010 ...... ..... @rd_rn_i6
ADDSVL 00000100 001 ..... 01011 ...... ..... @rd_rn_i6
ADDPL 00000100 011 ..... 01010 ...... ..... @rd_rn_i6
ADDSPL 00000100 011 ..... 01011 ...... ..... @rd_rn_i6
# SVE stack frame size
RDVL 00000100 101 11111 01010 imm:s6 rd:5
RDSVL 00000100 101 11111 01011 imm:s6 rd:5
### SVE Bitwise Shift - Unpredicated Group
@ -649,6 +652,7 @@ REVB 00000101 .. 1001 00 100 ... ..... ..... @rd_pg_rn
REVH 00000101 .. 1001 01 100 ... ..... ..... @rd_pg_rn
REVW 00000101 .. 1001 10 100 ... ..... ..... @rd_pg_rn
RBIT 00000101 .. 1001 11 100 ... ..... ..... @rd_pg_rn
REVD 00000101 00 1011 10 100 ... ..... ..... @rd_pg_rn_e0
# SVE vector splice (predicated, destructive)
SPLICE 00000101 .. 101 100 100 ... ..... ..... @rdn_pg_rm
@ -1183,10 +1187,10 @@ LD1RO_zpri 1010010 .. 01 0.... 001 ... ..... ..... \
@rpri_load_msz nreg=0
# SVE 32-bit gather prefetch (scalar plus 32-bit scaled offsets)
PRF 1000010 00 -1 ----- 0-- --- ----- 0 ----
PRF_ns 1000010 00 -1 ----- 0-- --- ----- 0 ----
# SVE 32-bit gather prefetch (vector plus immediate)
PRF 1000010 -- 00 ----- 111 --- ----- 0 ----
PRF_ns 1000010 -- 00 ----- 111 --- ----- 0 ----
# SVE contiguous prefetch (scalar plus immediate)
PRF 1000010 11 1- ----- 0-- --- ----- 0 ----
@ -1223,13 +1227,13 @@ LD1_zpiz 1100010 .. 01 ..... 1.. ... ..... ..... \
@rpri_g_load esz=3
# SVE 64-bit gather prefetch (scalar plus 64-bit scaled offsets)
PRF 1100010 00 11 ----- 1-- --- ----- 0 ----
PRF_ns 1100010 00 11 ----- 1-- --- ----- 0 ----
# SVE 64-bit gather prefetch (scalar plus unpacked 32-bit scaled offsets)
PRF 1100010 00 -1 ----- 0-- --- ----- 0 ----
PRF_ns 1100010 00 -1 ----- 0-- --- ----- 0 ----
# SVE 64-bit gather prefetch (vector plus immediate)
PRF 1100010 -- 00 ----- 111 --- ----- 0 ----
PRF_ns 1100010 -- 00 ----- 111 --- ----- 0 ----
### SVE Memory Store Group
@ -1671,3 +1675,28 @@ BFMLALT_zzxw 01100100 11 1 ..... 0100.1 ..... ..... @rrxr_3a esz=2
### SVE2 floating-point bfloat16 dot-product (indexed)
BFDOT_zzxz 01100100 01 1 ..... 010000 ..... ..... @rrxr_2 esz=2
### SVE broadcast predicate element
&psel esz pd pn pm rv imm
%psel_rv 16:2 !function=plus_12
%psel_imm_b 22:2 19:2
%psel_imm_h 22:2 20:1
%psel_imm_s 22:2
%psel_imm_d 23:1
@psel ........ .. . ... .. .. pn:4 . pm:4 . pd:4 \
&psel rv=%psel_rv
PSEL 00100101 .. 1 ..1 .. 01 .... 0 .... 0 .... \
@psel esz=0 imm=%psel_imm_b
PSEL 00100101 .. 1 .10 .. 01 .... 0 .... 0 .... \
@psel esz=1 imm=%psel_imm_h
PSEL 00100101 .. 1 100 .. 01 .... 0 .... 0 .... \
@psel esz=2 imm=%psel_imm_s
PSEL 00100101 .1 1 000 .. 01 .... 0 .... 0 .... \
@psel esz=3 imm=%psel_imm_d
### SVE clamp
SCLAMP 01000100 .. 0 ..... 110000 ..... ..... @rda_rn_rm
UCLAMP 01000100 .. 0 ..... 110001 ..... ..... @rda_rn_rm

28
target/arm/sve_helper.c
View File

@ -931,6 +931,22 @@ DO_ZPZ_D(sve_revh_d, uint64_t, hswap64)
DO_ZPZ_D(sve_revw_d, uint64_t, wswap64)
void HELPER(sme_revd_q)(void *vd, void *vn, void *vg, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc) / 8;
uint64_t *d = vd, *n = vn;
uint8_t *pg = vg;
for (i = 0; i < opr_sz; i += 2) {
if (pg[H1(i)] & 1) {
uint64_t n0 = n[i + 0];
uint64_t n1 = n[i + 1];
d[i + 0] = n1;
d[i + 1] = n0;
}
}
}
DO_ZPZ(sve_rbit_b, uint8_t, H1, revbit8)
DO_ZPZ(sve_rbit_h, uint16_t, H1_2, revbit16)
DO_ZPZ(sve_rbit_s, uint32_t, H1_4, revbit32)
@ -3565,6 +3581,18 @@ void HELPER(sve_sel_zpzz_d)(void *vd, void *vn, void *vm,
}
}
void HELPER(sve_sel_zpzz_q)(void *vd, void *vn, void *vm,
void *vg, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc) / 16;
Int128 *d = vd, *n = vn, *m = vm;
uint16_t *pg = vg;
for (i = 0; i < opr_sz; i += 1) {
d[i] = (pg[H2(i)] & 1 ? n : m)[i];
}
}
/* Two operand comparison controlled by a predicate.
* ??? It is very tempting to want to be able to expand this inline
* with x86 instructions, e.g.

103
target/arm/translate-a64.c
View File

@ -1155,7 +1155,7 @@ static void do_vec_ld(DisasContext *s, int destidx, int element,
* unallocated-encoding checks (otherwise the syndrome information
* for the resulting exception will be incorrect).
*/
static bool fp_access_check(DisasContext *s)
static bool fp_access_check_only(DisasContext *s)
{
if (s->fp_excp_el) {
assert(!s->fp_access_checked);
@ -1170,21 +1170,44 @@ static bool fp_access_check(DisasContext *s)
return true;
}
/* Check that SVE access is enabled. If it is, return true.
static bool fp_access_check(DisasContext *s)
{
if (!fp_access_check_only(s)) {
return false;
}
if (s->sme_trap_nonstreaming && s->is_nonstreaming) {
gen_exception_insn(s, s->pc_curr, EXCP_UDEF,
syn_smetrap(SME_ET_Streaming, false));
return false;
}
return true;
}
/*
* Check that SVE access is enabled. If it is, return true.
* If not, emit code to generate an appropriate exception and return false.
* This function corresponds to CheckSVEEnabled().
*/
bool sve_access_check(DisasContext *s)
{
if (s->sve_excp_el) {
assert(!s->sve_access_checked);
s->sve_access_checked = true;
if (s->pstate_sm || !dc_isar_feature(aa64_sve, s)) {
assert(dc_isar_feature(aa64_sme, s));
if (!sme_sm_enabled_check(s)) {
goto fail_exit;
}
} else if (s->sve_excp_el) {
gen_exception_insn_el(s, s->pc_curr, EXCP_UDEF,
syn_sve_access_trap(), s->sve_excp_el);
return false;
goto fail_exit;
}
s->sve_access_checked = true;
return fp_access_check(s);
fail_exit:
/* Assert that we only raise one exception per instruction. */
assert(!s->sve_access_checked);
s->sve_access_checked = true;
return false;
}
/*
@ -1203,6 +1226,40 @@ static bool sme_access_check(DisasContext *s)
return true;
}
/* This function corresponds to CheckSMEEnabled. */
bool sme_enabled_check(DisasContext *s)
{
/*
* Note that unlike sve_excp_el, we have not constrained sme_excp_el
* to be zero when fp_excp_el has priority. This is because we need
* sme_excp_el by itself for cpregs access checks.
*/
if (!s->fp_excp_el || s->sme_excp_el < s->fp_excp_el) {
s->fp_access_checked = true;
return sme_access_check(s);
}
return fp_access_check_only(s);
}
/* Common subroutine for CheckSMEAnd*Enabled. */
bool sme_enabled_check_with_svcr(DisasContext *s, unsigned req)
{
if (!sme_enabled_check(s)) {
return false;
}
if (FIELD_EX64(req, SVCR, SM) && !s->pstate_sm) {
gen_exception_insn(s, s->pc_curr, EXCP_UDEF,
syn_smetrap(SME_ET_NotStreaming, false));
return false;
}
if (FIELD_EX64(req, SVCR, ZA) && !s->pstate_za) {
gen_exception_insn(s, s->pc_curr, EXCP_UDEF,
syn_smetrap(SME_ET_InactiveZA, false));
return false;
}
return true;
}
/*
* This utility function is for doing register extension with an
* optional shift. You will likely want to pass a temporary for the
@ -1994,7 +2051,7 @@ static void handle_sys(DisasContext *s, uint32_t insn, bool isread,
default:
g_assert_not_reached();
}
if ((ri->type & ARM_CP_FPU) && !fp_access_check(s)) {
if ((ri->type & ARM_CP_FPU) && !fp_access_check_only(s)) {
return;
} else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) {
return;
@ -14530,6 +14587,23 @@ static void disas_data_proc_simd_fp(DisasContext *s, uint32_t insn)
}
}
/*
* Include the generated SME FA64 decoder.
*/
#include "decode-sme-fa64.c.inc"
static bool trans_OK(DisasContext *s, arg_OK *a)
{
return true;
}
static bool trans_FAIL(DisasContext *s, arg_OK *a)
{
s->is_nonstreaming = true;
return true;
}
/**
* is_guarded_page:
* @env: The cpu environment
@ -14657,6 +14731,7 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase,
dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE);
dc->pstate_sm = EX_TBFLAG_A64(tb_flags, PSTATE_SM);
dc->pstate_za = EX_TBFLAG_A64(tb_flags, PSTATE_ZA);
dc->sme_trap_nonstreaming = EX_TBFLAG_A64(tb_flags, SME_TRAP_NONSTREAMING);
dc->vec_len = 0;
dc->vec_stride = 0;
dc->cp_regs = arm_cpu->cp_regs;
@ -14805,8 +14880,18 @@ static void aarch64_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu)
}
}
s->is_nonstreaming = false;
if (s->sme_trap_nonstreaming) {
disas_sme_fa64(s, insn);
}
switch (extract32(insn, 25, 4)) {
case 0x0: case 0x1: case 0x3: /* UNALLOCATED */
case 0x0:
if (!extract32(insn, 31, 1) || !disas_sme(s, insn)) {
unallocated_encoding(s);
}
break;
case 0x1: case 0x3: /* UNALLOCATED */
unallocated_encoding(s);
break;
case 0x2:

45
target/arm/translate-a64.h
View File

@ -29,6 +29,27 @@ void write_fp_dreg(DisasContext *s, int reg, TCGv_i64 v);
bool logic_imm_decode_wmask(uint64_t *result, unsigned int immn,
unsigned int imms, unsigned int immr);
bool sve_access_check(DisasContext *s);
bool sme_enabled_check(DisasContext *s);
bool sme_enabled_check_with_svcr(DisasContext *s, unsigned);
/* This function corresponds to CheckStreamingSVEEnabled. */
static inline bool sme_sm_enabled_check(DisasContext *s)
{
return sme_enabled_check_with_svcr(s, R_SVCR_SM_MASK);
}
/* This function corresponds to CheckSMEAndZAEnabled. */
static inline bool sme_za_enabled_check(DisasContext *s)
{
return sme_enabled_check_with_svcr(s, R_SVCR_ZA_MASK);
}
/* Note that this function corresponds to CheckStreamingSVEAndZAEnabled. */
static inline bool sme_smza_enabled_check(DisasContext *s)
{
return sme_enabled_check_with_svcr(s, R_SVCR_SM_MASK | R_SVCR_ZA_MASK);
}
TCGv_i64 clean_data_tbi(DisasContext *s, TCGv_i64 addr);
TCGv_i64 gen_mte_check1(DisasContext *s, TCGv_i64 addr, bool is_write,
bool tag_checked, int log2_size);
@ -107,6 +128,12 @@ static inline int vec_full_reg_size(DisasContext *s)
return s->vl;
}
/* Return the byte size of the vector register, SVL / 8. */
static inline int streaming_vec_reg_size(DisasContext *s)
{
return s->svl;
}
/*
* Return the offset info CPUARMState of the predicate vector register Pn.
* Note for this purpose, FFR is P16.
@ -122,6 +149,12 @@ static inline int pred_full_reg_size(DisasContext *s)
return s->vl >> 3;
}
/* Return the byte size of the predicate register, SVL / 64. */
static inline int streaming_pred_reg_size(DisasContext *s)
{
return s->svl >> 3;
}
/*
* Round up the size of a register to a size allowed by
* the tcg vector infrastructure. Any operation which uses this
@ -145,7 +178,16 @@ static inline int pred_gvec_reg_size(DisasContext *s)
return size_for_gvec(pred_full_reg_size(s));
}
/* Return a newly allocated pointer to the predicate register. */
static inline TCGv_ptr pred_full_reg_ptr(DisasContext *s, int regno)
{
TCGv_ptr ret = tcg_temp_new_ptr();
tcg_gen_addi_ptr(ret, cpu_env, pred_full_reg_offset(s, regno));
return ret;
}
bool disas_sve(DisasContext *, uint32_t);
bool disas_sme(DisasContext *, uint32_t);
void gen_gvec_rax1(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz);
@ -153,4 +195,7 @@ void gen_gvec_xar(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
uint32_t rm_ofs, int64_t shift,
uint32_t opr_sz, uint32_t max_sz);
void gen_sve_ldr(DisasContext *s, TCGv_ptr, int vofs, int len, int rn, int imm);
void gen_sve_str(DisasContext *s, TCGv_ptr, int vofs, int len, int rn, int imm);
#endif /* TARGET_ARM_TRANSLATE_A64_H */

373
target/arm/translate-sme.c Normal file
View File

@ -0,0 +1,373 @@
/*
* AArch64 SME translation
*
* Copyright (c) 2022 Linaro, Ltd
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
#include "qemu/osdep.h"
#include "cpu.h"
#include "tcg/tcg-op.h"
#include "tcg/tcg-op-gvec.h"
#include "tcg/tcg-gvec-desc.h"
#include "translate.h"
#include "exec/helper-gen.h"
#include "translate-a64.h"
#include "fpu/softfloat.h"
/*
* Include the generated decoder.
*/
#include "decode-sme.c.inc"
/*
* Resolve tile.size[index] to a host pointer, where tile and index
* are always decoded together, dependent on the element size.
*/
static TCGv_ptr get_tile_rowcol(DisasContext *s, int esz, int rs,
int tile_index, bool vertical)
{
int tile = tile_index >> (4 - esz);
int index = esz == MO_128 ? 0 : extract32(tile_index, 0, 4 - esz);
int pos, len, offset;
TCGv_i32 tmp;
TCGv_ptr addr;
/* Compute the final index, which is Rs+imm. */
tmp = tcg_temp_new_i32();
tcg_gen_trunc_tl_i32(tmp, cpu_reg(s, rs));
tcg_gen_addi_i32(tmp, tmp, index);
/* Prepare a power-of-two modulo via extraction of @len bits. */
len = ctz32(streaming_vec_reg_size(s)) - esz;
if (vertical) {
/*
* Compute the byte offset of the index within the tile:
* (index % (svl / size)) * size
* = (index % (svl >> esz)) << esz
* Perform the power-of-two modulo via extraction of the low @len bits.
* Perform the multiply by shifting left by @pos bits.
* Perform these operations simultaneously via deposit into zero.
*/
pos = esz;
tcg_gen_deposit_z_i32(tmp, tmp, pos, len);
/*
* For big-endian, adjust the indexed column byte offset within
* the uint64_t host words that make up env->zarray[].
*/
if (HOST_BIG_ENDIAN && esz < MO_64) {
tcg_gen_xori_i32(tmp, tmp, 8 - (1 << esz));
}
} else {
/*
* Compute the byte offset of the index within the tile:
* (index % (svl / size)) * (size * sizeof(row))
* = (index % (svl >> esz)) << (esz + log2(sizeof(row)))
*/
pos = esz + ctz32(sizeof(ARMVectorReg));
tcg_gen_deposit_z_i32(tmp, tmp, pos, len);
/* Row slices are always aligned and need no endian adjustment. */
}
/* The tile byte offset within env->zarray is the row. */
offset = tile * sizeof(ARMVectorReg);
/* Include the byte offset of zarray to make this relative to env. */
offset += offsetof(CPUARMState, zarray);
tcg_gen_addi_i32(tmp, tmp, offset);
/* Add the byte offset to env to produce the final pointer. */
addr = tcg_temp_new_ptr();
tcg_gen_ext_i32_ptr(addr, tmp);
tcg_temp_free_i32(tmp);
tcg_gen_add_ptr(addr, addr, cpu_env);
return addr;
}
static bool trans_ZERO(DisasContext *s, arg_ZERO *a)
{
if (!dc_isar_feature(aa64_sme, s)) {
return false;
}
if (sme_za_enabled_check(s)) {
gen_helper_sme_zero(cpu_env, tcg_constant_i32(a->imm),
tcg_constant_i32(streaming_vec_reg_size(s)));
}
return true;
}
static bool trans_MOVA(DisasContext *s, arg_MOVA *a)
{
static gen_helper_gvec_4 * const h_fns[5] = {
gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h,
gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d,
gen_helper_sve_sel_zpzz_q
};
static gen_helper_gvec_3 * const cz_fns[5] = {
gen_helper_sme_mova_cz_b, gen_helper_sme_mova_cz_h,
gen_helper_sme_mova_cz_s, gen_helper_sme_mova_cz_d,
gen_helper_sme_mova_cz_q,
};
static gen_helper_gvec_3 * const zc_fns[5] = {
gen_helper_sme_mova_zc_b, gen_helper_sme_mova_zc_h,
gen_helper_sme_mova_zc_s, gen_helper_sme_mova_zc_d,
gen_helper_sme_mova_zc_q,
};
TCGv_ptr t_za, t_zr, t_pg;
TCGv_i32 t_desc;
int svl;
if (!dc_isar_feature(aa64_sme, s)) {
return false;
}
if (!sme_smza_enabled_check(s)) {
return true;
}
t_za = get_tile_rowcol(s, a->esz, a->rs, a->za_imm, a->v);
t_zr = vec_full_reg_ptr(s, a->zr);
t_pg = pred_full_reg_ptr(s, a->pg);
svl = streaming_vec_reg_size(s);
t_desc = tcg_constant_i32(simd_desc(svl, svl, 0));
if (a->v) {
/* Vertical slice -- use sme mova helpers. */
if (a->to_vec) {
zc_fns[a->esz](t_zr, t_za, t_pg, t_desc);
} else {
cz_fns[a->esz](t_za, t_zr, t_pg, t_desc);
}
} else {
/* Horizontal slice -- reuse sve sel helpers. */
if (a->to_vec) {
h_fns[a->esz](t_zr, t_za, t_zr, t_pg, t_desc);
} else {
h_fns[a->esz](t_za, t_zr, t_za, t_pg, t_desc);
}
}
tcg_temp_free_ptr(t_za);
tcg_temp_free_ptr(t_zr);
tcg_temp_free_ptr(t_pg);
return true;
}
static bool trans_LDST1(DisasContext *s, arg_LDST1 *a)
{
typedef void GenLdSt1(TCGv_env, TCGv_ptr, TCGv_ptr, TCGv, TCGv_i32);
/*
* Indexed by [esz][be][v][mte][st], which is (except for load/store)
* also the order in which the elements appear in the function names,
* and so how we must concatenate the pieces.
*/
#define FN_LS(F) { gen_helper_sme_ld1##F, gen_helper_sme_st1##F }
#define FN_MTE(F) { FN_LS(F), FN_LS(F##_mte) }
#define FN_HV(F) { FN_MTE(F##_h), FN_MTE(F##_v) }
#define FN_END(L, B) { FN_HV(L), FN_HV(B) }
static GenLdSt1 * const fns[5][2][2][2][2] = {
FN_END(b, b),
FN_END(h_le, h_be),
FN_END(s_le, s_be),
FN_END(d_le, d_be),
FN_END(q_le, q_be),
};
#undef FN_LS
#undef FN_MTE
#undef FN_HV
#undef FN_END
TCGv_ptr t_za, t_pg;
TCGv_i64 addr;
int svl, desc = 0;
bool be = s->be_data == MO_BE;
bool mte = s->mte_active[0];
if (!dc_isar_feature(aa64_sme, s)) {
return false;
}
if (!sme_smza_enabled_check(s)) {
return true;
}
t_za = get_tile_rowcol(s, a->esz, a->rs, a->za_imm, a->v);
t_pg = pred_full_reg_ptr(s, a->pg);
addr = tcg_temp_new_i64();
tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), a->esz);
tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
if (mte) {
desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
desc = FIELD_DP32(desc, MTEDESC, WRITE, a->st);
desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (1 << a->esz) - 1);
desc <<= SVE_MTEDESC_SHIFT;
} else {
addr = clean_data_tbi(s, addr);
}
svl = streaming_vec_reg_size(s);
desc = simd_desc(svl, svl, desc);
fns[a->esz][be][a->v][mte][a->st](cpu_env, t_za, t_pg, addr,
tcg_constant_i32(desc));
tcg_temp_free_ptr(t_za);
tcg_temp_free_ptr(t_pg);
tcg_temp_free_i64(addr);
return true;
}
typedef void GenLdStR(DisasContext *, TCGv_ptr, int, int, int, int);
static bool do_ldst_r(DisasContext *s, arg_ldstr *a, GenLdStR *fn)
{
int svl = streaming_vec_reg_size(s);
int imm = a->imm;
TCGv_ptr base;
if (!sme_za_enabled_check(s)) {
return true;
}
/* ZA[n] equates to ZA0H.B[n]. */
base = get_tile_rowcol(s, MO_8, a->rv, imm, false);
fn(s, base, 0, svl, a->rn, imm * svl);
tcg_temp_free_ptr(base);
return true;
}
TRANS_FEAT(LDR, aa64_sme, do_ldst_r, a, gen_sve_ldr)
TRANS_FEAT(STR, aa64_sme, do_ldst_r, a, gen_sve_str)
static bool do_adda(DisasContext *s, arg_adda *a, MemOp esz,
gen_helper_gvec_4 *fn)
{
int svl = streaming_vec_reg_size(s);
uint32_t desc = simd_desc(svl, svl, 0);
TCGv_ptr za, zn, pn, pm;
if (!sme_smza_enabled_check(s)) {
return true;
}
/* Sum XZR+zad to find ZAd. */
za = get_tile_rowcol(s, esz, 31, a->zad, false);
zn = vec_full_reg_ptr(s, a->zn);
pn = pred_full_reg_ptr(s, a->pn);
pm = pred_full_reg_ptr(s, a->pm);
fn(za, zn, pn, pm, tcg_constant_i32(desc));
tcg_temp_free_ptr(za);
tcg_temp_free_ptr(zn);
tcg_temp_free_ptr(pn);
tcg_temp_free_ptr(pm);
return true;
}
TRANS_FEAT(ADDHA_s, aa64_sme, do_adda, a, MO_32, gen_helper_sme_addha_s)
TRANS_FEAT(ADDVA_s, aa64_sme, do_adda, a, MO_32, gen_helper_sme_addva_s)
TRANS_FEAT(ADDHA_d, aa64_sme_i16i64, do_adda, a, MO_64, gen_helper_sme_addha_d)
TRANS_FEAT(ADDVA_d, aa64_sme_i16i64, do_adda, a, MO_64, gen_helper_sme_addva_d)
static bool do_outprod(DisasContext *s, arg_op *a, MemOp esz,
gen_helper_gvec_5 *fn)
{
int svl = streaming_vec_reg_size(s);
uint32_t desc = simd_desc(svl, svl, a->sub);
TCGv_ptr za, zn, zm, pn, pm;
if (!sme_smza_enabled_check(s)) {
return true;
}
/* Sum XZR+zad to find ZAd. */
za = get_tile_rowcol(s, esz, 31, a->zad, false);
zn = vec_full_reg_ptr(s, a->zn);
zm = vec_full_reg_ptr(s, a->zm);
pn = pred_full_reg_ptr(s, a->pn);
pm = pred_full_reg_ptr(s, a->pm);
fn(za, zn, zm, pn, pm, tcg_constant_i32(desc));
tcg_temp_free_ptr(za);
tcg_temp_free_ptr(zn);
tcg_temp_free_ptr(pn);
tcg_temp_free_ptr(pm);
return true;
}
static bool do_outprod_fpst(DisasContext *s, arg_op *a, MemOp esz,
gen_helper_gvec_5_ptr *fn)
{
int svl = streaming_vec_reg_size(s);
uint32_t desc = simd_desc(svl, svl, a->sub);
TCGv_ptr za, zn, zm, pn, pm, fpst;
if (!sme_smza_enabled_check(s)) {
return true;
}
/* Sum XZR+zad to find ZAd. */
za = get_tile_rowcol(s, esz, 31, a->zad, false);
zn = vec_full_reg_ptr(s, a->zn);
zm = vec_full_reg_ptr(s, a->zm);
pn = pred_full_reg_ptr(s, a->pn);
pm = pred_full_reg_ptr(s, a->pm);
fpst = fpstatus_ptr(FPST_FPCR);
fn(za, zn, zm, pn, pm, fpst, tcg_constant_i32(desc));
tcg_temp_free_ptr(za);
tcg_temp_free_ptr(zn);
tcg_temp_free_ptr(pn);
tcg_temp_free_ptr(pm);
tcg_temp_free_ptr(fpst);
return true;
}
TRANS_FEAT(FMOPA_h, aa64_sme, do_outprod_fpst, a, MO_32, gen_helper_sme_fmopa_h)
TRANS_FEAT(FMOPA_s, aa64_sme, do_outprod_fpst, a, MO_32, gen_helper_sme_fmopa_s)
TRANS_FEAT(FMOPA_d, aa64_sme_f64f64, do_outprod_fpst, a, MO_64, gen_helper_sme_fmopa_d)
/* TODO: FEAT_EBF16 */
TRANS_FEAT(BFMOPA, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_bfmopa)
TRANS_FEAT(SMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_smopa_s)
TRANS_FEAT(UMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_umopa_s)
TRANS_FEAT(SUMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_sumopa_s)
TRANS_FEAT(USMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_usmopa_s)
TRANS_FEAT(SMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_smopa_d)
TRANS_FEAT(UMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_umopa_d)
TRANS_FEAT(SUMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_sumopa_d)
TRANS_FEAT(USMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_usmopa_d)

393
target/arm/translate-sve.c
View File

@ -1286,6 +1286,19 @@ static bool trans_ADDVL(DisasContext *s, arg_ADDVL *a)
return true;
}
static bool trans_ADDSVL(DisasContext *s, arg_ADDSVL *a)
{
if (!dc_isar_feature(aa64_sme, s)) {
return false;
}
if (sme_enabled_check(s)) {
TCGv_i64 rd = cpu_reg_sp(s, a->rd);
TCGv_i64 rn = cpu_reg_sp(s, a->rn);
tcg_gen_addi_i64(rd, rn, a->imm * streaming_vec_reg_size(s));
}
return true;
}
static bool trans_ADDPL(DisasContext *s, arg_ADDPL *a)
{
if (!dc_isar_feature(aa64_sve, s)) {
@ -1299,6 +1312,19 @@ static bool trans_ADDPL(DisasContext *s, arg_ADDPL *a)
return true;
}
static bool trans_ADDSPL(DisasContext *s, arg_ADDSPL *a)
{
if (!dc_isar_feature(aa64_sme, s)) {
return false;
}
if (sme_enabled_check(s)) {
TCGv_i64 rd = cpu_reg_sp(s, a->rd);
TCGv_i64 rn = cpu_reg_sp(s, a->rn);
tcg_gen_addi_i64(rd, rn, a->imm * streaming_pred_reg_size(s));
}
return true;
}
static bool trans_RDVL(DisasContext *s, arg_RDVL *a)
{
if (!dc_isar_feature(aa64_sve, s)) {
@ -1311,6 +1337,18 @@ static bool trans_RDVL(DisasContext *s, arg_RDVL *a)
return true;
}
static bool trans_RDSVL(DisasContext *s, arg_RDSVL *a)
{
if (!dc_isar_feature(aa64_sme, s)) {
return false;
}
if (sme_enabled_check(s)) {
TCGv_i64 reg = cpu_reg(s, a->rd);
tcg_gen_movi_i64(reg, a->imm * streaming_vec_reg_size(s));
}
return true;
}
/*
*** SVE Compute Vector Address Group
*/
@ -1320,10 +1358,10 @@ static bool do_adr(DisasContext *s, arg_rrri *a, gen_helper_gvec_3 *fn)
return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, a->imm);
}
TRANS_FEAT(ADR_p32, aa64_sve, do_adr, a, gen_helper_sve_adr_p32)
TRANS_FEAT(ADR_p64, aa64_sve, do_adr, a, gen_helper_sve_adr_p64)
TRANS_FEAT(ADR_s32, aa64_sve, do_adr, a, gen_helper_sve_adr_s32)
TRANS_FEAT(ADR_u32, aa64_sve, do_adr, a, gen_helper_sve_adr_u32)
TRANS_FEAT_NONSTREAMING(ADR_p32, aa64_sve, do_adr, a, gen_helper_sve_adr_p32)
TRANS_FEAT_NONSTREAMING(ADR_p64, aa64_sve, do_adr, a, gen_helper_sve_adr_p64)
TRANS_FEAT_NONSTREAMING(ADR_s32, aa64_sve, do_adr, a, gen_helper_sve_adr_s32)
TRANS_FEAT_NONSTREAMING(ADR_u32, aa64_sve, do_adr, a, gen_helper_sve_adr_u32)
/*
*** SVE Integer Misc - Unpredicated Group
@ -1333,14 +1371,15 @@ static gen_helper_gvec_2 * const fexpa_fns[4] = {
NULL, gen_helper_sve_fexpa_h,
gen_helper_sve_fexpa_s, gen_helper_sve_fexpa_d,
};
TRANS_FEAT(FEXPA, aa64_sve, gen_gvec_ool_zz,
fexpa_fns[a->esz], a->rd, a->rn, 0)
TRANS_FEAT_NONSTREAMING(FEXPA, aa64_sve, gen_gvec_ool_zz,
fexpa_fns[a->esz], a->rd, a->rn, 0)
static gen_helper_gvec_3 * const ftssel_fns[4] = {
NULL, gen_helper_sve_ftssel_h,
gen_helper_sve_ftssel_s, gen_helper_sve_ftssel_d,
};
TRANS_FEAT(FTSSEL, aa64_sve, gen_gvec_ool_arg_zzz, ftssel_fns[a->esz], a, 0)
TRANS_FEAT_NONSTREAMING(FTSSEL, aa64_sve, gen_gvec_ool_arg_zzz,
ftssel_fns[a->esz], a, 0)
/*
*** SVE Predicate Logical Operations Group
@ -1785,7 +1824,8 @@ static bool do_predset(DisasContext *s, int esz, int rd, int pat, bool setflag)
TRANS_FEAT(PTRUE, aa64_sve, do_predset, a->esz, a->rd, a->pat, a->s)
/* Note pat == 31 is #all, to set all elements. */
TRANS_FEAT(SETFFR, aa64_sve, do_predset, 0, FFR_PRED_NUM, 31, false)
TRANS_FEAT_NONSTREAMING(SETFFR, aa64_sve,
do_predset, 0, FFR_PRED_NUM, 31, false)
/* Note pat == 32 is #unimp, to set no elements. */
TRANS_FEAT(PFALSE, aa64_sve, do_predset, 0, a->rd, 32, false)
@ -1799,11 +1839,13 @@ static bool trans_RDFFR_p(DisasContext *s, arg_RDFFR_p *a)
.rd = a->rd, .pg = a->pg, .s = a->s,
.rn = FFR_PRED_NUM, .rm = FFR_PRED_NUM,
};
s->is_nonstreaming = true;
return trans_AND_pppp(s, &alt_a);
}
TRANS_FEAT(RDFFR, aa64_sve, do_mov_p, a->rd, FFR_PRED_NUM)
TRANS_FEAT(WRFFR, aa64_sve, do_mov_p, FFR_PRED_NUM, a->rn)
TRANS_FEAT_NONSTREAMING(RDFFR, aa64_sve, do_mov_p, a->rd, FFR_PRED_NUM)
TRANS_FEAT_NONSTREAMING(WRFFR, aa64_sve, do_mov_p, FFR_PRED_NUM, a->rn)
static bool do_pfirst_pnext(DisasContext *s, arg_rr_esz *a,
void (*gen_fn)(TCGv_i32, TCGv_ptr,
@ -2533,7 +2575,8 @@ TRANS_FEAT(TRN2_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz,
static gen_helper_gvec_3 * const compact_fns[4] = {
NULL, NULL, gen_helper_sve_compact_s, gen_helper_sve_compact_d
};
TRANS_FEAT(COMPACT, aa64_sve, gen_gvec_ool_arg_zpz, compact_fns[a->esz], a, 0)
TRANS_FEAT_NONSTREAMING(COMPACT, aa64_sve, gen_gvec_ool_arg_zpz,
compact_fns[a->esz], a, 0)
/* Call the helper that computes the ARM LastActiveElement pseudocode
* function, scaled by the element size. This includes the not found
@ -2858,6 +2901,8 @@ TRANS_FEAT(REVH, aa64_sve, gen_gvec_ool_arg_zpz, revh_fns[a->esz], a, 0)
TRANS_FEAT(REVW, aa64_sve, gen_gvec_ool_arg_zpz,
a->esz == 3 ? gen_helper_sve_revw_d : NULL, a, 0)
TRANS_FEAT(REVD, aa64_sme, gen_gvec_ool_arg_zpz, gen_helper_sme_revd_q, a, 0)
TRANS_FEAT(SPLICE, aa64_sve, gen_gvec_ool_arg_zpzz,
gen_helper_sve_splice, a, a->esz)
@ -3856,9 +3901,9 @@ static gen_helper_gvec_3_ptr * const ftmad_fns[4] = {
NULL, gen_helper_sve_ftmad_h,
gen_helper_sve_ftmad_s, gen_helper_sve_ftmad_d,
};
TRANS_FEAT(FTMAD, aa64_sve, gen_gvec_fpst_zzz,
ftmad_fns[a->esz], a->rd, a->rn, a->rm, a->imm,
a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
TRANS_FEAT_NONSTREAMING(FTMAD, aa64_sve, gen_gvec_fpst_zzz,
ftmad_fns[a->esz], a->rd, a->rn, a->rm, a->imm,
a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
/*
*** SVE Floating Point Accumulating Reduction Group
@ -3881,6 +3926,7 @@ static bool trans_FADDA(DisasContext *s, arg_rprr_esz *a)
if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
return false;
}
s->is_nonstreaming = true;
if (!sve_access_check(s)) {
return true;
}
@ -3918,12 +3964,18 @@ static bool trans_FADDA(DisasContext *s, arg_rprr_esz *a)
DO_FP3(FADD_zzz, fadd)
DO_FP3(FSUB_zzz, fsub)
DO_FP3(FMUL_zzz, fmul)
DO_FP3(FTSMUL, ftsmul)
DO_FP3(FRECPS, recps)
DO_FP3(FRSQRTS, rsqrts)
#undef DO_FP3
static gen_helper_gvec_3_ptr * const ftsmul_fns[4] = {
NULL, gen_helper_gvec_ftsmul_h,
gen_helper_gvec_ftsmul_s, gen_helper_gvec_ftsmul_d
};
TRANS_FEAT_NONSTREAMING(FTSMUL, aa64_sve, gen_gvec_fpst_arg_zzz,
ftsmul_fns[a->esz], a, 0)
/*
*** SVE Floating Point Arithmetic - Predicated Group
*/
@ -4256,7 +4308,8 @@ TRANS_FEAT(UCVTF_dd, aa64_sve, gen_gvec_fpst_arg_zpz,
* The load should begin at the address Rn + IMM.
*/
static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
void gen_sve_ldr(DisasContext *s, TCGv_ptr base, int vofs,
int len, int rn, int imm)
{
int len_align = QEMU_ALIGN_DOWN(len, 8);
int len_remain = len % 8;
@ -4282,7 +4335,7 @@ static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
t0 = tcg_temp_new_i64();
for (i = 0; i < len_align; i += 8) {
tcg_gen_qemu_ld_i64(t0, clean_addr, midx, MO_LEUQ);
tcg_gen_st_i64(t0, cpu_env, vofs + i);
tcg_gen_st_i64(t0, base, vofs + i);
tcg_gen_addi_i64(clean_addr, clean_addr, 8);
}
tcg_temp_free_i64(t0);
@ -4295,6 +4348,12 @@ static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
clean_addr = new_tmp_a64_local(s);
tcg_gen_mov_i64(clean_addr, t0);
if (base != cpu_env) {
TCGv_ptr b = tcg_temp_local_new_ptr();
tcg_gen_mov_ptr(b, base);
base = b;
}
gen_set_label(loop);
t0 = tcg_temp_new_i64();
@ -4302,7 +4361,7 @@ static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
tcg_gen_addi_i64(clean_addr, clean_addr, 8);
tp = tcg_temp_new_ptr();
tcg_gen_add_ptr(tp, cpu_env, i);
tcg_gen_add_ptr(tp, base, i);
tcg_gen_addi_ptr(i, i, 8);
tcg_gen_st_i64(t0, tp, vofs);
tcg_temp_free_ptr(tp);
@ -4310,6 +4369,11 @@ static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
tcg_gen_brcondi_ptr(TCG_COND_LTU, i, len_align, loop);
tcg_temp_free_ptr(i);
if (base != cpu_env) {
tcg_temp_free_ptr(base);
assert(len_remain == 0);
}
}
/*
@ -4338,13 +4402,14 @@ static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
default:
g_assert_not_reached();
}
tcg_gen_st_i64(t0, cpu_env, vofs + len_align);
tcg_gen_st_i64(t0, base, vofs + len_align);
tcg_temp_free_i64(t0);
}
}
/* Similarly for stores. */
static void do_str(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
void gen_sve_str(DisasContext *s, TCGv_ptr base, int vofs,
int len, int rn, int imm)
{
int len_align = QEMU_ALIGN_DOWN(len, 8);
int len_remain = len % 8;
@ -4370,7 +4435,7 @@ static void do_str(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
t0 = tcg_temp_new_i64();
for (i = 0; i < len_align; i += 8) {
tcg_gen_ld_i64(t0, cpu_env, vofs + i);
tcg_gen_ld_i64(t0, base, vofs + i);
tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUQ);
tcg_gen_addi_i64(clean_addr, clean_addr, 8);
}
@ -4384,11 +4449,17 @@ static void do_str(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
clean_addr = new_tmp_a64_local(s);
tcg_gen_mov_i64(clean_addr, t0);
if (base != cpu_env) {
TCGv_ptr b = tcg_temp_local_new_ptr();
tcg_gen_mov_ptr(b, base);
base = b;
}
gen_set_label(loop);
t0 = tcg_temp_new_i64();
tp = tcg_temp_new_ptr();
tcg_gen_add_ptr(tp, cpu_env, i);
tcg_gen_add_ptr(tp, base, i);
tcg_gen_ld_i64(t0, tp, vofs);
tcg_gen_addi_ptr(i, i, 8);
tcg_temp_free_ptr(tp);
@ -4399,12 +4470,17 @@ static void do_str(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
tcg_gen_brcondi_ptr(TCG_COND_LTU, i, len_align, loop);
tcg_temp_free_ptr(i);
if (base != cpu_env) {
tcg_temp_free_ptr(base);
assert(len_remain == 0);
}
}
/* Predicate register stores can be any multiple of 2. */
if (len_remain) {
t0 = tcg_temp_new_i64();
tcg_gen_ld_i64(t0, cpu_env, vofs + len_align);
tcg_gen_ld_i64(t0, base, vofs + len_align);
switch (len_remain) {
case 2:
@ -4436,7 +4512,7 @@ static bool trans_LDR_zri(DisasContext *s, arg_rri *a)
if (sve_access_check(s)) {
int size = vec_full_reg_size(s);
int off = vec_full_reg_offset(s, a->rd);
do_ldr(s, off, size, a->rn, a->imm * size);
gen_sve_ldr(s, cpu_env, off, size, a->rn, a->imm * size);
}
return true;
}
@ -4449,7 +4525,7 @@ static bool trans_LDR_pri(DisasContext *s, arg_rri *a)
if (sve_access_check(s)) {
int size = pred_full_reg_size(s);
int off = pred_full_reg_offset(s, a->rd);
do_ldr(s, off, size, a->rn, a->imm * size);
gen_sve_ldr(s, cpu_env, off, size, a->rn, a->imm * size);
}
return true;
}
@ -4462,7 +4538,7 @@ static bool trans_STR_zri(DisasContext *s, arg_rri *a)
if (sve_access_check(s)) {
int size = vec_full_reg_size(s);
int off = vec_full_reg_offset(s, a->rd);
do_str(s, off, size, a->rn, a->imm * size);
gen_sve_str(s, cpu_env, off, size, a->rn, a->imm * size);
}
return true;
}
@ -4475,7 +4551,7 @@ static bool trans_STR_pri(DisasContext *s, arg_rri *a)
if (sve_access_check(s)) {
int size = pred_full_reg_size(s);
int off = pred_full_reg_offset(s, a->rd);
do_str(s, off, size, a->rn, a->imm * size);
gen_sve_str(s, cpu_env, off, size, a->rn, a->imm * size);
}
return true;
}
@ -4793,6 +4869,7 @@ static bool trans_LDFF1_zprr(DisasContext *s, arg_rprr_load *a)
if (!dc_isar_feature(aa64_sve, s)) {
return false;
}
s->is_nonstreaming = true;
if (sve_access_check(s)) {
TCGv_i64 addr = new_tmp_a64(s);
tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));
@ -4894,6 +4971,7 @@ static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a)
if (!dc_isar_feature(aa64_sve, s)) {
return false;
}
s->is_nonstreaming = true;
if (sve_access_check(s)) {
int vsz = vec_full_reg_size(s);
int elements = vsz >> dtype_esz[a->dtype];
@ -5048,6 +5126,7 @@ static bool trans_LD1RO_zprr(DisasContext *s, arg_rprr_load *a)
if (a->rm == 31) {
return false;
}
s->is_nonstreaming = true;
if (sve_access_check(s)) {
TCGv_i64 addr = new_tmp_a64(s);
tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));
@ -5062,6 +5141,7 @@ static bool trans_LD1RO_zpri(DisasContext *s, arg_rpri_load *a)
if (!dc_isar_feature(aa64_sve_f64mm, s)) {
return false;
}
s->is_nonstreaming = true;
if (sve_access_check(s)) {
TCGv_i64 addr = new_tmp_a64(s);
tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 32);
@ -5657,6 +5737,7 @@ static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a)
if (!dc_isar_feature(aa64_sve, s)) {
return false;
}
s->is_nonstreaming = true;
if (!sve_access_check(s)) {
return true;
}
@ -5688,6 +5769,7 @@ static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a)
if (!dc_isar_feature(aa64_sve, s)) {
return false;
}
s->is_nonstreaming = true;
if (!sve_access_check(s)) {
return true;
}
@ -5722,6 +5804,7 @@ static bool trans_LDNT1_zprz(DisasContext *s, arg_LD1_zprz *a)
if (!dc_isar_feature(aa64_sve2, s)) {
return false;
}
s->is_nonstreaming = true;
if (!sve_access_check(s)) {
return true;
}
@ -5845,6 +5928,7 @@ static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a)
if (!dc_isar_feature(aa64_sve, s)) {
return false;
}
s->is_nonstreaming = true;
if (!sve_access_check(s)) {
return true;
}
@ -5875,6 +5959,7 @@ static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a)
if (!dc_isar_feature(aa64_sve, s)) {
return false;
}
s->is_nonstreaming = true;
if (!sve_access_check(s)) {
return true;
}
@ -5909,6 +5994,7 @@ static bool trans_STNT1_zprz(DisasContext *s, arg_ST1_zprz *a)
if (!dc_isar_feature(aa64_sve2, s)) {
return false;
}
s->is_nonstreaming = true;
if (!sve_access_check(s)) {
return true;
}
@ -5953,6 +6039,17 @@ static bool trans_PRF_rr(DisasContext *s, arg_PRF_rr *a)
return true;
}
static bool trans_PRF_ns(DisasContext *s, arg_PRF_ns *a)
{
if (!dc_isar_feature(aa64_sve, s)) {
return false;
}
/* Prefetch is a nop within QEMU. */
s->is_nonstreaming = true;
(void)sve_access_check(s);
return true;
}
/*
* Move Prefix
*
@ -6181,9 +6278,13 @@ static bool do_trans_pmull(DisasContext *s, arg_rrr_esz *a, bool sel)
gen_helper_gvec_pmull_q, gen_helper_sve2_pmull_h,
NULL, gen_helper_sve2_pmull_d,
};
if (a->esz == 0
? !dc_isar_feature(aa64_sve2_pmull128, s)
: !dc_isar_feature(aa64_sve, s)) {
if (a->esz == 0) {
if (!dc_isar_feature(aa64_sve2_pmull128, s)) {
return false;
}
s->is_nonstreaming = true;
} else if (!dc_isar_feature(aa64_sve, s)) {
return false;
}
return gen_gvec_ool_arg_zzz(s, fns[a->esz], a, sel);
@ -6371,22 +6472,22 @@ static gen_helper_gvec_3 * const bext_fns[4] = {
gen_helper_sve2_bext_b, gen_helper_sve2_bext_h,
gen_helper_sve2_bext_s, gen_helper_sve2_bext_d,
};
TRANS_FEAT(BEXT, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz,
bext_fns[a->esz], a, 0)
TRANS_FEAT_NONSTREAMING(BEXT, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz,
bext_fns[a->esz], a, 0)
static gen_helper_gvec_3 * const bdep_fns[4] = {
gen_helper_sve2_bdep_b, gen_helper_sve2_bdep_h,
gen_helper_sve2_bdep_s, gen_helper_sve2_bdep_d,
};
TRANS_FEAT(BDEP, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz,
bdep_fns[a->esz], a, 0)
TRANS_FEAT_NONSTREAMING(BDEP, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz,
bdep_fns[a->esz], a, 0)
static gen_helper_gvec_3 * const bgrp_fns[4] = {
gen_helper_sve2_bgrp_b, gen_helper_sve2_bgrp_h,
gen_helper_sve2_bgrp_s, gen_helper_sve2_bgrp_d,
};
TRANS_FEAT(BGRP, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz,
bgrp_fns[a->esz], a, 0)
TRANS_FEAT_NONSTREAMING(BGRP, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz,
bgrp_fns[a->esz], a, 0)
static gen_helper_gvec_3 * const cadd_fns[4] = {
gen_helper_sve2_cadd_b, gen_helper_sve2_cadd_h,
@ -7094,21 +7195,21 @@ DO_SVE2_ZZZ_NARROW(RSUBHNT, rsubhnt)
static gen_helper_gvec_flags_4 * const match_fns[4] = {
gen_helper_sve2_match_ppzz_b, gen_helper_sve2_match_ppzz_h, NULL, NULL
};
TRANS_FEAT(MATCH, aa64_sve2, do_ppzz_flags, a, match_fns[a->esz])
TRANS_FEAT_NONSTREAMING(MATCH, aa64_sve2, do_ppzz_flags, a, match_fns[a->esz])
static gen_helper_gvec_flags_4 * const nmatch_fns[4] = {
gen_helper_sve2_nmatch_ppzz_b, gen_helper_sve2_nmatch_ppzz_h, NULL, NULL
};
TRANS_FEAT(NMATCH, aa64_sve2, do_ppzz_flags, a, nmatch_fns[a->esz])
TRANS_FEAT_NONSTREAMING(NMATCH, aa64_sve2, do_ppzz_flags, a, nmatch_fns[a->esz])
static gen_helper_gvec_4 * const histcnt_fns[4] = {
NULL, NULL, gen_helper_sve2_histcnt_s, gen_helper_sve2_histcnt_d
};
TRANS_FEAT(HISTCNT, aa64_sve2, gen_gvec_ool_arg_zpzz,
histcnt_fns[a->esz], a, 0)
TRANS_FEAT_NONSTREAMING(HISTCNT, aa64_sve2, gen_gvec_ool_arg_zpzz,
histcnt_fns[a->esz], a, 0)
TRANS_FEAT(HISTSEG, aa64_sve2, gen_gvec_ool_arg_zzz,
a->esz == 0 ? gen_helper_sve2_histseg : NULL, a, 0)
TRANS_FEAT_NONSTREAMING(HISTSEG, aa64_sve2, gen_gvec_ool_arg_zzz,
a->esz == 0 ? gen_helper_sve2_histseg : NULL, a, 0)
DO_ZPZZ_FP(FADDP, aa64_sve2, sve2_faddp_zpzz)
DO_ZPZZ_FP(FMAXNMP, aa64_sve2, sve2_fmaxnmp_zpzz)
@ -7120,10 +7221,12 @@ DO_ZPZZ_FP(FMINP, aa64_sve2, sve2_fminp_zpzz)
* SVE Integer Multiply-Add (unpredicated)
*/
TRANS_FEAT(FMMLA_s, aa64_sve_f32mm, gen_gvec_fpst_zzzz, gen_helper_fmmla_s,
a->rd, a->rn, a->rm, a->ra, 0, FPST_FPCR)
TRANS_FEAT(FMMLA_d, aa64_sve_f64mm, gen_gvec_fpst_zzzz, gen_helper_fmmla_d,
a->rd, a->rn, a->rm, a->ra, 0, FPST_FPCR)
TRANS_FEAT_NONSTREAMING(FMMLA_s, aa64_sve_f32mm, gen_gvec_fpst_zzzz,
gen_helper_fmmla_s, a->rd, a->rn, a->rm, a->ra,
0, FPST_FPCR)
TRANS_FEAT_NONSTREAMING(FMMLA_d, aa64_sve_f64mm, gen_gvec_fpst_zzzz,
gen_helper_fmmla_d, a->rd, a->rn, a->rm, a->ra,
0, FPST_FPCR)
static gen_helper_gvec_4 * const sqdmlal_zzzw_fns[] = {
NULL, gen_helper_sve2_sqdmlal_zzzw_h,
@ -7220,20 +7323,21 @@ TRANS_FEAT(SQRDCMLAH_zzzz, aa64_sve2, gen_gvec_ool_zzzz,
TRANS_FEAT(USDOT_zzzz, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
a->esz == 2 ? gen_helper_gvec_usdot_b : NULL, a, 0)
TRANS_FEAT(AESMC, aa64_sve2_aes, gen_gvec_ool_zz,
gen_helper_crypto_aesmc, a->rd, a->rd, a->decrypt)
TRANS_FEAT_NONSTREAMING(AESMC, aa64_sve2_aes, gen_gvec_ool_zz,
gen_helper_crypto_aesmc, a->rd, a->rd, a->decrypt)
TRANS_FEAT(AESE, aa64_sve2_aes, gen_gvec_ool_arg_zzz,
gen_helper_crypto_aese, a, false)
TRANS_FEAT(AESD, aa64_sve2_aes, gen_gvec_ool_arg_zzz,
gen_helper_crypto_aese, a, true)
TRANS_FEAT_NONSTREAMING(AESE, aa64_sve2_aes, gen_gvec_ool_arg_zzz,
gen_helper_crypto_aese, a, false)
TRANS_FEAT_NONSTREAMING(AESD, aa64_sve2_aes, gen_gvec_ool_arg_zzz,
gen_helper_crypto_aese, a, true)
TRANS_FEAT(SM4E, aa64_sve2_sm4, gen_gvec_ool_arg_zzz,
gen_helper_crypto_sm4e, a, 0)
TRANS_FEAT(SM4EKEY, aa64_sve2_sm4, gen_gvec_ool_arg_zzz,
gen_helper_crypto_sm4ekey, a, 0)
TRANS_FEAT_NONSTREAMING(SM4E, aa64_sve2_sm4, gen_gvec_ool_arg_zzz,
gen_helper_crypto_sm4e, a, 0)
TRANS_FEAT_NONSTREAMING(SM4EKEY, aa64_sve2_sm4, gen_gvec_ool_arg_zzz,
gen_helper_crypto_sm4ekey, a, 0)
TRANS_FEAT(RAX1, aa64_sve2_sha3, gen_gvec_fn_arg_zzz, gen_gvec_rax1, a)
TRANS_FEAT_NONSTREAMING(RAX1, aa64_sve2_sha3, gen_gvec_fn_arg_zzz,
gen_gvec_rax1, a)
TRANS_FEAT(FCVTNT_sh, aa64_sve2, gen_gvec_fpst_arg_zpz,
gen_helper_sve2_fcvtnt_sh, a, 0, FPST_FPCR)
@ -7284,20 +7388,20 @@ TRANS_FEAT(FMLALT_zzxw, aa64_sve2, do_FMLAL_zzxw, a, false, true)
TRANS_FEAT(FMLSLB_zzxw, aa64_sve2, do_FMLAL_zzxw, a, true, false)
TRANS_FEAT(FMLSLT_zzxw, aa64_sve2, do_FMLAL_zzxw, a, true, true)
TRANS_FEAT(SMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
gen_helper_gvec_smmla_b, a, 0)
TRANS_FEAT(USMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
gen_helper_gvec_usmmla_b, a, 0)
TRANS_FEAT(UMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
gen_helper_gvec_ummla_b, a, 0)
TRANS_FEAT_NONSTREAMING(SMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
gen_helper_gvec_smmla_b, a, 0)
TRANS_FEAT_NONSTREAMING(USMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
gen_helper_gvec_usmmla_b, a, 0)
TRANS_FEAT_NONSTREAMING(UMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
gen_helper_gvec_ummla_b, a, 0)
TRANS_FEAT(BFDOT_zzzz, aa64_sve_bf16, gen_gvec_ool_arg_zzzz,
gen_helper_gvec_bfdot, a, 0)
TRANS_FEAT(BFDOT_zzxz, aa64_sve_bf16, gen_gvec_ool_arg_zzxz,
gen_helper_gvec_bfdot_idx, a)
TRANS_FEAT(BFMMLA, aa64_sve_bf16, gen_gvec_ool_arg_zzzz,
gen_helper_gvec_bfmmla, a, 0)
TRANS_FEAT_NONSTREAMING(BFMMLA, aa64_sve_bf16, gen_gvec_ool_arg_zzzz,
gen_helper_gvec_bfmmla, a, 0)
static bool do_BFMLAL_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel)
{
@ -7317,3 +7421,162 @@ static bool do_BFMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a, bool sel)
TRANS_FEAT(BFMLALB_zzxw, aa64_sve_bf16, do_BFMLAL_zzxw, a, false)
TRANS_FEAT(BFMLALT_zzxw, aa64_sve_bf16, do_BFMLAL_zzxw, a, true)
static bool trans_PSEL(DisasContext *s, arg_psel *a)
{
int vl = vec_full_reg_size(s);
int pl = pred_gvec_reg_size(s);
int elements = vl >> a->esz;
TCGv_i64 tmp, didx, dbit;
TCGv_ptr ptr;
if (!dc_isar_feature(aa64_sme, s)) {
return false;
}
if (!sve_access_check(s)) {
return true;
}
tmp = tcg_temp_new_i64();
dbit = tcg_temp_new_i64();
didx = tcg_temp_new_i64();
ptr = tcg_temp_new_ptr();
/* Compute the predicate element. */
tcg_gen_addi_i64(tmp, cpu_reg(s, a->rv), a->imm);
if (is_power_of_2(elements)) {
tcg_gen_andi_i64(tmp, tmp, elements - 1);
} else {
tcg_gen_remu_i64(tmp, tmp, tcg_constant_i64(elements));
}
/* Extract the predicate byte and bit indices. */
tcg_gen_shli_i64(tmp, tmp, a->esz);
tcg_gen_andi_i64(dbit, tmp, 7);
tcg_gen_shri_i64(didx, tmp, 3);
if (HOST_BIG_ENDIAN) {
tcg_gen_xori_i64(didx, didx, 7);
}
/* Load the predicate word. */
tcg_gen_trunc_i64_ptr(ptr, didx);
tcg_gen_add_ptr(ptr, ptr, cpu_env);
tcg_gen_ld8u_i64(tmp, ptr, pred_full_reg_offset(s, a->pm));
/* Extract the predicate bit and replicate to MO_64. */
tcg_gen_shr_i64(tmp, tmp, dbit);
tcg_gen_andi_i64(tmp, tmp, 1);
tcg_gen_neg_i64(tmp, tmp);
/* Apply to either copy the source, or write zeros. */
tcg_gen_gvec_ands(MO_64, pred_full_reg_offset(s, a->pd),
pred_full_reg_offset(s, a->pn), tmp, pl, pl);
tcg_temp_free_i64(tmp);
tcg_temp_free_i64(dbit);
tcg_temp_free_i64(didx);
tcg_temp_free_ptr(ptr);
return true;
}
static void gen_sclamp_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_i32 a)
{
tcg_gen_smax_i32(d, a, n);
tcg_gen_smin_i32(d, d, m);
}
static void gen_sclamp_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 a)
{
tcg_gen_smax_i64(d, a, n);
tcg_gen_smin_i64(d, d, m);
}
static void gen_sclamp_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
TCGv_vec m, TCGv_vec a)
{
tcg_gen_smax_vec(vece, d, a, n);
tcg_gen_smin_vec(vece, d, d, m);
}
static void gen_sclamp(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
uint32_t a, uint32_t oprsz, uint32_t maxsz)
{
static const TCGOpcode vecop[] = {
INDEX_op_smin_vec, INDEX_op_smax_vec, 0
};
static const GVecGen4 ops[4] = {
{ .fniv = gen_sclamp_vec,
.fno = gen_helper_gvec_sclamp_b,
.opt_opc = vecop,
.vece = MO_8 },
{ .fniv = gen_sclamp_vec,
.fno = gen_helper_gvec_sclamp_h,
.opt_opc = vecop,
.vece = MO_16 },
{ .fni4 = gen_sclamp_i32,
.fniv = gen_sclamp_vec,
.fno = gen_helper_gvec_sclamp_s,
.opt_opc = vecop,
.vece = MO_32 },
{ .fni8 = gen_sclamp_i64,
.fniv = gen_sclamp_vec,
.fno = gen_helper_gvec_sclamp_d,
.opt_opc = vecop,
.vece = MO_64,
.prefer_i64 = TCG_TARGET_REG_BITS == 64 }
};
tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &ops[vece]);
}
TRANS_FEAT(SCLAMP, aa64_sme, gen_gvec_fn_arg_zzzz, gen_sclamp, a)
static void gen_uclamp_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_i32 a)
{
tcg_gen_umax_i32(d, a, n);
tcg_gen_umin_i32(d, d, m);
}
static void gen_uclamp_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 a)
{
tcg_gen_umax_i64(d, a, n);
tcg_gen_umin_i64(d, d, m);
}
static void gen_uclamp_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
TCGv_vec m, TCGv_vec a)
{
tcg_gen_umax_vec(vece, d, a, n);
tcg_gen_umin_vec(vece, d, d, m);
}
static void gen_uclamp(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
uint32_t a, uint32_t oprsz, uint32_t maxsz)
{
static const TCGOpcode vecop[] = {
INDEX_op_umin_vec, INDEX_op_umax_vec, 0
};
static const GVecGen4 ops[4] = {
{ .fniv = gen_uclamp_vec,
.fno = gen_helper_gvec_uclamp_b,
.opt_opc = vecop,
.vece = MO_8 },
{ .fniv = gen_uclamp_vec,
.fno = gen_helper_gvec_uclamp_h,
.opt_opc = vecop,
.vece = MO_16 },
{ .fni4 = gen_uclamp_i32,
.fniv = gen_uclamp_vec,
.fno = gen_helper_gvec_uclamp_s,
.opt_opc = vecop,
.vece = MO_32 },
{ .fni8 = gen_uclamp_i64,
.fniv = gen_uclamp_vec,
.fno = gen_helper_gvec_uclamp_d,
.opt_opc = vecop,
.vece = MO_64,
.prefer_i64 = TCG_TARGET_REG_BITS == 64 }
};
tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &ops[vece]);
}
TRANS_FEAT(UCLAMP, aa64_sme, gen_gvec_fn_arg_zzzz, gen_uclamp, a)

12
target/arm/translate-vfp.c
View File

@ -234,6 +234,18 @@ static bool vfp_access_check_a(DisasContext *s, bool ignore_vfp_enabled)
return false;
}
/*
* Note that rebuild_hflags_a32 has already accounted for being in EL0
* and the higher EL in A64 mode, etc. Unlike A64 mode, there do not
* appear to be any insns which touch VFP which are allowed.
*/
if (s->sme_trap_nonstreaming) {
gen_exception_insn(s, s->pc_curr, EXCP_UDEF,
syn_smetrap(SME_ET_Streaming,
s->base.pc_next - s->pc_curr == 2));
return false;
}
if (!s->vfp_enabled && !ignore_vfp_enabled) {
assert(!arm_dc_feature(s, ARM_FEATURE_M));
unallocated_encoding(s);

2
target/arm/translate.c
View File

@ -9378,6 +9378,8 @@ static void arm_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs)
dc->vec_len = EX_TBFLAG_A32(tb_flags, VECLEN);
dc->vec_stride = EX_TBFLAG_A32(tb_flags, VECSTRIDE);
}
dc->sme_trap_nonstreaming =
EX_TBFLAG_A32(tb_flags, SME_TRAP_NONSTREAMING);
}
dc->cp_regs = cpu->cp_regs;
dc->features = env->features;

16
target/arm/translate.h
View File

@ -102,6 +102,10 @@ typedef struct DisasContext {
bool pstate_sm;
/* True if PSTATE.ZA is set. */
bool pstate_za;
/* True if non-streaming insns should raise an SME Streaming exception. */
bool sme_trap_nonstreaming;
/* True if the current instruction is non-streaming. */
bool is_nonstreaming;
/* True if MVE insns are definitely not predicated by VPR or LTPSIZE */
bool mve_no_pred;
/*
@ -152,6 +156,11 @@ static inline int plus_2(DisasContext *s, int x)
return x + 2;
}
static inline int plus_12(DisasContext *s, int x)
{
return x + 12;
}
static inline int times_2(DisasContext *s, int x)
{
return x * 2;
@ -562,4 +571,11 @@ uint64_t asimd_imm_const(uint32_t imm, int cmode, int op);
static bool trans_##NAME(DisasContext *s, arg_##NAME *a) \
{ return dc_isar_feature(FEAT, s) && FUNC(s, __VA_ARGS__); }
#define TRANS_FEAT_NONSTREAMING(NAME, FEAT, FUNC, ...) \
static bool trans_##NAME(DisasContext *s, arg_##NAME *a) \
{ \
s->is_nonstreaming = true; \
return dc_isar_feature(FEAT, s) && FUNC(s, __VA_ARGS__); \
}
#endif /* TARGET_ARM_TRANSLATE_H */

24
target/arm/vec_helper.c
View File

@ -2690,3 +2690,27 @@ void HELPER(gvec_bfmlal_idx)(void *vd, void *vn, void *vm,
}
clear_tail(d, opr_sz, simd_maxsz(desc));
}
#define DO_CLAMP(NAME, TYPE) \
void HELPER(NAME)(void *d, void *n, void *m, void *a, uint32_t desc) \
{ \
intptr_t i, opr_sz = simd_oprsz(desc); \
for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \
TYPE aa = *(TYPE *)(a + i); \
TYPE nn = *(TYPE *)(n + i); \
TYPE mm = *(TYPE *)(m + i); \
TYPE dd = MIN(MAX(aa, nn), mm); \
*(TYPE *)(d + i) = dd; \
} \
clear_tail(d, opr_sz, simd_maxsz(desc)); \
}
DO_CLAMP(gvec_sclamp_b, int8_t)
DO_CLAMP(gvec_sclamp_h, int16_t)
DO_CLAMP(gvec_sclamp_s, int32_t)
DO_CLAMP(gvec_sclamp_d, int64_t)
DO_CLAMP(gvec_uclamp_b, uint8_t)
DO_CLAMP(gvec_uclamp_h, uint16_t)
DO_CLAMP(gvec_uclamp_s, uint32_t)
DO_CLAMP(gvec_uclamp_d, uint64_t)