mirrors/qemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

target/arm: Implement SVE2 complex integer dot product

Browse Source 
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
Message-id: 20210525010358.152808-64-richard.henderson@linaro.org
Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Richard Henderson 2021-05-24 18:03:29 -07:00 committed by Peter Maydell
parent 3b787ed808
commit 21068f3972
4 changed files with 135 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
target/arm/helper-sve.h
View File

@ -2724,3 +2724,13 @@ DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_h, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_sqrdcmlah_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cdot_zzzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cdot_zzzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cdot_idx_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_cdot_idx_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)

9
target/arm/sve.decode
View File

@ -789,6 +789,9 @@ MUL_zzi 00100101 .. 110 000 110 ........ ..... @rdn_i8s
DOT_zzzz 01000100 1 sz:1 0 rm:5 00000 u:1 rn:5 rd:5 \
ra=%reg_movprfx
# SVE2 complex dot product (vectors)
CDOT_zzzz 01000100 esz:2 0 rm:5 0001 rot:2 rn:5 rd:5 ra=%reg_movprfx
#### SVE Multiply - Indexed
# SVE integer dot product (indexed)
@ -823,6 +826,12 @@ SQDMLSLB_zzxw_d 01000100 11 1 ..... 0011.0 ..... ..... @rrxr_2a esz=3
SQDMLSLT_zzxw_s 01000100 10 1 ..... 0011.1 ..... ..... @rrxr_3a esz=2
SQDMLSLT_zzxw_d 01000100 11 1 ..... 0011.1 ..... ..... @rrxr_2a esz=3
# SVE2 complex integer dot product (indexed)
CDOT_zzxw_s 01000100 10 1 index:2 rm:3 0100 rot:2 rn:5 rd:5 \
ra=%reg_movprfx
CDOT_zzxw_d 01000100 11 1 index:1 rm:4 0100 rot:2 rn:5 rd:5 \
ra=%reg_movprfx
# SVE2 complex integer multiply-add (indexed)
CMLA_zzxz_h 01000100 10 1 index:2 rm:3 0110 rot:2 rn:5 rd:5 \
ra=%reg_movprfx

99
target/arm/sve_helper.c
View File

@ -1527,6 +1527,105 @@ DO_CMLA_IDX_FUNC(sve2_sqrdcmlah_idx_s, int32_t, H4, DO_SQRDMLAH_S)
#undef DO_SQRDMLAH_S
#undef DO_SQRDMLAH_D
/* Note N and M are 4 elements bundled into one unit. */
static int32_t do_cdot_s(uint32_t n, uint32_t m, int32_t a,
int sel_a, int sel_b, int sub_i)
{
for (int i = 0; i <= 1; i++) {
int32_t elt1_r = (int8_t)(n >> (16 * i));
int32_t elt1_i = (int8_t)(n >> (16 * i + 8));
int32_t elt2_a = (int8_t)(m >> (16 * i + 8 * sel_a));
int32_t elt2_b = (int8_t)(m >> (16 * i + 8 * sel_b));
a += elt1_r * elt2_a + elt1_i * elt2_b * sub_i;
}
return a;
}
static int64_t do_cdot_d(uint64_t n, uint64_t m, int64_t a,
int sel_a, int sel_b, int sub_i)
{
for (int i = 0; i <= 1; i++) {
int64_t elt1_r = (int16_t)(n >> (32 * i + 0));
int64_t elt1_i = (int16_t)(n >> (32 * i + 16));
int64_t elt2_a = (int16_t)(m >> (32 * i + 16 * sel_a));
int64_t elt2_b = (int16_t)(m >> (32 * i + 16 * sel_b));
a += elt1_r * elt2_a + elt1_i * elt2_b * sub_i;
}
return a;
}
void HELPER(sve2_cdot_zzzz_s)(void *vd, void *vn, void *vm,
void *va, uint32_t desc)
{
int opr_sz = simd_oprsz(desc);
int rot = simd_data(desc);
int sel_a = rot & 1;
int sel_b = sel_a ^ 1;
int sub_i = (rot == 0 || rot == 3 ? -1 : 1);
uint32_t *d = vd, *n = vn, *m = vm, *a = va;
for (int e = 0; e < opr_sz / 4; e++) {
d[e] = do_cdot_s(n[e], m[e], a[e], sel_a, sel_b, sub_i);
}
}
void HELPER(sve2_cdot_zzzz_d)(void *vd, void *vn, void *vm,
void *va, uint32_t desc)
{
int opr_sz = simd_oprsz(desc);
int rot = simd_data(desc);
int sel_a = rot & 1;
int sel_b = sel_a ^ 1;
int sub_i = (rot == 0 || rot == 3 ? -1 : 1);
uint64_t *d = vd, *n = vn, *m = vm, *a = va;
for (int e = 0; e < opr_sz / 8; e++) {
d[e] = do_cdot_d(n[e], m[e], a[e], sel_a, sel_b, sub_i);
}
}
void HELPER(sve2_cdot_idx_s)(void *vd, void *vn, void *vm,
void *va, uint32_t desc)
{
int opr_sz = simd_oprsz(desc);
int rot = extract32(desc, SIMD_DATA_SHIFT, 2);
int idx = H4(extract32(desc, SIMD_DATA_SHIFT + 2, 2));
int sel_a = rot & 1;
int sel_b = sel_a ^ 1;
int sub_i = (rot == 0 || rot == 3 ? -1 : 1);
uint32_t *d = vd, *n = vn, *m = vm, *a = va;
for (int seg = 0; seg < opr_sz / 4; seg += 4) {
uint32_t seg_m = m[seg + idx];
for (int e = 0; e < 4; e++) {
d[seg + e] = do_cdot_s(n[seg + e], seg_m, a[seg + e],
sel_a, sel_b, sub_i);
}
}
}
void HELPER(sve2_cdot_idx_d)(void *vd, void *vn, void *vm,
void *va, uint32_t desc)
{
int seg, opr_sz = simd_oprsz(desc);
int rot = extract32(desc, SIMD_DATA_SHIFT, 2);
int idx = extract32(desc, SIMD_DATA_SHIFT + 2, 2);
int sel_a = rot & 1;
int sel_b = sel_a ^ 1;
int sub_i = (rot == 0 || rot == 3 ? -1 : 1);
uint64_t *d = vd, *n = vn, *m = vm, *a = va;
for (seg = 0; seg < opr_sz / 8; seg += 2) {
uint64_t seg_m = m[seg + idx];
for (int e = 0; e < 2; e++) {
d[seg + e] = do_cdot_d(n[seg + e], seg_m, a[seg + e],
sel_a, sel_b, sub_i);
}
}
}
#define DO_ZZXZ(NAME, TYPE, H, OP) \
void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
{ \

17
target/arm/translate-sve.c
View File

@ -3989,6 +3989,9 @@ DO_SVE2_RRXR_ROT(CMLA_zzxz_s, gen_helper_sve2_cmla_idx_s)
DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h, gen_helper_sve2_sqrdcmlah_idx_h)
DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s, gen_helper_sve2_sqrdcmlah_idx_s)
DO_SVE2_RRXR_ROT(CDOT_zzxw_s, gen_helper_sve2_cdot_idx_s)
DO_SVE2_RRXR_ROT(CDOT_zzxw_d, gen_helper_sve2_cdot_idx_d)
#undef DO_SVE2_RRXR_ROT
/*
@ -8084,6 +8087,20 @@ static bool trans_CMLA_zzzz(DisasContext *s, arg_CMLA_zzzz *a)
return true;
}
static bool trans_CDOT_zzzz(DisasContext *s, arg_CMLA_zzzz *a)
{
if (!dc_isar_feature(aa64_sve2, s) || a->esz < MO_32) {
return false;
}
if (sve_access_check(s)) {
gen_helper_gvec_4 *fn = (a->esz == MO_32
? gen_helper_sve2_cdot_zzzz_s
: gen_helper_sve2_cdot_zzzz_d);
gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->rot);
}
return true;
}
static bool trans_SQRDCMLAH_zzzz(DisasContext *s, arg_SQRDCMLAH_zzzz *a)
{
static gen_helper_gvec_4 * const fns[] = {