mirrors/qemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

include/host: Split out atomic128-ldst.h

Browse Source 
Separates the aarch64-specific portion into its own file.

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
This commit is contained in:
Richard Henderson 2023-05-18 08:37:08 -07:00
parent 412db3d5b5
commit 154691335e
3 changed files with 107 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
host/include/aarch64/host/atomic128-ldst.h Normal file
View File

@ -0,0 +1,49 @@
/*
* SPDX-License-Identifier: GPL-2.0-or-later
* Load/store for 128-bit atomic operations, AArch64 version.
*
* Copyright (C) 2018, 2023 Linaro, Ltd.
*
* See docs/devel/atomics.rst for discussion about the guarantees each
* atomic primitive is meant to provide.
*/
#ifndef AARCH64_ATOMIC128_LDST_H
#define AARCH64_ATOMIC128_LDST_H
/* Through gcc 10, aarch64 has no support for 128-bit atomics. */
#if !defined(CONFIG_ATOMIC128) && !defined(CONFIG_USER_ONLY)
/* We can do better than cmpxchg for AArch64. */
static inline Int128 atomic16_read(Int128 *ptr)
{
uint64_t l, h;
uint32_t tmp;
/* The load must be paired with the store to guarantee not tearing. */
asm("0: ldxp %[l], %[h], %[mem]\n\t"
"stxp %w[tmp], %[l], %[h], %[mem]\n\t"
"cbnz %w[tmp], 0b"
: [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h));
return int128_make128(l, h);
}
static inline void atomic16_set(Int128 *ptr, Int128 val)
{
uint64_t l = int128_getlo(val), h = int128_gethi(val);
uint64_t t1, t2;
/* Load into temporaries to acquire the exclusive access lock. */
asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
"stxp %w[t1], %[l], %[h], %[mem]\n\t"
"cbnz %w[t1], 0b"
: [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
: [l] "r"(l), [h] "r"(h));
}
# define HAVE_ATOMIC128 1
#else
#include "host/include/generic/host/atomic128-ldst.h"
#endif
#endif /* AARCH64_ATOMIC128_LDST_H */

57
host/include/generic/host/atomic128-ldst.h Normal file
View File

@ -0,0 +1,57 @@
/*
* SPDX-License-Identifier: GPL-2.0-or-later
* Load/store for 128-bit atomic operations, generic version.
*
* Copyright (C) 2018, 2023 Linaro, Ltd.
*
* See docs/devel/atomics.rst for discussion about the guarantees each
* atomic primitive is meant to provide.
*/
#ifndef HOST_ATOMIC128_LDST_H
#define HOST_ATOMIC128_LDST_H
#if defined(CONFIG_ATOMIC128)
static inline Int128 atomic16_read(Int128 *ptr)
{
Int128Alias r;
r.i = qatomic_read__nocheck((__int128_t *)ptr);
return r.s;
}
static inline void atomic16_set(Int128 *ptr, Int128 val)
{
Int128Alias v;
v.s = val;
qatomic_set__nocheck((__int128_t *)ptr, v.i);
}
# define HAVE_ATOMIC128 1
#elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128
static inline Int128 atomic16_read(Int128 *ptr)
{
/* Maybe replace 0 with 0, returning the old value. */
Int128 z = int128_make64(0);
return atomic16_cmpxchg(ptr, z, z);
}
static inline void atomic16_set(Int128 *ptr, Int128 val)
{
Int128 old = *ptr, cmp;
do {
cmp = old;
old = atomic16_cmpxchg(ptr, cmp, val);
} while (int128_ne(old, cmp));
}
# define HAVE_ATOMIC128 1
#else
/* Fallback definitions that must be optimized away, or error. */
Int128 QEMU_ERROR("unsupported atomic") atomic16_read(Int128 *ptr);
void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val);
# define HAVE_ATOMIC128 0
#endif
#endif /* HOST_ATOMIC128_LDST_H */

74
include/qemu/atomic128.h
View File

@ -42,78 +42,6 @@
*/
#include "host/atomic128-cas.h"
#if defined(CONFIG_ATOMIC128)
static inline Int128 atomic16_read(Int128 *ptr)
{
Int128Alias r;
r.i = qatomic_read__nocheck((__int128_t *)ptr);
return r.s;
}
static inline void atomic16_set(Int128 *ptr, Int128 val)
{
Int128Alias v;
v.s = val;
qatomic_set__nocheck((__int128_t *)ptr, v.i);
}
# define HAVE_ATOMIC128 1
#elif !defined(CONFIG_USER_ONLY) && defined(__aarch64__)
/* We can do better than cmpxchg for AArch64. */
static inline Int128 atomic16_read(Int128 *ptr)
{
uint64_t l, h;
uint32_t tmp;
/* The load must be paired with the store to guarantee not tearing. */
asm("0: ldxp %[l], %[h], %[mem]\n\t"
"stxp %w[tmp], %[l], %[h], %[mem]\n\t"
"cbnz %w[tmp], 0b"
: [mem] "+m"(*ptr), [tmp] "=r"(tmp), [l] "=r"(l), [h] "=r"(h));
return int128_make128(l, h);
}
static inline void atomic16_set(Int128 *ptr, Int128 val)
{
uint64_t l = int128_getlo(val), h = int128_gethi(val);
uint64_t t1, t2;
/* Load into temporaries to acquire the exclusive access lock. */
asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
"stxp %w[t1], %[l], %[h], %[mem]\n\t"
"cbnz %w[t1], 0b"
: [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
: [l] "r"(l), [h] "r"(h));
}
# define HAVE_ATOMIC128 1
#elif !defined(CONFIG_USER_ONLY) && HAVE_CMPXCHG128
static inline Int128 atomic16_read(Int128 *ptr)
{
/* Maybe replace 0 with 0, returning the old value. */
Int128 z = int128_make64(0);
return atomic16_cmpxchg(ptr, z, z);
}
static inline void atomic16_set(Int128 *ptr, Int128 val)
{
Int128 old = *ptr, cmp;
do {
cmp = old;
old = atomic16_cmpxchg(ptr, cmp, val);
} while (int128_ne(old, cmp));
}
# define HAVE_ATOMIC128 1
#else
/* Fallback definitions that must be optimized away, or error. */
Int128 QEMU_ERROR("unsupported atomic") atomic16_read(Int128 *ptr);
void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val);
# define HAVE_ATOMIC128 0
#endif /* Some definition for HAVE_ATOMIC128 */
#include "host/atomic128-ldst.h"
#endif /* QEMU_ATOMIC128_H */