mirrors/qemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
4baf3978c0
qemu/accel/tcg/ldst_atomicity.c.inc
Richard Henderson e61f1efeb7 meson: Detect atomic128 support with optimization 
There is an edge condition prior to gcc13 for which optimization
is required to generate 16-byte atomic sequences.  Detect this.

Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
2023-05-16 15:21:39 -07:00

1263 lines
31 KiB
C++
Raw Blame History

/*
* Routines common to user and system emulation of load/store.
*
* Copyright (c) 2022 Linaro, Ltd.
*
* SPDX-License-Identifier: GPL-2.0-or-later
*
* This work is licensed under the terms of the GNU GPL, version 2 or later.
* See the COPYING file in the top-level directory.
*/
#ifdef CONFIG_ATOMIC64
# define HAVE_al8 true
#else
# define HAVE_al8 false
#endif
#define HAVE_al8_fast (ATOMIC_REG_SIZE >= 8)
/*
* If __alignof(unsigned __int128) < 16, GCC may refuse to inline atomics
* that are supported by the host, e.g. s390x. We can force the pointer to
* have our known alignment with __builtin_assume_aligned, however prior to
* GCC 13 that was only reliable with optimization enabled. See
* https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107389
*/
#if defined(CONFIG_ATOMIC128_OPT)
# if !defined(__OPTIMIZE__)
# define ATTRIBUTE_ATOMIC128_OPT __attribute__((optimize("O1")))
# endif
# define CONFIG_ATOMIC128
#endif
#ifndef ATTRIBUTE_ATOMIC128_OPT
# define ATTRIBUTE_ATOMIC128_OPT
#endif
#if defined(CONFIG_ATOMIC128)
# define HAVE_al16_fast true
#else
# define HAVE_al16_fast false
#endif
#if defined(CONFIG_ATOMIC128) || defined(CONFIG_CMPXCHG128)
# define HAVE_al16 true
#else
# define HAVE_al16 false
#endif
/**
* required_atomicity:
*
* Return the lg2 bytes of atomicity required by @memop for @p.
* If the operation must be split into two operations to be
* examined separately for atomicity, return -lg2.
*/
static int required_atomicity(CPUArchState *env, uintptr_t p, MemOp memop)
{
MemOp atom = memop & MO_ATOM_MASK;
MemOp size = memop & MO_SIZE;
MemOp half = size ? size - 1 : 0;
unsigned tmp;
int atmax;
switch (atom) {
case MO_ATOM_NONE:
atmax = MO_8;
break;
case MO_ATOM_IFALIGN_PAIR:
size = half;
/* fall through */
case MO_ATOM_IFALIGN:
tmp = (1 << size) - 1;
atmax = p & tmp ? MO_8 : size;
break;
case MO_ATOM_WITHIN16:
tmp = p & 15;
atmax = (tmp + (1 << size) <= 16 ? size : MO_8);
break;
case MO_ATOM_WITHIN16_PAIR:
tmp = p & 15;
if (tmp + (1 << size) <= 16) {
atmax = size;
} else if (tmp + (1 << half) == 16) {
/*
* The pair exactly straddles the boundary.
* Both halves are naturally aligned and atomic.
*/
atmax = half;
} else {
/*
* One of the pair crosses the boundary, and is non-atomic.
* The other of the pair does not cross, and is atomic.
*/
atmax = -half;
}
break;
case MO_ATOM_SUBALIGN:
/*
* Examine the alignment of p to determine if there are subobjects
* that must be aligned. Note that we only really need ctz4() --
* any more sigificant bits are discarded by the immediately
* following comparison.
*/
tmp = ctz32(p);
atmax = MIN(size, tmp);
break;
default:
g_assert_not_reached();
}
/*
* Here we have the architectural atomicity of the operation.
* However, when executing in a serial context, we need no extra
* host atomicity in order to avoid racing. This reduction
* avoids looping with cpu_loop_exit_atomic.
*/
if (cpu_in_serial_context(env_cpu(env))) {
return MO_8;
}
return atmax;
}
/**
* load_atomic2:
* @pv: host address
*
* Atomically load 2 aligned bytes from @pv.
*/
static inline uint16_t load_atomic2(void *pv)
{
uint16_t *p = __builtin_assume_aligned(pv, 2);
return qatomic_read(p);
}
/**
* load_atomic4:
* @pv: host address
*
* Atomically load 4 aligned bytes from @pv.
*/
static inline uint32_t load_atomic4(void *pv)
{
uint32_t *p = __builtin_assume_aligned(pv, 4);
return qatomic_read(p);
}
/**
* load_atomic8:
* @pv: host address
*
* Atomically load 8 aligned bytes from @pv.
*/
static inline uint64_t load_atomic8(void *pv)
{
uint64_t *p = __builtin_assume_aligned(pv, 8);
qemu_build_assert(HAVE_al8);
return qatomic_read__nocheck(p);
}
/**
* load_atomic16:
* @pv: host address
*
* Atomically load 16 aligned bytes from @pv.
*/
static inline Int128 ATTRIBUTE_ATOMIC128_OPT
load_atomic16(void *pv)
{
#ifdef CONFIG_ATOMIC128
__uint128_t *p = __builtin_assume_aligned(pv, 16);
Int128Alias r;
r.u = qatomic_read__nocheck(p);
return r.s;
#else
qemu_build_not_reached();
#endif
}
/**
* load_atomic8_or_exit:
* @env: cpu context
* @ra: host unwind address
* @pv: host address
*
* Atomically load 8 aligned bytes from @pv.
* If this is not possible, longjmp out to restart serially.
*/
static uint64_t load_atomic8_or_exit(CPUArchState *env, uintptr_t ra, void *pv)
{
if (HAVE_al8) {
return load_atomic8(pv);
}
#ifdef CONFIG_USER_ONLY
/*
* If the page is not writable, then assume the value is immutable
* and requires no locking. This ignores the case of MAP_SHARED with
* another process, because the fallback start_exclusive solution
* provides no protection across processes.
*/
if (!page_check_range(h2g(pv), 8, PAGE_WRITE)) {
uint64_t *p = __builtin_assume_aligned(pv, 8);
return *p;
}
#endif
/* Ultimate fallback: re-execute in serial context. */
cpu_loop_exit_atomic(env_cpu(env), ra);
}
/**
* load_atomic16_or_exit:
* @env: cpu context
* @ra: host unwind address
* @pv: host address
*
* Atomically load 16 aligned bytes from @pv.
* If this is not possible, longjmp out to restart serially.
*/
static Int128 load_atomic16_or_exit(CPUArchState *env, uintptr_t ra, void *pv)
{
Int128 *p = __builtin_assume_aligned(pv, 16);
if (HAVE_al16_fast) {
return load_atomic16(p);
}
#ifdef CONFIG_USER_ONLY
/*
* We can only use cmpxchg to emulate a load if the page is writable.
* If the page is not writable, then assume the value is immutable
* and requires no locking. This ignores the case of MAP_SHARED with
* another process, because the fallback start_exclusive solution
* provides no protection across processes.
*/
if (!page_check_range(h2g(p), 16, PAGE_WRITE)) {
return *p;
}
#endif
/*
* In system mode all guest pages are writable, and for user-only
* we have just checked writability. Try cmpxchg.
*/
#if defined(CONFIG_CMPXCHG128)
/* Swap 0 with 0, with the side-effect of returning the old value. */
{
Int128Alias r;
r.u = __sync_val_compare_and_swap_16((__uint128_t *)p, 0, 0);
return r.s;
}
#endif
/* Ultimate fallback: re-execute in serial context. */
cpu_loop_exit_atomic(env_cpu(env), ra);
}
/**
* load_atom_extract_al4x2:
* @pv: host address
*
* Load 4 bytes from @p, from two sequential atomic 4-byte loads.
*/
static uint32_t load_atom_extract_al4x2(void *pv)
{
uintptr_t pi = (uintptr_t)pv;
int sh = (pi & 3) * 8;
uint32_t a, b;
pv = (void *)(pi & ~3);
a = load_atomic4(pv);
b = load_atomic4(pv + 4);
if (HOST_BIG_ENDIAN) {
return (a << sh) | (b >> (-sh & 31));
} else {
return (a >> sh) | (b << (-sh & 31));
}
}
/**
* load_atom_extract_al8x2:
* @pv: host address
*
* Load 8 bytes from @p, from two sequential atomic 8-byte loads.
*/
static uint64_t load_atom_extract_al8x2(void *pv)
{
uintptr_t pi = (uintptr_t)pv;
int sh = (pi & 7) * 8;
uint64_t a, b;
pv = (void *)(pi & ~7);
a = load_atomic8(pv);
b = load_atomic8(pv + 8);
if (HOST_BIG_ENDIAN) {
return (a << sh) | (b >> (-sh & 63));
} else {
return (a >> sh) | (b << (-sh & 63));
}
}
/**
* load_atom_extract_al8_or_exit:
* @env: cpu context
* @ra: host unwind address
* @pv: host address
* @s: object size in bytes, @s <= 4.
*
* Atomically load @s bytes from @p, when p % s != 0, and [p, p+s-1] does
* not cross an 8-byte boundary. This means that we can perform an atomic
* 8-byte load and extract.
* The value is returned in the low bits of a uint32_t.
*/
static uint32_t load_atom_extract_al8_or_exit(CPUArchState *env, uintptr_t ra,
void *pv, int s)
{
uintptr_t pi = (uintptr_t)pv;
int o = pi & 7;
int shr = (HOST_BIG_ENDIAN ? 8 - s - o : o) * 8;
pv = (void *)(pi & ~7);
return load_atomic8_or_exit(env, ra, pv) >> shr;
}
/**
* load_atom_extract_al16_or_exit:
* @env: cpu context
* @ra: host unwind address
* @p: host address
* @s: object size in bytes, @s <= 8.
*
* Atomically load @s bytes from @p, when p % 16 < 8
* and p % 16 + s > 8. I.e. does not cross a 16-byte
* boundary, but *does* cross an 8-byte boundary.
* This is the slow version, so we must have eliminated
* any faster load_atom_extract_al8_or_exit case.
*
* If this is not possible, longjmp out to restart serially.
*/
static uint64_t load_atom_extract_al16_or_exit(CPUArchState *env, uintptr_t ra,
void *pv, int s)
{
uintptr_t pi = (uintptr_t)pv;
int o = pi & 7;
int shr = (HOST_BIG_ENDIAN ? 16 - s - o : o) * 8;
Int128 r;
/*
* Note constraints above: p & 8 must be clear.
* Provoke SIGBUS if possible otherwise.
*/
pv = (void *)(pi & ~7);
r = load_atomic16_or_exit(env, ra, pv);
r = int128_urshift(r, shr);
return int128_getlo(r);
}
/**
* load_atom_extract_al16_or_al8:
* @p: host address
* @s: object size in bytes, @s <= 8.
*
* Load @s bytes from @p, when p % s != 0. If [p, p+s-1] does not
* cross an 16-byte boundary then the access must be 16-byte atomic,
* otherwise the access must be 8-byte atomic.
*/
static inline uint64_t ATTRIBUTE_ATOMIC128_OPT
load_atom_extract_al16_or_al8(void *pv, int s)
{
#if defined(CONFIG_ATOMIC128)
uintptr_t pi = (uintptr_t)pv;
int o = pi & 7;
int shr = (HOST_BIG_ENDIAN ? 16 - s - o : o) * 8;
__uint128_t r;
pv = (void *)(pi & ~7);
if (pi & 8) {
uint64_t *p8 = __builtin_assume_aligned(pv, 16, 8);
uint64_t a = qatomic_read__nocheck(p8);
uint64_t b = qatomic_read__nocheck(p8 + 1);
if (HOST_BIG_ENDIAN) {
r = ((__uint128_t)a << 64) | b;
} else {
r = ((__uint128_t)b << 64) | a;
}
} else {
__uint128_t *p16 = __builtin_assume_aligned(pv, 16, 0);
r = qatomic_read__nocheck(p16);
}
return r >> shr;
#else
qemu_build_not_reached();
#endif
}
/**
* load_atom_4_by_2:
* @pv: host address
*
* Load 4 bytes from @pv, with two 2-byte atomic loads.
*/
static inline uint32_t load_atom_4_by_2(void *pv)
{
uint32_t a = load_atomic2(pv);
uint32_t b = load_atomic2(pv + 2);
if (HOST_BIG_ENDIAN) {
return (a << 16) | b;
} else {
return (b << 16) | a;
}
}
/**
* load_atom_8_by_2:
* @pv: host address
*
* Load 8 bytes from @pv, with four 2-byte atomic loads.
*/
static inline uint64_t load_atom_8_by_2(void *pv)
{
uint32_t a = load_atom_4_by_2(pv);
uint32_t b = load_atom_4_by_2(pv + 4);
if (HOST_BIG_ENDIAN) {
return ((uint64_t)a << 32) | b;
} else {
return ((uint64_t)b << 32) | a;
}
}
/**
* load_atom_8_by_4:
* @pv: host address
*
* Load 8 bytes from @pv, with two 4-byte atomic loads.
*/
static inline uint64_t load_atom_8_by_4(void *pv)
{
uint32_t a = load_atomic4(pv);
uint32_t b = load_atomic4(pv + 4);
if (HOST_BIG_ENDIAN) {
return ((uint64_t)a << 32) | b;
} else {
return ((uint64_t)b << 32) | a;
}
}
/**
* load_atom_8_by_8_or_4:
* @pv: host address
*
* Load 8 bytes from aligned @pv, with at least 4-byte atomicity.
*/
static inline uint64_t load_atom_8_by_8_or_4(void *pv)
{
if (HAVE_al8_fast) {
return load_atomic8(pv);
} else {
return load_atom_8_by_4(pv);
}
}
/**
* load_atom_2:
* @p: host address
* @memop: the full memory op
*
* Load 2 bytes from @p, honoring the atomicity of @memop.
*/
static uint16_t load_atom_2(CPUArchState *env, uintptr_t ra,
void *pv, MemOp memop)
{
uintptr_t pi = (uintptr_t)pv;
int atmax;
if (likely((pi & 1) == 0)) {
return load_atomic2(pv);
}
if (HAVE_al16_fast) {
return load_atom_extract_al16_or_al8(pv, 2);
}
atmax = required_atomicity(env, pi, memop);
switch (atmax) {
case MO_8:
return lduw_he_p(pv);
case MO_16:
/* The only case remaining is MO_ATOM_WITHIN16. */
if (!HAVE_al8_fast && (pi & 3) == 1) {
/* Big or little endian, we want the middle two bytes. */
return load_atomic4(pv - 1) >> 8;
}
if ((pi & 15) != 7) {
return load_atom_extract_al8_or_exit(env, ra, pv, 2);
}
return load_atom_extract_al16_or_exit(env, ra, pv, 2);
default:
g_assert_not_reached();
}
}
/**
* load_atom_4:
* @p: host address
* @memop: the full memory op
*
* Load 4 bytes from @p, honoring the atomicity of @memop.
*/
static uint32_t load_atom_4(CPUArchState *env, uintptr_t ra,
void *pv, MemOp memop)
{
uintptr_t pi = (uintptr_t)pv;
int atmax;
if (likely((pi & 3) == 0)) {
return load_atomic4(pv);
}
if (HAVE_al16_fast) {
return load_atom_extract_al16_or_al8(pv, 4);
}
atmax = required_atomicity(env, pi, memop);
switch (atmax) {
case MO_8:
case MO_16:
case -MO_16:
/*
* For MO_ATOM_IFALIGN, this is more atomicity than required,
* but it's trivially supported on all hosts, better than 4
* individual byte loads (when the host requires alignment),
* and overlaps with the MO_ATOM_SUBALIGN case of p % 2 == 0.
*/
return load_atom_extract_al4x2(pv);
case MO_32:
if (!(pi & 4)) {
return load_atom_extract_al8_or_exit(env, ra, pv, 4);
}
return load_atom_extract_al16_or_exit(env, ra, pv, 4);
default:
g_assert_not_reached();
}
}
/**
* load_atom_8:
* @p: host address
* @memop: the full memory op
*
* Load 8 bytes from @p, honoring the atomicity of @memop.
*/
static uint64_t load_atom_8(CPUArchState *env, uintptr_t ra,
void *pv, MemOp memop)
{
uintptr_t pi = (uintptr_t)pv;
int atmax;
/*
* If the host does not support 8-byte atomics, wait until we have
* examined the atomicity parameters below.
*/
if (HAVE_al8 && likely((pi & 7) == 0)) {
return load_atomic8(pv);
}
if (HAVE_al16_fast) {
return load_atom_extract_al16_or_al8(pv, 8);
}
atmax = required_atomicity(env, pi, memop);
if (atmax == MO_64) {
if (!HAVE_al8 && (pi & 7) == 0) {
load_atomic8_or_exit(env, ra, pv);
}
return load_atom_extract_al16_or_exit(env, ra, pv, 8);
}
if (HAVE_al8_fast) {
return load_atom_extract_al8x2(pv);
}
switch (atmax) {
case MO_8:
return ldq_he_p(pv);
case MO_16:
return load_atom_8_by_2(pv);
case MO_32:
return load_atom_8_by_4(pv);
case -MO_32:
if (HAVE_al8) {
return load_atom_extract_al8x2(pv);
}
cpu_loop_exit_atomic(env_cpu(env), ra);
default:
g_assert_not_reached();
}
}
/**
* load_atom_16:
* @p: host address
* @memop: the full memory op
*
* Load 16 bytes from @p, honoring the atomicity of @memop.
*/
static Int128 load_atom_16(CPUArchState *env, uintptr_t ra,
void *pv, MemOp memop)
{
uintptr_t pi = (uintptr_t)pv;
int atmax;
Int128 r;
uint64_t a, b;
/*
* If the host does not support 16-byte atomics, wait until we have
* examined the atomicity parameters below.
*/
if (HAVE_al16_fast && likely((pi & 15) == 0)) {
return load_atomic16(pv);
}
atmax = required_atomicity(env, pi, memop);
switch (atmax) {
case MO_8:
memcpy(&r, pv, 16);
return r;
case MO_16:
a = load_atom_8_by_2(pv);
b = load_atom_8_by_2(pv + 8);
break;
case MO_32:
a = load_atom_8_by_4(pv);
b = load_atom_8_by_4(pv + 8);
break;
case MO_64:
if (!HAVE_al8) {
cpu_loop_exit_atomic(env_cpu(env), ra);
}
a = load_atomic8(pv);
b = load_atomic8(pv + 8);
break;
case -MO_64:
if (!HAVE_al8) {
cpu_loop_exit_atomic(env_cpu(env), ra);
}
a = load_atom_extract_al8x2(pv);
b = load_atom_extract_al8x2(pv + 8);
break;
case MO_128:
return load_atomic16_or_exit(env, ra, pv);
default:
g_assert_not_reached();
}
return int128_make128(HOST_BIG_ENDIAN ? b : a, HOST_BIG_ENDIAN ? a : b);
}
/**
* store_atomic2:
* @pv: host address
* @val: value to store
*
* Atomically store 2 aligned bytes to @pv.
*/
static inline void store_atomic2(void *pv, uint16_t val)
{
uint16_t *p = __builtin_assume_aligned(pv, 2);
qatomic_set(p, val);
}
/**
* store_atomic4:
* @pv: host address
* @val: value to store
*
* Atomically store 4 aligned bytes to @pv.
*/
static inline void store_atomic4(void *pv, uint32_t val)
{
uint32_t *p = __builtin_assume_aligned(pv, 4);
qatomic_set(p, val);
}
/**
* store_atomic8:
* @pv: host address
* @val: value to store
*
* Atomically store 8 aligned bytes to @pv.
*/
static inline void store_atomic8(void *pv, uint64_t val)
{
uint64_t *p = __builtin_assume_aligned(pv, 8);
qemu_build_assert(HAVE_al8);
qatomic_set__nocheck(p, val);
}
/**
* store_atomic16:
* @pv: host address
* @val: value to store
*
* Atomically store 16 aligned bytes to @pv.
*/
static inline void ATTRIBUTE_ATOMIC128_OPT
store_atomic16(void *pv, Int128Alias val)
{
#if defined(CONFIG_ATOMIC128)
__uint128_t *pu = __builtin_assume_aligned(pv, 16);
qatomic_set__nocheck(pu, val.u);
#elif defined(CONFIG_CMPXCHG128)
__uint128_t *pu = __builtin_assume_aligned(pv, 16);
__uint128_t o;
/*
* Without CONFIG_ATOMIC128, __atomic_compare_exchange_n will always
* defer to libatomic, so we must use __sync_*_compare_and_swap_16
* and accept the sequential consistency that comes with it.
*/
do {
o = *pu;
} while (!__sync_bool_compare_and_swap_16(pu, o, val.u));
#else
qemu_build_not_reached();
#endif
}
/**
* store_atom_4x2
*/
static inline void store_atom_4_by_2(void *pv, uint32_t val)
{
store_atomic2(pv, val >> (HOST_BIG_ENDIAN ? 16 : 0));
store_atomic2(pv + 2, val >> (HOST_BIG_ENDIAN ? 0 : 16));
}
/**
* store_atom_8_by_2
*/
static inline void store_atom_8_by_2(void *pv, uint64_t val)
{
store_atom_4_by_2(pv, val >> (HOST_BIG_ENDIAN ? 32 : 0));
store_atom_4_by_2(pv + 4, val >> (HOST_BIG_ENDIAN ? 0 : 32));
}
/**
* store_atom_8_by_4
*/
static inline void store_atom_8_by_4(void *pv, uint64_t val)
{
store_atomic4(pv, val >> (HOST_BIG_ENDIAN ? 32 : 0));
store_atomic4(pv + 4, val >> (HOST_BIG_ENDIAN ? 0 : 32));
}
/**
* store_atom_insert_al4:
* @p: host address
* @val: shifted value to store
* @msk: mask for value to store
*
* Atomically store @val to @p, masked by @msk.
*/
static void store_atom_insert_al4(uint32_t *p, uint32_t val, uint32_t msk)
{
uint32_t old, new;
p = __builtin_assume_aligned(p, 4);
old = qatomic_read(p);
do {
new = (old & ~msk) | val;
} while (!__atomic_compare_exchange_n(p, &old, new, true,
__ATOMIC_RELAXED, __ATOMIC_RELAXED));
}
/**
* store_atom_insert_al8:
* @p: host address
* @val: shifted value to store
* @msk: mask for value to store
*
* Atomically store @val to @p masked by @msk.
*/
static void store_atom_insert_al8(uint64_t *p, uint64_t val, uint64_t msk)
{
uint64_t old, new;
qemu_build_assert(HAVE_al8);
p = __builtin_assume_aligned(p, 8);
old = qatomic_read__nocheck(p);
do {
new = (old & ~msk) | val;
} while (!__atomic_compare_exchange_n(p, &old, new, true,
__ATOMIC_RELAXED, __ATOMIC_RELAXED));
}
/**
* store_atom_insert_al16:
* @p: host address
* @val: shifted value to store
* @msk: mask for value to store
*
* Atomically store @val to @p masked by @msk.
*/
static void ATTRIBUTE_ATOMIC128_OPT
store_atom_insert_al16(Int128 *ps, Int128Alias val, Int128Alias msk)
{
#if defined(CONFIG_ATOMIC128)
__uint128_t *pu, old, new;
/* With CONFIG_ATOMIC128, we can avoid the memory barriers. */
pu = __builtin_assume_aligned(ps, 16);
old = *pu;
do {
new = (old & ~msk.u) | val.u;
} while (!__atomic_compare_exchange_n(pu, &old, new, true,
__ATOMIC_RELAXED, __ATOMIC_RELAXED));
#elif defined(CONFIG_CMPXCHG128)
__uint128_t *pu, old, new;
/*
* Without CONFIG_ATOMIC128, __atomic_compare_exchange_n will always
* defer to libatomic, so we must use __sync_*_compare_and_swap_16
* and accept the sequential consistency that comes with it.
*/
pu = __builtin_assume_aligned(ps, 16);
do {
old = *pu;
new = (old & ~msk.u) | val.u;
} while (!__sync_bool_compare_and_swap_16(pu, old, new));
#else
qemu_build_not_reached();
#endif
}
/**
* store_bytes_leN:
* @pv: host address
* @size: number of bytes to store
* @val_le: data to store
*
* Store @size bytes at @p. The bytes to store are extracted in little-endian order
* from @val_le; return the bytes of @val_le beyond @size that have not been stored.
*/
static uint64_t store_bytes_leN(void *pv, int size, uint64_t val_le)
{
uint8_t *p = pv;
for (int i = 0; i < size; i++, val_le >>= 8) {
p[i] = val_le;
}
return val_le;
}
/**
* store_parts_leN
* @pv: host address
* @size: number of bytes to store
* @val_le: data to store
*
* As store_bytes_leN, but atomically on each aligned part.
*/
G_GNUC_UNUSED
static uint64_t store_parts_leN(void *pv, int size, uint64_t val_le)
{
do {
int n;
/* Find minimum of alignment and size */
switch (((uintptr_t)pv | size) & 7) {
case 4:
store_atomic4(pv, le32_to_cpu(val_le));
val_le >>= 32;
n = 4;
break;
case 2:
case 6:
store_atomic2(pv, le16_to_cpu(val_le));
val_le >>= 16;
n = 2;
break;
default:
*(uint8_t *)pv = val_le;
val_le >>= 8;
n = 1;
break;
case 0:
g_assert_not_reached();
}
pv += n;
size -= n;
} while (size != 0);
return val_le;
}
/**
* store_whole_le4
* @pv: host address
* @size: number of bytes to store
* @val_le: data to store
*
* As store_bytes_leN, but atomically as a whole.
* Four aligned bytes are guaranteed to cover the store.
*/
static uint64_t store_whole_le4(void *pv, int size, uint64_t val_le)
{
int sz = size * 8;
int o = (uintptr_t)pv & 3;
int sh = o * 8;
uint32_t m = MAKE_64BIT_MASK(0, sz);
uint32_t v;
if (HOST_BIG_ENDIAN) {
v = bswap32(val_le) >> sh;
m = bswap32(m) >> sh;
} else {
v = val_le << sh;
m <<= sh;
}
store_atom_insert_al4(pv - o, v, m);
return val_le >> sz;
}
/**
* store_whole_le8
* @pv: host address
* @size: number of bytes to store
* @val_le: data to store
*
* As store_bytes_leN, but atomically as a whole.
* Eight aligned bytes are guaranteed to cover the store.
*/
static uint64_t store_whole_le8(void *pv, int size, uint64_t val_le)
{
int sz = size * 8;
int o = (uintptr_t)pv & 7;
int sh = o * 8;
uint64_t m = MAKE_64BIT_MASK(0, sz);
uint64_t v;
qemu_build_assert(HAVE_al8);
if (HOST_BIG_ENDIAN) {
v = bswap64(val_le) >> sh;
m = bswap64(m) >> sh;
} else {
v = val_le << sh;
m <<= sh;
}
store_atom_insert_al8(pv - o, v, m);
return val_le >> sz;
}
/**
* store_whole_le16
* @pv: host address
* @size: number of bytes to store
* @val_le: data to store
*
* As store_bytes_leN, but atomically as a whole.
* 16 aligned bytes are guaranteed to cover the store.
*/
static uint64_t store_whole_le16(void *pv, int size, Int128 val_le)
{
int sz = size * 8;
int o = (uintptr_t)pv & 15;
int sh = o * 8;
Int128 m, v;
qemu_build_assert(HAVE_al16);
/* Like MAKE_64BIT_MASK(0, sz), but larger. */
if (sz <= 64) {
m = int128_make64(MAKE_64BIT_MASK(0, sz));
} else {
m = int128_make128(-1, MAKE_64BIT_MASK(0, sz - 64));
}
if (HOST_BIG_ENDIAN) {
v = int128_urshift(bswap128(val_le), sh);
m = int128_urshift(bswap128(m), sh);
} else {
v = int128_lshift(val_le, sh);
m = int128_lshift(m, sh);
}
store_atom_insert_al16(pv - o, v, m);
/* Unused if sz <= 64. */
return int128_gethi(val_le) >> (sz - 64);
}
/**
* store_atom_2:
* @p: host address
* @val: the value to store
* @memop: the full memory op
*
* Store 2 bytes to @p, honoring the atomicity of @memop.
*/
static void store_atom_2(CPUArchState *env, uintptr_t ra,
void *pv, MemOp memop, uint16_t val)
{
uintptr_t pi = (uintptr_t)pv;
int atmax;
if (likely((pi & 1) == 0)) {
store_atomic2(pv, val);
return;
}
atmax = required_atomicity(env, pi, memop);
if (atmax == MO_8) {
stw_he_p(pv, val);
return;
}
/*
* The only case remaining is MO_ATOM_WITHIN16.
* Big or little endian, we want the middle two bytes in each test.
*/
if ((pi & 3) == 1) {
store_atom_insert_al4(pv - 1, (uint32_t)val << 8, MAKE_64BIT_MASK(8, 16));
return;
} else if ((pi & 7) == 3) {
if (HAVE_al8) {
store_atom_insert_al8(pv - 3, (uint64_t)val << 24, MAKE_64BIT_MASK(24, 16));
return;
}
} else if ((pi & 15) == 7) {
if (HAVE_al16) {
Int128 v = int128_lshift(int128_make64(val), 56);
Int128 m = int128_lshift(int128_make64(0xffff), 56);
store_atom_insert_al16(pv - 7, v, m);
return;
}
} else {
g_assert_not_reached();
}
cpu_loop_exit_atomic(env_cpu(env), ra);
}
/**
* store_atom_4:
* @p: host address
* @val: the value to store
* @memop: the full memory op
*
* Store 4 bytes to @p, honoring the atomicity of @memop.
*/
static void store_atom_4(CPUArchState *env, uintptr_t ra,
void *pv, MemOp memop, uint32_t val)
{
uintptr_t pi = (uintptr_t)pv;
int atmax;
if (likely((pi & 3) == 0)) {
store_atomic4(pv, val);
return;
}
atmax = required_atomicity(env, pi, memop);
switch (atmax) {
case MO_8:
stl_he_p(pv, val);
return;
case MO_16:
store_atom_4_by_2(pv, val);
return;
case -MO_16:
{
uint32_t val_le = cpu_to_le32(val);
int s2 = pi & 3;
int s1 = 4 - s2;
switch (s2) {
case 1:
val_le = store_whole_le4(pv, s1, val_le);
*(uint8_t *)(pv + 3) = val_le;
break;
case 3:
*(uint8_t *)pv = val_le;
store_whole_le4(pv + 1, s2, val_le >> 8);
break;
case 0: /* aligned */
case 2: /* atmax MO_16 */
default:
g_assert_not_reached();
}
}
return;
case MO_32:
if ((pi & 7) < 4) {
if (HAVE_al8) {
store_whole_le8(pv, 4, cpu_to_le32(val));
return;
}
} else {
if (HAVE_al16) {
store_whole_le16(pv, 4, int128_make64(cpu_to_le32(val)));
return;
}
}
cpu_loop_exit_atomic(env_cpu(env), ra);
default:
g_assert_not_reached();
}
}
/**
* store_atom_8:
* @p: host address
* @val: the value to store
* @memop: the full memory op
*
* Store 8 bytes to @p, honoring the atomicity of @memop.
*/
static void store_atom_8(CPUArchState *env, uintptr_t ra,
void *pv, MemOp memop, uint64_t val)
{
uintptr_t pi = (uintptr_t)pv;
int atmax;
if (HAVE_al8 && likely((pi & 7) == 0)) {
store_atomic8(pv, val);
return;
}
atmax = required_atomicity(env, pi, memop);
switch (atmax) {
case MO_8:
stq_he_p(pv, val);
return;
case MO_16:
store_atom_8_by_2(pv, val);
return;
case MO_32:
store_atom_8_by_4(pv, val);
return;
case -MO_32:
if (HAVE_al8) {
uint64_t val_le = cpu_to_le64(val);
int s2 = pi & 7;
int s1 = 8 - s2;
switch (s2) {
case 1 ... 3:
val_le = store_whole_le8(pv, s1, val_le);
store_bytes_leN(pv + s1, s2, val_le);
break;
case 5 ... 7:
val_le = store_bytes_leN(pv, s1, val_le);
store_whole_le8(pv + s1, s2, val_le);
break;
case 0: /* aligned */
case 4: /* atmax MO_32 */
default:
g_assert_not_reached();
}
return;
}
break;
case MO_64:
if (HAVE_al16) {
store_whole_le16(pv, 8, int128_make64(cpu_to_le64(val)));
return;
}
break;
default:
g_assert_not_reached();
}
cpu_loop_exit_atomic(env_cpu(env), ra);
}
/**
* store_atom_16:
* @p: host address
* @val: the value to store
* @memop: the full memory op
*
* Store 16 bytes to @p, honoring the atomicity of @memop.
*/
static void store_atom_16(CPUArchState *env, uintptr_t ra,
void *pv, MemOp memop, Int128 val)
{
uintptr_t pi = (uintptr_t)pv;
uint64_t a, b;
int atmax;
if (HAVE_al16_fast && likely((pi & 15) == 0)) {
store_atomic16(pv, val);
return;
}
atmax = required_atomicity(env, pi, memop);
a = HOST_BIG_ENDIAN ? int128_gethi(val) : int128_getlo(val);
b = HOST_BIG_ENDIAN ? int128_getlo(val) : int128_gethi(val);
switch (atmax) {
case MO_8:
memcpy(pv, &val, 16);
return;
case MO_16:
store_atom_8_by_2(pv, a);
store_atom_8_by_2(pv + 8, b);
return;
case MO_32:
store_atom_8_by_4(pv, a);
store_atom_8_by_4(pv + 8, b);
return;
case MO_64:
if (HAVE_al8) {
store_atomic8(pv, a);
store_atomic8(pv + 8, b);
return;
}
break;
case -MO_64:
if (HAVE_al16) {
uint64_t val_le;
int s2 = pi & 15;
int s1 = 16 - s2;
if (HOST_BIG_ENDIAN) {
val = bswap128(val);
}
switch (s2) {
case 1 ... 7:
val_le = store_whole_le16(pv, s1, val);
store_bytes_leN(pv + s1, s2, val_le);
break;
case 9 ... 15:
store_bytes_leN(pv, s1, int128_getlo(val));
val = int128_urshift(val, s1 * 8);
store_whole_le16(pv + s1, s2, val);
break;
case 0: /* aligned */
case 8: /* atmax MO_64 */
default:
g_assert_not_reached();
}
return;
}
break;
case MO_128:
if (HAVE_al16) {
store_atomic16(pv, val);
return;
}
break;
default:
g_assert_not_reached();
}
cpu_loop_exit_atomic(env_cpu(env), ra);
}
Reference in New Issue View Git Blame Copy Permalink