Aren/NetBSD
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Part I of ad@'s performance improvements for aarch64

Browse Source 
- Remove memory barriers from the atomic ops.  I don't understand why those
  are there.  Is it some architectural thing, or for a CPU bug, or just
  over-caution maybe?  They're not needed for correctness.

- Have unlikely conditional branches go forwards to help the static branch
  predictor.
This commit is contained in:
skrll 2020-08-12 12:59:57 +00:00
parent 54695ccbd8
commit e16659bb50
16 changed files with 64 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_cas_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_cas_16.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -37,9 +37,9 @@ ENTRY_NP(_atomic_cas_16)
cmp w0, w1 /* compare? */
b.ne 2f
stxrh w3, w2, [x4] /* store new value */
cbnz w3, 1b /* succeed? nope, try again. */
dmb st /* data memory barrier */
cbnz w3, 3f /* succeed? nope, try again. */
2: ret /* return. */
3: b 1b
END(_atomic_cas_16)
ATOMIC_OP_ALIAS(atomic_cas_16,_atomic_cas_16)

6
common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_cas_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_cas_32.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -37,9 +37,9 @@ ENTRY_NP(_atomic_cas_32)
cmp w0, w1 /* compare? */
b.ne 2f /* return if different */
stxr w3, w2, [x4] /* store new value */
cbnz w3, 1b /* succeed? nope, try again. */
dmb st
cbnz w3, 3f /* succeed? nope, try again. */
2: ret /* return. */
3: b 1b
END(_atomic_cas_32)
ATOMIC_OP_ALIAS(atomic_cas_32,_atomic_cas_32)

6
common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_cas_64.S,v 1.3 2019/02/19 12:51:44 rin Exp $ */
/* $NetBSD: atomic_cas_64.S,v 1.4 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -37,9 +37,9 @@ ENTRY_NP(_atomic_cas_64)
cmp x0, x1 /* compare? */
b.ne 2f /* return if different */
stxr w3, x2, [x4] /* store new value */
cbnz w3, 1b /* succeed? nope, try again. */
dmb st
cbnz w3, 3f /* succeed? nope, try again. */
2: ret /* return. */
3: b 1b
END(_atomic_cas_64)
ATOMIC_OP_ALIAS(atomic_cas_64,_atomic_cas_64)

6
common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_cas_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_cas_8.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -37,9 +37,9 @@ ENTRY_NP(_atomic_cas_8)
cmp w0, w1 /* compare? */
b.ne 2f
stxrb w3, w2, [x4] /* store new value */
cbnz w3, 1b /* succeed? nope, try again. */
dmb st /* data memory barrier */
cbnz w3, 3f /* succeed? nope, try again. */
2: ret /* return. */
3: b 1b
END(_atomic_cas_8)
ATOMIC_OP_ALIAS(atomic_cas_8,_atomic_cas_8)

10
common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_dec_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_dec_32.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -35,9 +35,9 @@ ENTRY_NP(_atomic_dec_32)
1: ldxr w3, [x0] /* load old value (return value) */
sub w3, w3, #1 /* calculate new value */
stxr w2, w3, [x0] /* try to store */
cbnz w2, 1b /* succeed? no, try again? */
dmb st
cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
2: b 1b
END(_atomic_dec_32)
ATOMIC_OP_ALIAS(atomic_dec_32,_atomic_dec_32)
@ -49,9 +49,9 @@ ENTRY_NP(_atomic_dec_32_nv)
1: ldxr w0, [x4] /* load old value */
sub w0, w0, #1 /* calculate new value (return value) */
stxr w3, w0, [x4] /* try to store */
cbnz w3, 1b /* succeed? no, try again? */
dmb st
cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
2: b 1b
END(_atomic_dec_32_nv)
ATOMIC_OP_ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv)

10
common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_dec_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_dec_64.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -35,9 +35,9 @@ ENTRY_NP(_atomic_dec_64)
1: ldxr x2, [x0] /* load old value (return value) */
sub x2, x2, #1 /* calculate new value */
stxr w3, x2, [x0] /* try to store */
cbnz w3, 1b /* succeed? no, try again? */
dmb st
cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
2: b 1b
END(_atomic_dec_64)
ATOMIC_OP_ALIAS(atomic_dec_64,_atomic_dec_64)
@ -51,9 +51,9 @@ ENTRY_NP(_atomic_dec_64_nv)
1: ldxr x0, [x4] /* load old value */
sub x0, x0, #1 /* calculate new value (return value) */
stxr w3, x0, [x4] /* try to store */
cbnz w3, 1b /* succeed? no, try again? */
dmb st
cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
2: b 1b
END(_atomic_dec_64_nv)
ATOMIC_OP_ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv)

10
common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_inc_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_inc_32.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -35,9 +35,9 @@ ENTRY_NP(_atomic_inc_32)
1: ldxr w3, [x0] /* load old value (return value) */
add w3, w3, #1 /* calculate new value */
stxr w2, w3, [x0] /* try to store */
cbnz w2, 1b /* succeed? no, try again? */
dmb st
cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
2: b 1b
END(_atomic_inc_32)
ATOMIC_OP_ALIAS(atomic_inc_32,_atomic_inc_32)
@ -49,9 +49,9 @@ ENTRY_NP(_atomic_inc_32_nv)
1: ldxr w0, [x4] /* load old value */
add w0, w0, #1 /* calculate new value (return value) */
stxr w3, w0, [x4] /* try to store */
cbnz w3, 1b /* succeed? no, try again? */
dmb st
cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
2: b 1b
END(_atomic_inc_32_nv)
ATOMIC_OP_ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv)

10
common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_inc_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_inc_64.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -35,9 +35,9 @@ ENTRY_NP(_atomic_inc_64)
1: ldxr x2, [x0] /* load old value (return value) */
add x2, x2, #1 /* calculate new value */
stxr w3, x2, [x0] /* try to store */
cbnz w3, 1b /* succeed? no, try again? */
dmb st
cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
2: b 1b
END(_atomic_inc_64)
ATOMIC_OP_ALIAS(atomic_inc_64,_atomic_inc_64)
@ -51,9 +51,9 @@ ENTRY_NP(_atomic_inc_64_nv)
1: ldxr x0, [x4] /* load old value */
add x0, x0, #1 /* calculate new value (return value) */
stxr w3, x0, [x4] /* try to store */
cbnz w3, 1b /* succeed? no, try again? */
dmb st
cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
2: b 1b
END(_atomic_inc_64_nv)
ATOMIC_OP_ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv)

10
common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_nand_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_nand_16.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -37,9 +37,9 @@ ENTRY_NP(_atomic_nand_16)
mvn w3, w0 /* complement source */
and w3, w3, w1 /* calculate new value */
stxrh w2, w3, [x4] /* try to store */
cbnz w2, 1b /* succeed? no, try again */
dmb st
cbnz w2, 2f /* succeed? no, try again */
ret /* return old value */
2: b 1b
END(_atomic_nand_16)
ATOMIC_OP_ALIAS(atomic_nand_16,_atomic_nand_16)
@ -53,9 +53,9 @@ ENTRY_NP(_atomic_nand_16_nv)
mvn w0, w0 /* complement source */
and w0, w0, w1 /* calculate new value (return value) */
stxrh w2, w0, [x4] /* try to store */
cbnz w2, 1b /* succeed? no, try again? */
dmb st
cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
2: b 1b
END(_atomic_nand_16_nv)
ATOMIC_OP_ALIAS(atomic_nand_16_nv,_atomic_nand_16_nv)

10
common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_nand_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_nand_32.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -37,9 +37,9 @@ ENTRY_NP(_atomic_nand_32)
mvn w3, w0 /* complement source */
and w3, w3, w1 /* calculate new value */
stxr w2, w3, [x4] /* try to store */
cbnz w2, 1b /* succeed? no, try again */
dmb st
cbnz w2, 2f /* succeed? no, try again */
ret /* return old value */
2: b 1b
END(_atomic_nand_32)
ATOMIC_OP_ALIAS(atomic_nand_32,_atomic_nand_32)
@ -53,9 +53,9 @@ ENTRY_NP(_atomic_nand_32_nv)
mvn w0, w0 /* complement source */
and w0, w0, w1 /* calculate new value (return value) */
stxr w2, w0, [x4] /* try to store */
cbnz w2, 1b /* succeed? no, try again? */
dmb st
cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
2: b 1b
END(_atomic_nand_32_nv)
ATOMIC_OP_ALIAS(atomic_nand_32_nv,_atomic_nand_32_nv)

10
common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_nand_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_nand_64.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -37,9 +37,9 @@ ENTRY_NP(_atomic_nand_64)
mvn x2, x0 /* complement source */
and x2, x2, x1 /* calculate new value */
stxr w3, x2, [x4] /* try to store */
cbnz w3, 1b /* succeed? no, try again */
dmb st
cbnz w3, 2f /* succeed? no, try again */
ret /* return old value */
2: b 1b
END(_atomic_nand_64)
ATOMIC_OP_ALIAS(atomic_nand_64,_atomic_nand_64)
@ -53,9 +53,9 @@ ENTRY_NP(_atomic_nand_64_nv)
mvn x0, x0 /* complement source */
and x0, x0, x1 /* calculate new value (return value) */
stxr w3, x0, [x4] /* try to store */
cbnz w3, 1b /* succeed? no, try again? */
dmb st
cbnz w3, 2f /* succeed? no, try again? */
ret /* return new value */
2: b 1b
END(_atomic_nand_64_nv)
ATOMIC_OP_ALIAS(atomic_nand_64_nv,_atomic_nand_64_nv)

10
common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_nand_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_nand_8.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -37,9 +37,9 @@ ENTRY_NP(_atomic_nand_8)
mvn w3, w0 /* complement source */
and w3, w3, w1 /* calculate new value */
stxrb w2, w3, [x4] /* try to store */
cbnz w2, 1b /* succeed? no, try again */
dmb st
cbnz w2, 2f /* succeed? no, try again */
ret /* return old value */
2: b 1b
END(_atomic_nand_8)
ATOMIC_OP_ALIAS(atomic_nand_8,_atomic_nand_8)
@ -53,9 +53,9 @@ ENTRY_NP(_atomic_nand_8_nv)
mvn w0, w0 /* complement source */
and w0, w0, w1 /* calculate new value (return value) */
stxrb w2, w0, [x4] /* try to store */
cbnz w2, 1b /* succeed? no, try again? */
dmb st
cbnz w2, 2f /* succeed? no, try again? */
ret /* return new value */
2: b 1b
END(_atomic_nand_8_nv)
ATOMIC_OP_ALIAS(atomic_nand_8_nv,_atomic_nand_8_nv)

6
common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_swap_16.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_swap_16.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -35,9 +35,9 @@ ENTRY_NP(_atomic_swap_16)
mov x4, x0
1: ldxrh w0, [x4]
stxrh w3, w1, [x4]
cbnz w3, 1b
dmb st
cbnz w3, 2f
ret
2: b 1b
END(_atomic_swap_16)
ATOMIC_OP_ALIAS(atomic_swap_16,_atomic_swap_16)

6
common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_swap_32.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_swap_32.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -35,9 +35,9 @@ ENTRY_NP(_atomic_swap_32)
mov x4, x0
1: ldxr w0, [x4]
stxr w3, w1, [x4]
cbnz w3, 1b
dmb st
cbnz w3, 2f
ret
2: b 1b
END(_atomic_swap_32)
ATOMIC_OP_ALIAS(atomic_swap_32,_atomic_swap_32)

6
common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_swap_64.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_swap_64.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -35,9 +35,9 @@ ENTRY_NP(_atomic_swap_64)
mov x4, x0
1: ldxr x0, [x4]
stxr w3, x1, [x4]
cbnz w3, 1b
dmb st
cbnz w3, 2f
ret
2: b 1b
END(_atomic_swap_64)
ATOMIC_OP_ALIAS(atomic_swap_64,_atomic_swap_64)

6
common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S
View File

@ -1,4 +1,4 @@
/* $NetBSD: atomic_swap_8.S,v 1.1 2014/08/10 05:47:35 matt Exp $ */
/* $NetBSD: atomic_swap_8.S,v 1.2 2020/08/12 12:59:57 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@ -35,9 +35,9 @@ ENTRY_NP(_atomic_swap_8)
mov x4, x0
1: ldxrb w0, [x4]
stxrb w3, w1, [x4]
cbnz w3, 1b
dmb st
cbnz w3, 2f
ret
2: b 1b
END(_atomic_swap_8)
ATOMIC_OP_ALIAS(atomic_swap_8,_atomic_swap_8)