From 8cf8e537740789b1b103f0aa0736dbfcf55359c2 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 6 Jan 2014 06:49:34 +0100 Subject: [PATCH] kernel/x86: Inline atomic functions and memory barriers --- headers/private/kernel/arch/atomic.h | 22 +++ headers/private/kernel/arch/cpu.h | 4 - headers/private/kernel/arch/x86/32/atomic.h | 97 +++++++++++++ headers/private/kernel/arch/x86/64/atomic.h | 153 ++++++++++++++++++++ headers/private/kernel/lock.h | 3 + headers/private/kernel/smp.h | 7 +- headers/private/kernel/thread.h | 5 +- src/system/kernel/arch/x86/arch_cpu.cpp | 35 ----- src/system/kernel/arch/x86/arch_smp.cpp | 7 +- src/system/kernel/smp.cpp | 23 ++- 10 files changed, 301 insertions(+), 55 deletions(-) create mode 100644 headers/private/kernel/arch/atomic.h create mode 100644 headers/private/kernel/arch/x86/32/atomic.h create mode 100644 headers/private/kernel/arch/x86/64/atomic.h diff --git a/headers/private/kernel/arch/atomic.h b/headers/private/kernel/arch/atomic.h new file mode 100644 index 0000000000..c958b3c6d6 --- /dev/null +++ b/headers/private/kernel/arch/atomic.h @@ -0,0 +1,22 @@ +/* + * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef _KERNEL_ARCH_ATOMIC_H +#define _KERNEL_ARCH_ATOMIC_H + + +#include + +#include + + +#ifdef __x86_64__ +# include +#elif __INTEL__ +# include +#endif + + +#endif // _KERNEL_ARCH_ATOMIC_H + diff --git a/headers/private/kernel/arch/cpu.h b/headers/private/kernel/arch/cpu.h index d5ea2de81e..188df59081 100644 --- a/headers/private/kernel/arch/cpu.h +++ b/headers/private/kernel/arch/cpu.h @@ -43,10 +43,6 @@ status_t arch_cpu_user_memset(void *s, char c, size_t count, void arch_cpu_sync_icache(void *address, size_t length); -void arch_cpu_memory_read_barrier(void); -void arch_cpu_memory_write_barrier(void); -void arch_cpu_memory_read_write_barrier(void); - #ifdef __cplusplus } diff --git a/headers/private/kernel/arch/x86/32/atomic.h b/headers/private/kernel/arch/x86/32/atomic.h new file mode 100644 index 0000000000..adec9c05c3 --- /dev/null +++ b/headers/private/kernel/arch/x86/32/atomic.h @@ -0,0 +1,97 @@ +/* + * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef _KERNEL_ARCH_X86_32_ATOMIC_H +#define _KERNEL_ARCH_X86_32_ATOMIC_H + + +static inline void +memory_read_barrier_inline(void) +{ + asm volatile("lock; addl $0, (%%esp)" : : : "memory"); +} + + +static inline void +memory_write_barrier_inline(void) +{ + asm volatile("lock; addl $0, (%%esp)" : : : "memory"); +} + + +static inline void +memory_full_barrier_inline(void) +{ + asm volatile("lock; addl $0, (%%esp)" : : : "memory"); +} + + +#define memory_read_barrier memory_read_barrier_inline +#define memory_write_barrier memory_write_barrier_inline +#define memory_full_barrier memory_full_barrier_inline + + +static inline void +atomic_set_inline(int32* value, int32 newValue) +{ + memory_write_barrier(); + *(volatile int32*)value = newValue; +} + + +static inline int32 +atomic_get_and_set_inline(int32* value, int32 newValue) +{ + asm volatile("xchgl %0, (%1)" + : "+r" (newValue) + : "r" (value) + : "memory"); + return newValue; +} + + +static inline int32 +atomic_test_and_set_inline(int32* value, int32 newValue, int32 testAgainst) +{ + asm volatile("lock; cmpxchgl %2, (%3)" + : "=a" (newValue) + : "0" (testAgainst), "r" (newValue), "r" (value) + : "memory"); + return newValue; +} + + +static inline int32 +atomic_add_inline(int32* value, int32 newValue) +{ + asm volatile("lock; xaddl %0, (%1)" + : "+r" (newValue) + : "r" (value) + : "memory"); + return newValue; +} + + +static inline int32 +atomic_get_inline(int32* value) +{ + int32 newValue = *(volatile int32*)value; + memory_read_barrier(); + return newValue; +} + + +#define atomic_set atomic_set_inline +#define atomic_get_and_set atomic_get_and_set_inline +#ifndef atomic_test_and_set +# define atomic_test_and_set atomic_test_and_set_inline +#endif +#ifndef atomic_add +# define atomic_add atomic_add_inline +#endif +#define atomic_get atomic_get_inline + + +#endif // _KERNEL_ARCH_X86_32_ATOMIC_H + diff --git a/headers/private/kernel/arch/x86/64/atomic.h b/headers/private/kernel/arch/x86/64/atomic.h new file mode 100644 index 0000000000..fd567319f6 --- /dev/null +++ b/headers/private/kernel/arch/x86/64/atomic.h @@ -0,0 +1,153 @@ +/* + * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef _KERNEL_ARCH_X86_64_ATOMIC_H +#define _KERNEL_ARCH_X86_64_ATOMIC_H + + +static inline void +memory_read_barrier_inline(void) +{ + asm volatile("lfence" : : : "memory"); +} + + +static inline void +memory_write_barrier_inline(void) +{ + asm volatile("sfence" : : : "memory"); +} + + +static inline void +memory_full_barrier_inline(void) +{ + asm volatile("mfence" : : : "memory"); +} + + +#define memory_read_barrier memory_read_barrier_inline +#define memory_write_barrier memory_write_barrier_inline +#define memory_full_barrier memory_full_barrier_inline + + +static inline void +atomic_set_inline(int32* value, int32 newValue) +{ + memory_write_barrier(); + *(volatile int32*)value = newValue; +} + + +static inline int32 +atomic_get_and_set_inline(int32* value, int32 newValue) +{ + asm volatile("xchg %0, (%1)" + : "+r" (newValue) + : "r" (value) + : "memory"); + return newValue; +} + + +static inline int32 +atomic_test_and_set_inline(int32* value, int32 newValue, int32 testAgainst) +{ + asm volatile("lock; cmpxchgl %2, (%3)" + : "=a" (newValue) + : "0" (testAgainst), "r" (newValue), "r" (value) + : "memory"); + return newValue; +} + + +static inline int32 +atomic_add_inline(int32* value, int32 newValue) +{ + asm volatile("lock; xaddl %0, (%1)" + : "+r" (newValue) + : "r" (value) + : "memory"); + return newValue; +} + + +static inline int32 +atomic_get_inline(int32* value) +{ + int32 newValue = *(volatile int32*)value; + memory_read_barrier(); + return newValue; +} + + +static inline void +atomic_set64_inline(int64* value, int64 newValue) +{ + memory_write_barrier(); + *(volatile int64*)value = newValue; +} + + +static inline int64 +atomic_get_and_set64_inline(int64* value, int64 newValue) +{ + asm volatile("xchgq %0, (%1)" + : "+r" (newValue) + : "r" (value) + : "memory"); + return newValue; +} + + +static inline int64 +atomic_test_and_set64_inline(int64* value, int64 newValue, int64 testAgainst) +{ + asm volatile("lock; cmpxchgq %2, (%3)" + : "=a" (newValue) + : "0" (testAgainst), "r" (newValue), "r" (value) + : "memory"); + return newValue; +} + + +static inline int64 +atomic_add64_inline(int64* value, int64 newValue) +{ + asm volatile("lock; xaddq %0, (%1)" + : "+r" (newValue) + : "r" (value) + : "memory"); + return newValue; +} + + +static inline int64 +atomic_get64_inline(int64* value) +{ + int64 newValue = *(volatile int64*)value; + memory_read_barrier(); + return newValue; +} + + +#define atomic_set atomic_set_inline +#define atomic_get_and_set atomic_get_and_set_inline +#ifndef atomic_test_and_set +# define atomic_test_and_set atomic_test_and_set_inline +#endif +#ifndef atomic_add +# define atomic_add atomic_add_inline +#endif +#define atomic_get atomic_get_inline + +#define atomic_set64 atomic_set64_inline +#define atomic_get_and_set64 atomic_get_and_set64_inline +#define atomic_test_and_set64 atomic_test_and_set64_inline +#define atomic_add64 atomic_add64_inline +#define atomic_get64 atomic_get64_inline + + +#endif // _KERNEL_ARCH_X86_64_ATOMIC_H + diff --git a/headers/private/kernel/lock.h b/headers/private/kernel/lock.h index 64667a72f7..d3534aa188 100644 --- a/headers/private/kernel/lock.h +++ b/headers/private/kernel/lock.h @@ -9,7 +9,10 @@ #ifndef _KERNEL_LOCK_H #define _KERNEL_LOCK_H + #include + +#include #include diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 3378f23d26..685e3d5a03 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -9,6 +9,7 @@ #define KERNEL_SMP_H +#include #include #include @@ -185,7 +186,7 @@ release_spinlock_inline(spinlock* lock) static inline bool try_acquire_write_spinlock_inline(rw_spinlock* lock) { - return atomic_test_and_set(&lock->lock, 1 << 31, 0) == 0; + return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0; } @@ -209,9 +210,9 @@ static inline bool try_acquire_read_spinlock_inline(rw_spinlock* lock) { uint32 previous = atomic_add(&lock->lock, 1); - if ((previous & (1 << 31)) == 0) + if ((previous & (1u << 31)) == 0) return true; - atomic_test_and_set(&lock->lock, 1 << 31, previous); + atomic_test_and_set(&lock->lock, 1u << 31, previous); return false; } diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h index f41987c8c4..f502c94251 100644 --- a/headers/private/kernel/thread.h +++ b/headers/private/kernel/thread.h @@ -11,12 +11,13 @@ #include -#include -#include +#include +#include // For the thread blocking inline functions only. #include #include +#include struct arch_fork_arg; diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 3de6d53835..a21afcde52 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -1288,38 +1288,3 @@ arch_cpu_sync_icache(void* address, size_t length) // instruction cache is always consistent on x86 } - -void -arch_cpu_memory_read_barrier(void) -{ -#ifdef __x86_64__ - asm volatile("lfence" : : : "memory"); -#else - asm volatile ("lock;" : : : "memory"); - asm volatile ("addl $0, 0(%%esp);" : : : "memory"); -#endif -} - - -void -arch_cpu_memory_write_barrier(void) -{ -#ifdef __x86_64__ - asm volatile("sfence" : : : "memory"); -#else - asm volatile ("lock;" : : : "memory"); - asm volatile ("addl $0, 0(%%esp);" : : : "memory"); -#endif -} - - -void -arch_cpu_memory_read_write_barrier(void) -{ -#ifdef __x86_64__ - asm volatile("mfence" : : : "memory"); -#else - asm volatile ("lock;" : : : "memory"); - asm volatile ("addl $0, 0(%%esp);" : : : "memory"); -#endif -} diff --git a/src/system/kernel/arch/x86/arch_smp.cpp b/src/system/kernel/arch/x86/arch_smp.cpp index e9d4b4c545..fd5535157f 100644 --- a/src/system/kernel/arch/x86/arch_smp.cpp +++ b/src/system/kernel/arch/x86/arch_smp.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -139,7 +140,7 @@ arch_smp_send_multicast_ici(CPUSet& cpuSet) panic("arch_smp_send_multicast_ici: called with interrupts enabled"); #endif - arch_cpu_memory_write_barrier(); + memory_write_barrier(); int32 i = 0; int32 cpuCount = smp_get_num_cpus(); @@ -189,7 +190,7 @@ arch_smp_send_broadcast_ici(void) panic("arch_smp_send_broadcast_ici: called with interrupts enabled"); #endif - arch_cpu_memory_write_barrier(); + memory_write_barrier(); uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED | APIC_INTR_COMMAND_1_ASSERT @@ -210,7 +211,7 @@ arch_smp_send_ici(int32 target_cpu) panic("arch_smp_send_ici: called with interrupts enabled"); #endif - arch_cpu_memory_write_barrier(); + memory_write_barrier(); uint32 destination = sCPUAPICIds[target_cpu]; uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index 3631234771..a82c7a273e 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -616,10 +617,10 @@ try_acquire_read_spinlock(rw_spinlock* lock) #endif uint32 previous = atomic_add(&lock->lock, 1); - if ((previous & (1 << 31)) == 0) + if ((previous & (1u << 31)) == 0) return true; - atomic_test_and_set(&lock->lock, 1 << 31, previous); + atomic_test_and_set(&lock->lock, 1u << 31, previous); return false; } @@ -659,7 +660,7 @@ release_read_spinlock(rw_spinlock* lock) { #if DEBUG_SPINLOCKS uint32 previous = atomic_add(&lock->lock, -1); - if ((previous & 1 << 31) != 0) { + if ((previous & 1u << 31) != 0) { panic("release_read_spinlock: lock %p was already released (value:" " %#" B_PRIx32 ")\n", lock, previous); } @@ -701,7 +702,8 @@ acquire_read_seqlock(seqlock* lock) { bool release_read_seqlock(seqlock* lock, uint32 count) { - arch_cpu_memory_read_barrier(); + memory_read_barrier(); + uint32 current = atomic_get((int32*)&lock->count); if (count % 2 == 1 || current != count) { @@ -1517,15 +1519,20 @@ call_all_cpus_sync(void (*func)(void*, int), void* cookie) } +#undef memory_read_barrier +#undef memory_write_barrier + + void -memory_read_barrier(void) +memory_read_barrier() { - arch_cpu_memory_read_barrier(); + memory_read_barrier_inline(); } void -memory_write_barrier(void) +memory_write_barrier() { - arch_cpu_memory_write_barrier(); + memory_write_barrier_inline(); } +