kernel/x86: Inline atomic functions and memory barriers

headers/private/kernel/arch/atomic.h

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _KERNEL_ARCH_ATOMIC_H
+#define _KERNEL_ARCH_ATOMIC_H
+
+
+#include <SupportDefs.h>
+
+#include <KernelExport.h>
+
+
+#ifdef __x86_64__
+#	include <arch/x86/64/atomic.h>
+#elif __INTEL__
+#	include <arch/x86/32/atomic.h>
+#endif
+
+
+#endif	// _KERNEL_ARCH_ATOMIC_H
+

headers/private/kernel/arch/cpu.h

@@ -43,10 +43,6 @@ status_t arch_cpu_user_memset(void *s, char c, size_t count,
 
 void arch_cpu_sync_icache(void *address, size_t length);
 
-void arch_cpu_memory_read_barrier(void);
-void arch_cpu_memory_write_barrier(void);
-void arch_cpu_memory_read_write_barrier(void);
-
 
 #ifdef __cplusplus
 }

headers/private/kernel/arch/x86/32/atomic.h

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _KERNEL_ARCH_X86_32_ATOMIC_H
+#define _KERNEL_ARCH_X86_32_ATOMIC_H
+
+
+static inline void
+memory_read_barrier_inline(void)
+{
+	asm volatile("lock; addl $0, (%%esp)" : : : "memory");
+}
+
+
+static inline void
+memory_write_barrier_inline(void)
+{
+	asm volatile("lock; addl $0, (%%esp)" : : : "memory");
+}
+
+
+static inline void
+memory_full_barrier_inline(void)
+{
+	asm volatile("lock; addl $0, (%%esp)" : : : "memory");
+}
+
+
+#define memory_read_barrier		memory_read_barrier_inline
+#define memory_write_barrier	memory_write_barrier_inline
+#define memory_full_barrier		memory_full_barrier_inline
+
+
+static inline void
+atomic_set_inline(int32* value, int32 newValue)
+{
+	memory_write_barrier();
+	*(volatile int32*)value = newValue;
+}
+
+
+static inline int32
+atomic_get_and_set_inline(int32* value, int32 newValue)
+{
+	asm volatile("xchgl %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_test_and_set_inline(int32* value, int32 newValue, int32 testAgainst)
+{
+	asm volatile("lock; cmpxchgl %2, (%3)"
+		: "=a" (newValue)
+		: "0" (testAgainst), "r" (newValue), "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_add_inline(int32* value, int32 newValue)
+{
+	asm volatile("lock; xaddl %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_get_inline(int32* value)
+{
+	int32 newValue = *(volatile int32*)value;
+	memory_read_barrier();
+	return newValue;
+}
+
+
+#define atomic_set				atomic_set_inline
+#define atomic_get_and_set		atomic_get_and_set_inline
+#ifndef atomic_test_and_set
+#	define atomic_test_and_set	atomic_test_and_set_inline
+#endif
+#ifndef atomic_add
+#	define atomic_add			atomic_add_inline
+#endif
+#define atomic_get				atomic_get_inline
+
+
+#endif	// _KERNEL_ARCH_X86_32_ATOMIC_H
+

headers/private/kernel/arch/x86/64/atomic.h

@@ -0,0 +1,153 @@
+/*
+ * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _KERNEL_ARCH_X86_64_ATOMIC_H
+#define _KERNEL_ARCH_X86_64_ATOMIC_H
+
+
+static inline void
+memory_read_barrier_inline(void)
+{
+	asm volatile("lfence" : : : "memory");
+}
+
+
+static inline void
+memory_write_barrier_inline(void)
+{
+	asm volatile("sfence" : : : "memory");
+}
+
+
+static inline void
+memory_full_barrier_inline(void)
+{
+	asm volatile("mfence" : : : "memory");
+}
+
+
+#define memory_read_barrier		memory_read_barrier_inline
+#define memory_write_barrier	memory_write_barrier_inline
+#define memory_full_barrier		memory_full_barrier_inline
+
+
+static inline void
+atomic_set_inline(int32* value, int32 newValue)
+{
+	memory_write_barrier();
+	*(volatile int32*)value = newValue;
+}
+
+
+static inline int32
+atomic_get_and_set_inline(int32* value, int32 newValue)
+{
+	asm volatile("xchg %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_test_and_set_inline(int32* value, int32 newValue, int32 testAgainst)
+{
+	asm volatile("lock; cmpxchgl %2, (%3)"
+		: "=a" (newValue)
+		: "0" (testAgainst), "r" (newValue), "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_add_inline(int32* value, int32 newValue)
+{
+	asm volatile("lock; xaddl %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int32
+atomic_get_inline(int32* value)
+{
+	int32 newValue = *(volatile int32*)value;
+	memory_read_barrier();
+	return newValue;
+}
+
+
+static inline void
+atomic_set64_inline(int64* value, int64 newValue)
+{
+	memory_write_barrier();
+	*(volatile int64*)value = newValue;
+}
+
+
+static inline int64
+atomic_get_and_set64_inline(int64* value, int64 newValue)
+{
+	asm volatile("xchgq %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int64
+atomic_test_and_set64_inline(int64* value, int64 newValue, int64 testAgainst)
+{
+	asm volatile("lock; cmpxchgq %2, (%3)"
+		: "=a" (newValue)
+		: "0" (testAgainst), "r" (newValue), "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int64
+atomic_add64_inline(int64* value, int64 newValue)
+{
+	asm volatile("lock; xaddq %0, (%1)"
+		: "+r" (newValue)
+		: "r" (value)
+		: "memory");
+	return newValue;
+}
+
+
+static inline int64
+atomic_get64_inline(int64* value)
+{
+	int64 newValue = *(volatile int64*)value;
+	memory_read_barrier();
+	return newValue;
+}
+
+
+#define atomic_set				atomic_set_inline
+#define atomic_get_and_set		atomic_get_and_set_inline
+#ifndef atomic_test_and_set
+#	define atomic_test_and_set	atomic_test_and_set_inline
+#endif
+#ifndef atomic_add
+#	define atomic_add			atomic_add_inline
+#endif
+#define atomic_get				atomic_get_inline
+
+#define atomic_set64			atomic_set64_inline
+#define atomic_get_and_set64	atomic_get_and_set64_inline
+#define atomic_test_and_set64	atomic_test_and_set64_inline
+#define atomic_add64			atomic_add64_inline
+#define atomic_get64			atomic_get64_inline
+
+
+#endif	// _KERNEL_ARCH_X86_64_ATOMIC_H
+

headers/private/kernel/lock.h

@@ -9,7 +9,10 @@
 #ifndef _KERNEL_LOCK_H
 #define _KERNEL_LOCK_H
 
+
 #include <OS.h>
+
+#include <arch/atomic.h>
 #include <debug.h>
 
 

headers/private/kernel/smp.h

@@ -9,6 +9,7 @@
 #define KERNEL_SMP_H
 
 
+#include <arch/atomic.h>
 #include <boot/kernel_args.h>
 #include <kernel.h>
 
@@ -185,7 +186,7 @@ release_spinlock_inline(spinlock* lock)
 static inline bool
 try_acquire_write_spinlock_inline(rw_spinlock* lock)
 {
-	return atomic_test_and_set(&lock->lock, 1 << 31, 0) == 0;
+	return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0;
 }
 
 
@@ -209,9 +210,9 @@ static inline bool
 try_acquire_read_spinlock_inline(rw_spinlock* lock)
 {
 	uint32 previous = atomic_add(&lock->lock, 1);
-	if ((previous & (1 << 31)) == 0)
+	if ((previous & (1u << 31)) == 0)
 		return true;
-	atomic_test_and_set(&lock->lock, 1 << 31, previous);
+	atomic_test_and_set(&lock->lock, 1u << 31, previous);
 	return false;
 }
 

headers/private/kernel/thread.h

@@ -11,12 +11,13 @@
 
 
 #include <OS.h>
-#include <thread_types.h>
-#include <arch/thread.h>
 
+#include <arch/atomic.h>
+#include <arch/thread.h>
 // For the thread blocking inline functions only.
 #include <kscheduler.h>
 #include <ksignal.h>
+#include <thread_types.h>
 
 
 struct arch_fork_arg;

src/system/kernel/arch/x86/arch_cpu.cpp

@@ -1288,38 +1288,3 @@ arch_cpu_sync_icache(void* address, size_t length)
 	// instruction cache is always consistent on x86
 }
 
-
-void
-arch_cpu_memory_read_barrier(void)
-{
-#ifdef __x86_64__
-	asm volatile("lfence" : : : "memory");
-#else
-	asm volatile ("lock;" : : : "memory");
-	asm volatile ("addl $0, 0(%%esp);" : : : "memory");
-#endif
-}
-
-
-void
-arch_cpu_memory_write_barrier(void)
-{
-#ifdef __x86_64__
-	asm volatile("sfence" : : : "memory");
-#else
-	asm volatile ("lock;" : : : "memory");
-	asm volatile ("addl $0, 0(%%esp);" : : : "memory");
-#endif
-}
-
-
-void
-arch_cpu_memory_read_write_barrier(void)
-{
-#ifdef __x86_64__
-	asm volatile("mfence" : : : "memory");
-#else
-	asm volatile ("lock;" : : : "memory");
-	asm volatile ("addl $0, 0(%%esp);" : : : "memory");
-#endif
-}

src/system/kernel/arch/x86/arch_smp.cpp

@@ -15,6 +15,7 @@
 #include <smp.h>
 #include <smp_priv.h>
 
+#include <arch/atomic.h>
 #include <arch/cpu.h>
 #include <arch/vm.h>
 #include <arch/smp.h>
@@ -139,7 +140,7 @@ arch_smp_send_multicast_ici(CPUSet& cpuSet)
 		panic("arch_smp_send_multicast_ici: called with interrupts enabled");
 #endif
 
-	arch_cpu_memory_write_barrier();
+	memory_write_barrier();
 
 	int32 i = 0;
 	int32 cpuCount = smp_get_num_cpus();
@@ -189,7 +190,7 @@ arch_smp_send_broadcast_ici(void)
 		panic("arch_smp_send_broadcast_ici: called with interrupts enabled");
 #endif
 
-	arch_cpu_memory_write_barrier();
+	memory_write_barrier();
 
 	uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED
 			| APIC_INTR_COMMAND_1_ASSERT
@@ -210,7 +211,7 @@ arch_smp_send_ici(int32 target_cpu)
 		panic("arch_smp_send_ici: called with interrupts enabled");
 #endif
 
-	arch_cpu_memory_write_barrier();
+	memory_write_barrier();
 
 	uint32 destination = sCPUAPICIds[target_cpu];
 	uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED

src/system/kernel/smp.cpp

@@ -17,6 +17,7 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <arch/atomic.h>
 #include <arch/cpu.h>
 #include <arch/debug.h>
 #include <arch/int.h>
@@ -616,10 +617,10 @@ try_acquire_read_spinlock(rw_spinlock* lock)
 #endif
 
 	uint32 previous = atomic_add(&lock->lock, 1);
-	if ((previous & (1 << 31)) == 0)
+	if ((previous & (1u << 31)) == 0)
 		return true;
 
-	atomic_test_and_set(&lock->lock, 1 << 31, previous);
+	atomic_test_and_set(&lock->lock, 1u << 31, previous);
 	return false;
 }
 
@@ -659,7 +660,7 @@ release_read_spinlock(rw_spinlock* lock)
 {
 #if DEBUG_SPINLOCKS
 	uint32 previous = atomic_add(&lock->lock, -1);
-	if ((previous & 1 << 31) != 0) {
+	if ((previous & 1u << 31) != 0) {
 		panic("release_read_spinlock: lock %p was already released (value:"
 			" %#" B_PRIx32 ")\n", lock, previous);
 	}
@@ -701,7 +702,8 @@ acquire_read_seqlock(seqlock* lock) {
 
 bool
 release_read_seqlock(seqlock* lock, uint32 count) {
-	arch_cpu_memory_read_barrier();
+	memory_read_barrier();
+
 	uint32 current = atomic_get((int32*)&lock->count);
 
 	if (count % 2 == 1 || current != count) {
@@ -1517,15 +1519,20 @@ call_all_cpus_sync(void (*func)(void*, int), void* cookie)
 }
 
 
+#undef memory_read_barrier
+#undef memory_write_barrier
+
+
 void
-memory_read_barrier(void)
+memory_read_barrier()
 {
-	arch_cpu_memory_read_barrier();
+	memory_read_barrier_inline();
 }
 
 
 void
-memory_write_barrier(void)
+memory_write_barrier()
 {
-	arch_cpu_memory_write_barrier();
+	memory_write_barrier_inline();
 }
+