diff --git a/headers/os/drivers/KernelExport.h b/headers/os/drivers/KernelExport.h index d4168557dd..de40d0c76d 100644 --- a/headers/os/drivers/KernelExport.h +++ b/headers/os/drivers/KernelExport.h @@ -40,6 +40,15 @@ typedef ulong cpu_status; # define B_SPINLOCK_IS_LOCKED(lock) (*(lock) > 0) #endif +typedef struct { + int32 lock; +} rw_spinlock; + +#define B_RW_SPINLOCK_INITIALIZER { 0 } +#define B_INITIALIZE_RW_SPINLOCK(rw_spinlock) do { \ + (rw_spinlock)->lock = 0; \ + } while (false) + typedef struct { spinlock lock; uint32 count; @@ -137,6 +146,13 @@ extern void restore_interrupts(cpu_status status); extern void acquire_spinlock(spinlock *lock); extern void release_spinlock(spinlock *lock); +extern bool try_acquire_write_spinlock(rw_spinlock* lock); +extern void acquire_write_spinlock(rw_spinlock* lock); +extern void release_write_spinlock(rw_spinlock* lock); +extern bool try_acquire_read_spinlock(rw_spinlock* lock); +extern void acquire_read_spinlock(rw_spinlock* lock); +extern void release_read_spinlock(rw_spinlock* lock); + extern bool try_acquire_write_seqlock(seqlock* lock); extern void acquire_write_seqlock(seqlock* lock); extern void release_write_seqlock(seqlock* lock); diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index c7aa163762..8ef0e82a34 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -103,6 +103,65 @@ release_spinlock_inline(spinlock* lock) #endif // !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION +static inline bool +try_acquire_write_spinlock_inline(rw_spinlock* lock) +{ + return atomic_test_and_set(&lock->lock, 1 << 31, 0) == 0; +} + + +static inline void +acquire_write_spinlock_inline(rw_spinlock* lock) +{ + if (try_acquire_write_spinlock(lock)) + return; + acquire_write_spinlock(lock); +} + + +static inline void +release_write_spinlock_inline(rw_spinlock* lock) +{ + atomic_set(&lock->lock, 0); +} + + +static inline bool +try_acquire_read_spinlock_inline(rw_spinlock* lock) +{ + uint32 previous = atomic_add(&lock->lock, 1); + if ((previous & (1 << 31)) == 0) + return true; + atomic_test_and_set(&lock->lock, 1 << 31, previous); + return false; +} + + +static inline void +acquire_read_spinlock_inline(rw_spinlock* lock) +{ + if (try_acquire_read_spinlock(lock)) + return; + acquire_read_spinlock(lock); +} + + +static inline void +release_read_spinlock_inline(rw_spinlock* lock) +{ + atomic_add(&lock->lock, -1); +} + + +#define try_acquire_read_spinlock(lock) try_acquire_read_spinlock_inline(lock) +#define acquire_read_spinlock(lock) acquire_read_spinlock_inline(lock) +#define release_read_spinlock(lock) release_read_spinlock_inline(lock) +#define try_acquire_write_spinlock(lock) \ + try_acquire_write_spinlock(lock) +#define acquire_write_spinlock(lock) acquire_write_spinlock_inline(lock) +#define release_write_spinlock(lock) release_write_spinlock_inline(lock) + + static inline bool try_acquire_write_seqlock_inline(seqlock* lock) { bool succeed = try_acquire_spinlock(&lock->lock); diff --git a/headers/private/kernel/util/AutoLock.h b/headers/private/kernel/util/AutoLock.h index 5a334fbc1a..3151065857 100644 --- a/headers/private/kernel/util/AutoLock.h +++ b/headers/private/kernel/util/AutoLock.h @@ -160,6 +160,98 @@ private: typedef AutoLocker InterruptsSpinLocker; +class ReadSpinLocking { +public: + inline bool Lock(rw_spinlock* lockable) + { + acquire_read_spinlock(lockable); + return true; + } + + inline void Unlock(rw_spinlock* lockable) + { + release_read_spinlock(lockable); + } +}; + +typedef AutoLocker ReadSpinLocker; + + +class InterruptsReadSpinLocking { +public: + InterruptsReadSpinLocking() + : + fState(0) + { + } + + inline bool Lock(rw_spinlock* lockable) + { + fState = disable_interrupts(); + acquire_read_spinlock(lockable); + return true; + } + + inline void Unlock(rw_spinlock* lockable) + { + release_read_spinlock(lockable); + restore_interrupts(fState); + } + +private: + int fState; +}; + +typedef AutoLocker + InterruptsReadSpinLocker; + + +class WriteSpinLocking { +public: + inline bool Lock(rw_spinlock* lockable) + { + acquire_write_spinlock(lockable); + return true; + } + + inline void Unlock(rw_spinlock* lockable) + { + release_write_spinlock(lockable); + } +}; + +typedef AutoLocker WriteSpinLocker; + + +class InterruptsWriteSpinLocking { +public: + InterruptsWriteSpinLocking() + : + fState(0) + { + } + + inline bool Lock(rw_spinlock* lockable) + { + fState = disable_interrupts(); + acquire_write_spinlock(lockable); + return true; + } + + inline void Unlock(rw_spinlock* lockable) + { + release_write_spinlock(lockable); + restore_interrupts(fState); + } + +private: + int fState; +}; + +typedef AutoLocker + InterruptsWriteSpinLocker; + + class WriteSequentialLocking { public: inline bool Lock(seqlock* lockable) @@ -237,6 +329,10 @@ using BPrivate::WriteLocker; using BPrivate::InterruptsLocker; using BPrivate::SpinLocker; using BPrivate::InterruptsSpinLocker; +using BPrivate::ReadSpinLocker; +using BPrivate::InterruptsReadSpinLocker; +using BPrivate::WriteSpinLocker; +using BPrivate::InterruptsWriteSpinLocker; using BPrivate::WriteSequentialLocker; using BPrivate::InterruptsWriteSequentialLocker; using BPrivate::ThreadCPUPinner; diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index caf29bfe9a..950b499a9b 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -45,6 +45,13 @@ #undef acquire_spinlock #undef release_spinlock +#undef try_acquire_read_spinlock +#undef acquire_read_spinlock +#undef release_read_spinlock +#undef try_acquire_write_spinlock +#undef acquire_write_spinlock +#undef release_write_spinlock + #undef try_acquire_write_seqlock #undef acquire_write_seqlock #undef release_write_seqlock @@ -520,6 +527,86 @@ release_spinlock(spinlock *lock) } +bool +try_acquire_write_spinlock(rw_spinlock* lock) +{ + return atomic_test_and_set(&lock->lock, 1 << 31, 0) == 0; +} + + +void +acquire_write_spinlock(rw_spinlock* lock) +{ + if (sNumCPUs < 2) + return; + + uint32 count = 0; + int currentCPU = smp_get_current_cpu(); + while (true) { + if (try_acquire_write_spinlock(lock)) + break; + + if (++count == SPINLOCK_DEADLOCK_COUNT) { + panic("acquire_write_spinlock(): Failed to acquire spinlock %p " + "for a long time!", lock); + count = 0; + } + + process_all_pending_ici(currentCPU); + PAUSE(); + } +} + + +void +release_write_spinlock(rw_spinlock* lock) +{ + atomic_set(&lock->lock, 0); +} + + +bool +try_acquire_read_spinlock(rw_spinlock* lock) +{ + uint32 previous = atomic_add(&lock->lock, 1); + if ((previous & (1 << 31)) == 0) + return true; + atomic_test_and_set(&lock->lock, 1 << 31, previous); + return false; +} + + +void +acquire_read_spinlock(rw_spinlock* lock) +{ + if (sNumCPUs < 2) + return; + + uint32 count = 0; + int currentCPU = smp_get_current_cpu(); + while (1) { + if (try_acquire_read_spinlock(lock)) + break; + + if (++count == SPINLOCK_DEADLOCK_COUNT) { + panic("acquire_read_spinlock(): Failed to acquire spinlock %p " + "for a long time!", lock); + count = 0; + } + + process_all_pending_ici(currentCPU); + PAUSE(); + } +} + + +void +release_read_spinlock(rw_spinlock* lock) +{ + atomic_add(&lock->lock, -1); +} + + bool try_acquire_write_seqlock(seqlock* lock) { bool succeed = try_acquire_spinlock(&lock->lock);