From 265927509dc56e82b12cd68750ae1e96601fd558 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 30 Dec 2013 20:03:36 +0100 Subject: [PATCH] scheduler: Protect per CPU run queue with its own lock --- src/system/kernel/scheduler/scheduler.cpp | 2 +- src/system/kernel/scheduler/scheduler_cpu.cpp | 22 ++++--- src/system/kernel/scheduler/scheduler_cpu.h | 36 ++++++++++++ .../kernel/scheduler/scheduler_thread.h | 57 ++++++++++++------- 4 files changed, 86 insertions(+), 31 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index f8d8f802a9..fc941b34fd 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -517,7 +517,7 @@ reschedule(int32 nextState) ThreadData* nextThreadData; if (gCPU[thisCPU].disabled) { if (!thread_is_idle_thread(oldThread)) { - CoreRunQueueLocker _(core); + CPURunQueueLocker _(cpu); nextThreadData = cpu->PeekIdleThread(); cpu->Remove(nextThreadData); diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index f1a5b5e377..08987fd49a 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -56,6 +56,7 @@ CPUEntry::CPUEntry() fMeasureTime(0) { B_INITIALIZE_RW_SPINLOCK(&fSchedulerModeLock); + B_INITIALIZE_SPINLOCK(&fQueueLock); } @@ -185,25 +186,26 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack) { SCHEDULER_ENTER_FUNCTION(); - CoreRunQueueLocker _(fCore); + int32 oldPriority = -1; + if (oldThread != NULL) + oldPriority = oldThread->GetEffectivePriority(); + + CPURunQueueLocker cpuLocker(this); - ThreadData* sharedThread = fCore->PeekThread(); ThreadData* pinnedThread = fRunQueue.PeekMaximum(); - - ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL); - int32 pinnedPriority = -1; if (pinnedThread != NULL) pinnedPriority = pinnedThread->GetEffectivePriority(); + CoreRunQueueLocker coreLocker(fCore); + + ThreadData* sharedThread = fCore->PeekThread(); + ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL); + int32 sharedPriority = -1; if (sharedThread != NULL) sharedPriority = sharedThread->GetEffectivePriority(); - int32 oldPriority = -1; - if (oldThread != NULL) - oldPriority = oldThread->GetEffectivePriority(); - int32 rest = std::max(pinnedPriority, sharedPriority); if (oldPriority > rest || (!putAtBack && oldPriority == rest)) return oldThread; @@ -213,6 +215,8 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack) return sharedThread; } + coreLocker.Unlock(); + Remove(pinnedThread); return pinnedThread; } diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 92de5d1daa..bd152644e7 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -60,6 +60,9 @@ public: inline void LockScheduler(); inline void UnlockScheduler(); + inline void LockRunQueue(); + inline void UnlockRunQueue(); + void PushFront(ThreadData* thread, int32 priority); void PushBack(ThreadData* thread, @@ -91,6 +94,7 @@ private: rw_spinlock fSchedulerModeLock; ThreadRunQueue fRunQueue; + spinlock fQueueLock; int32 fLoad; @@ -108,6 +112,22 @@ public: void Dump(); }; +class CPURunQueueLocking { +public: + inline bool Lock(CPUEntry* cpu) + { + cpu->LockRunQueue(); + return true; + } + + inline void Unlock(CPUEntry* cpu) + { + cpu->UnlockRunQueue(); + } +}; + +typedef AutoLocker CPURunQueueLocker; + class CoreEntry : public MinMaxHeapLinkImpl, public DoublyLinkedListLinkImpl { public: @@ -309,6 +329,22 @@ CPUEntry::UnlockScheduler() } +inline void +CPUEntry::LockRunQueue() +{ + SCHEDULER_ENTER_FUNCTION(); + acquire_spinlock(&fQueueLock); +} + + +inline void +CPUEntry::UnlockRunQueue() +{ + SCHEDULER_ENTER_FUNCTION(); + release_spinlock(&fQueueLock); +} + + /* static */ inline CPUEntry* CPUEntry::GetCPU(int32 cpu) { diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 7bea75bf41..d37d8bf5b2 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -234,16 +234,22 @@ ThreadData::PutBack() int32 priority = GetEffectivePriority(); - CoreRunQueueLocker _(fCore); - ASSERT(!fEnqueued); - fEnqueued = true; if (fThread->pinned_to_cpu > 0) { ASSERT(fThread->cpu != NULL); - CPUEntry* cpu = CPUEntry::GetCPU(fThread->cpu->cpu_num); + + CPURunQueueLocker _(cpu); + ASSERT(!fEnqueued); + fEnqueued = true; + cpu->PushFront(this, priority); - } else + } else { + CoreRunQueueLocker _(fCore); + ASSERT(!fEnqueued); + fEnqueued = true; + fCore->PushFront(this, priority); + } } @@ -258,16 +264,22 @@ ThreadData::Enqueue() int32 priority = GetEffectivePriority(); - CoreRunQueueLocker _(fCore); - ASSERT(!fEnqueued); - fEnqueued = true; if (fThread->pinned_to_cpu > 0) { ASSERT(fThread->previous_cpu != NULL); - CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num); + + CPURunQueueLocker _(cpu); + ASSERT(!fEnqueued); + fEnqueued = true; + cpu->PushBack(this, priority); - } else + } else { + CoreRunQueueLocker _(fCore); + ASSERT(!fEnqueued); + fEnqueued = true; + fCore->PushBack(this, priority); + } } @@ -276,20 +288,23 @@ ThreadData::Dequeue() { SCHEDULER_ENTER_FUNCTION(); + if (fThread->pinned_to_cpu > 0) { + ASSERT(fThread->previous_cpu != NULL); + CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num); + + CPURunQueueLocker _(cpu); + if (!fEnqueued) + return false; + cpu->Remove(this); + ASSERT(!fEnqueued); + return true; + } + CoreRunQueueLocker _(fCore); if (!fEnqueued) return false; - - if (fThread->pinned_to_cpu > 0) { - ASSERT(fThread->previous_cpu != NULL); - - CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num); - cpu->Remove(this); - } else { - ASSERT(fWentSleepCount < 1); - fCore->Remove(this); - } - + ASSERT(fWentSleepCount < 1); + fCore->Remove(this); ASSERT(!fEnqueued); return true; }