From a08b40d4087b35c586959dc7da44035171d4cf15 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 23 Dec 2013 20:12:03 +0100 Subject: [PATCH] scheduler: Encapsulate CPUEntry fields --- src/system/kernel/scheduler/low_latency.cpp | 2 +- src/system/kernel/scheduler/power_saving.cpp | 4 +- src/system/kernel/scheduler/scheduler.cpp | 91 +++++++++------ src/system/kernel/scheduler/scheduler_cpu.cpp | 107 +++++++++++++++--- src/system/kernel/scheduler/scheduler_cpu.h | 84 +++++++++++++- .../kernel/scheduler/scheduler_thread.cpp | 4 +- .../kernel/scheduler/scheduler_thread.h | 16 ++- 7 files changed, 236 insertions(+), 72 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 74cddd6801..dd6399c526 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -132,7 +132,7 @@ rebalance_irqs(bool idle) other = gCoreHighLoadHeap.PeekMinimum(); coreLocker.Unlock(); - int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber; + int32 newCPU = other->CPUHeap()->PeekMinimum()->ID(); ASSERT(other != NULL); diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index d6f3a7d5bf..33bd769ad3 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -164,7 +164,7 @@ pack_irqs() irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); locker.Unlock(); - int32 newCPU = smallTaskCore->CPUHeap()->PeekMinimum()->fCPUNumber; + int32 newCPU = smallTaskCore->CPUHeap()->PeekMinimum()->ID(); if (newCPU != cpu->cpu_num) assign_io_interrupt_to_cpu(irq->irq, newCPU); @@ -207,7 +207,7 @@ rebalance_irqs(bool idle) coreLocker.Unlock(); if (other == NULL) return; - int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber; + int32 newCPU = other->CPUHeap()->PeekMinimum()->ID(); CoreEntry* core = CoreEntry::GetCore(smp_get_current_cpu()); if (other == core) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 97e2e5256d..a4797efc09 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -39,26 +39,60 @@ namespace Scheduler { -class SchedulerModeLocker : public ReadSpinLocker { +class SchedulerModeLocking { +public: + bool Lock(int* /* lockable */) + { + CPUEntry::GetCPU(smp_get_current_cpu())->EnterScheduler(); + return true; + } + + void Unlock(int* /* lockable */) + { + CPUEntry::GetCPU(smp_get_current_cpu())->ExitScheduler(); + } +}; + +class SchedulerModeLocker : + public AutoLocker { public: SchedulerModeLocker(bool alreadyLocked = false, bool lockIfNotLocked = true) : - ReadSpinLocker(gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, - alreadyLocked, lockIfNotLocked) + AutoLocker(NULL, alreadyLocked, + lockIfNotLocked) { } }; -class InterruptsSchedulerModeLocker : public InterruptsReadSpinLocker { +class InterruptsSchedulerModeLocking { +public: + bool Lock(int* lockable) + { + *lockable = disable_interrupts(); + CPUEntry::GetCPU(smp_get_current_cpu())->EnterScheduler(); + return true; + } + + void Unlock(int* lockable) + { + CPUEntry::GetCPU(smp_get_current_cpu())->ExitScheduler(); + restore_interrupts(*lockable); + } +}; + +class InterruptsSchedulerModeLocker : + public AutoLocker { public: InterruptsSchedulerModeLocker(bool alreadyLocked = false, bool lockIfNotLocked = true) : - InterruptsReadSpinLocker( - gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, - alreadyLocked, lockIfNotLocked) + AutoLocker(&fState, alreadyLocked, + lockIfNotLocked) { } + +private: + int fState; }; class InterruptsBigSchedulerLocking { @@ -67,14 +101,14 @@ public: { *lockable = disable_interrupts(); for (int32 i = 0; i < smp_get_num_cpus(); i++) - acquire_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); + CPUEntry::GetCPU(i)->LockScheduler(); return true; } void Unlock(int* lockable) { for (int32 i = 0; i < smp_get_num_cpus(); i++) - release_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); + CPUEntry::GetCPU(i)->UnlockScheduler(); restore_interrupts(*lockable); } }; @@ -193,10 +227,10 @@ enqueue(Thread* thread, bool newOne) if (threadPriority > heapPriority || (threadPriority == heapPriority && rescheduleNeeded)) { - if (targetCPU->fCPUNumber == smp_get_current_cpu()) - gCPU[targetCPU->fCPUNumber].invoke_scheduler = true; + if (targetCPU->ID() == smp_get_current_cpu()) + gCPU[targetCPU->ID()].invoke_scheduler = true; else { - smp_send_ici(targetCPU->fCPUNumber, SMP_MSG_RESCHEDULE, 0, 0, 0, + smp_send_ici(targetCPU->ID(), SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, SMP_MSG_FLAG_ASYNC); } } @@ -484,8 +518,8 @@ reschedule(int32 nextState) if (!thread_is_idle_thread(oldThread)) { CoreRunQueueLocker _(core); - nextThreadData = cpu->fRunQueue.GetHead(B_IDLE_PRIORITY); - cpu->fRunQueue.Remove(nextThreadData); + nextThreadData = cpu->PeekIdleThread(); + cpu->Remove(nextThreadData); nextThreadData->fEnqueued = false; putOldThreadAtBack = oldThread->pinned_to_cpu == 0; @@ -662,14 +696,13 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) gCurrentMode->set_cpu_enabled(cpuID, enabled); CPUEntry* cpu = &gCPUEntries[cpuID]; - CoreEntry* core = cpu->fCore; + CoreEntry* core = cpu->Core(); int32 oldCPUCount = core->CPUCount(); ASSERT(oldCPUCount >= 0); - if (enabled) { - cpu->fLoad = 0; - core->AddCPU(cpu); - } else { + if (enabled) + cpu->Start(); + else { cpu->UpdatePriority(B_IDLE_PRIORITY); ThreadEnqueuer enqueuer; @@ -679,21 +712,7 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) gCPU[cpuID].disabled = !enabled; if (!enabled) { - cpu_ent* entry = &gCPU[cpuID]; - - // get rid of irqs - SpinLocker locker(entry->irqs_lock); - irq_assignment* irq - = (irq_assignment*)list_get_first_item(&entry->irqs); - while (irq != NULL) { - locker.Unlock(); - - assign_io_interrupt_to_cpu(irq->irq, -1); - - locker.Lock(); - irq = (irq_assignment*)list_get_first_item(&entry->irqs); - } - locker.Unlock(); + cpu->Stop(); // don't wait until the thread quantum ends if (smp_get_current_cpu() != cpuID) { @@ -808,9 +827,7 @@ init() package->Init(sCPUToPackage[i]); core->Init(sCPUToCore[i], package); - - gCPUEntries[i].fCPUNumber = i; - gCPUEntries[i].fCore = core; + gCPUEntries[i].Init(i, core); core->AddCPU(&gCPUEntries[i]); } diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index 896447e765..0899a9b4d0 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -18,6 +18,7 @@ using namespace Scheduler; class Scheduler::DebugDumper { public: + static void DumpCPURunQueue(CPUEntry* cpu); static void DumpCoreRunQueue(CoreEntry* core); static void DumpIdleCoresInPackage(PackageEntry* package); @@ -58,6 +59,78 @@ CPUEntry::CPUEntry() } +void +CPUEntry::Init(int32 id, CoreEntry* core) +{ + fCPUNumber = id; + fCore = core; +} + + +void +CPUEntry::Start() +{ + fLoad = 0; + fCore->AddCPU(this); +} + + +void +CPUEntry::Stop() +{ + cpu_ent* entry = &gCPU[fCPUNumber]; + + // get rid of irqs + SpinLocker locker(entry->irqs_lock); + irq_assignment* irq + = (irq_assignment*)list_get_first_item(&entry->irqs); + while (irq != NULL) { + locker.Unlock(); + + assign_io_interrupt_to_cpu(irq->irq, -1); + + locker.Lock(); + irq = (irq_assignment*)list_get_first_item(&entry->irqs); + } + locker.Unlock(); +} + + +void +CPUEntry::PushFront(ThreadData* thread, int32 priority) +{ + fRunQueue.PushFront(thread, priority); +} + + +void +CPUEntry::PushBack(ThreadData* thread, int32 priority) +{ + fRunQueue.PushBack(thread, priority); +} + + +void +CPUEntry::Remove(ThreadData* thread) +{ + fRunQueue.Remove(thread); +} + + +inline ThreadData* +CPUEntry::PeekThread() const +{ + return fRunQueue.PeekMaximum(); +} + + +ThreadData* +CPUEntry::PeekIdleThread() const +{ + return fRunQueue.GetHead(B_IDLE_PRIORITY); +} + + void CPUEntry::UpdatePriority(int32 priority) { @@ -213,10 +286,10 @@ CPUPriorityHeap::Dump() kprintf("cpu priority load\n"); CPUEntry* entry = PeekMinimum(); while (entry) { - int32 cpu = entry->fCPUNumber; + int32 cpu = entry->ID(); int32 key = GetKey(entry); kprintf("%3" B_PRId32 " %8" B_PRId32 " %3" B_PRId32 "%%\n", cpu, key, - entry->fLoad / 10); + entry->GetLoad() / 10); RemoveMinimum(); sDebugCPUHeap.Insert(entry, key); @@ -406,8 +479,8 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) ASSERT(fCPUHeap.PeekMaximum() == cpu); fCPUHeap.RemoveMaximum(); - ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad); - fLoad -= cpu->fLoad; + ASSERT(cpu->GetLoad() >= 0 && cpu->GetLoad() <= kMaxLoad); + fLoad -= cpu->GetLoad(); ASSERT(fLoad >= 0); } @@ -533,6 +606,19 @@ PackageEntry::RemoveIdleCore(CoreEntry* core) } +/* static */ void +DebugDumper::DumpCPURunQueue(CPUEntry* cpu) +{ + ThreadRunQueue::ConstIterator iterator = cpu->fRunQueue.GetConstIterator(); + + if (iterator.HasNext() + && !thread_is_idle_thread(iterator.Next()->GetThread())) { + kprintf("\nCPU %" B_PRId32 " run queue:\n", cpu->ID()); + cpu->fRunQueue.Dump(); + } +} + + /* static */ void DebugDumper::DumpCoreRunQueue(CoreEntry* core) { @@ -570,17 +656,8 @@ dump_run_queue(int argc, char **argv) DebugDumper::DumpCoreRunQueue(&gCoreEntries[i]); } - for (int32 i = 0; i < cpuCount; i++) { - CPUEntry* cpu = &gCPUEntries[i]; - ThreadRunQueue::ConstIterator iterator - = cpu->fRunQueue.GetConstIterator(); - - if (iterator.HasNext() - && !thread_is_idle_thread(iterator.Next()->GetThread())) { - kprintf("\nCPU %" B_PRId32 " run queue:\n", i); - cpu->fRunQueue.Dump(); - } - } + for (int32 i = 0; i < cpuCount; i++) + DebugDumper::DumpCPURunQueue(&gCPUEntries[i]); return 0; } diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 6df157ea13..9480619ad0 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -40,11 +40,38 @@ public: void Dump() const; }; -struct CPUEntry : public MinMaxHeapLinkImpl { +class CPUEntry : public MinMaxHeapLinkImpl { +public: CPUEntry(); + void Init(int32 id, CoreEntry* core); + + inline int32 ID() const { return fCPUNumber; } + inline CoreEntry* Core() const { return fCore; } + + void Start(); + void Stop(); + + inline void EnterScheduler(); + inline void ExitScheduler(); + + inline void LockScheduler(); + inline void UnlockScheduler(); + + void PushFront(ThreadData* thread, + int32 priority); + void PushBack(ThreadData* thread, + int32 priority); + void Remove(ThreadData* thread); + inline ThreadData* PeekThread() const; + ThreadData* PeekIdleThread() const; + void UpdatePriority(int32 priority); + inline void IncreaseActiveTime( + bigtime_t activeTime); + + inline int32 GetLoad() const { return fLoad; } void ComputeLoad(); ThreadData* ChooseNextThread(ThreadData* oldThread, @@ -53,6 +80,12 @@ struct CPUEntry : public MinMaxHeapLinkImpl { void TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData); + static inline CPUEntry* GetCPU(int32 cpu); + +private: + inline void _RequestPerformanceLevel( + ThreadData* threadData); + int32 fCPUNumber; CoreEntry* fCore; @@ -65,10 +98,7 @@ struct CPUEntry : public MinMaxHeapLinkImpl { bigtime_t fMeasureActiveTime; bigtime_t fMeasureTime; -private: - inline void _RequestPerformanceLevel( - ThreadData* threadData); - + friend class DebugDumper; } CACHE_LINE_ALIGN; class CPUPriorityHeap : public MinMaxHeap { @@ -245,6 +275,48 @@ extern rw_spinlock gIdlePackageLock; extern int32 gPackageCount; +inline void +CPUEntry::EnterScheduler() +{ + acquire_read_spinlock(&fSchedulerModeLock); +} + + +inline void +CPUEntry::ExitScheduler() +{ + release_read_spinlock(&fSchedulerModeLock); +} + + +inline void +CPUEntry::LockScheduler() +{ + acquire_write_spinlock(&fSchedulerModeLock); +} + + +inline void +CPUEntry::UnlockScheduler() +{ + release_write_spinlock(&fSchedulerModeLock); +} + + +inline void +CPUEntry::IncreaseActiveTime(bigtime_t activeTime) +{ + fMeasureActiveTime += activeTime; +} + + +/* static */ inline CPUEntry* +CPUEntry::GetCPU(int32 cpu) +{ + return &gCPUEntries[cpu]; +} + + inline void CoreEntry::LockCPUHeap() { @@ -321,7 +393,7 @@ CoreEntry::StarvationCounter() const /* static */ inline CoreEntry* CoreEntry::GetCore(int32 cpu) { - return gCPUEntries[cpu].fCore; + return gCPUEntries[cpu].Core(); } diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 5b933a5fbe..3bcb4d79be 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -77,7 +77,7 @@ ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU) bool rescheduleNeeded = false; if (targetCore == NULL && targetCPU != NULL) - targetCore = targetCPU->fCore; + targetCore = targetCPU->Core(); else if (targetCore != NULL && targetCPU == NULL) targetCPU = _ChooseCPU(targetCore, rescheduleNeeded); else if (targetCore == NULL && targetCPU == NULL) { @@ -136,7 +136,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const if (fThread->previous_cpu != NULL) { CPUEntry* previousCPU = &gCPUEntries[fThread->previous_cpu->cpu_num]; - if (previousCPU->fCore == core) { + if (previousCPU->Core() == core) { CoreCPUHeapLocker _(core); if (CPUPriorityHeap::GetKey(previousCPU) < threadPriority) { previousCPU->UpdatePriority(threadPriority); diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 2a49b68925..f0d06903c1 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -203,8 +203,8 @@ ThreadData::PutBack() if (fThread->pinned_to_cpu > 0) { ASSERT(fThread->cpu != NULL); - CPUEntry* cpu = &gCPUEntries[fThread->cpu->cpu_num]; - cpu->fRunQueue.PushFront(this, priority); + CPUEntry* cpu = CPUEntry::GetCPU(fThread->cpu->cpu_num); + cpu->PushFront(this, priority); } else fCore->PushFront(this, priority); fCore->UnlockRunQueue(); @@ -226,8 +226,8 @@ ThreadData::Enqueue() if (fThread->pinned_to_cpu > 0) { ASSERT(fThread->previous_cpu != NULL); - CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num]; - cpu->fRunQueue.PushBack(this, priority); + CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num); + cpu->PushBack(this, priority); } else fCore->PushBack(this, priority); } @@ -244,8 +244,8 @@ ThreadData::Dequeue() if (fThread->pinned_to_cpu > 0) { ASSERT(fThread->previous_cpu != NULL); - CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num]; - cpu->fRunQueue.Remove(this); + CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num); + cpu->Remove(this); } else { ASSERT(fWentSleepCount < 1); fCore->Remove(this, fWentSleepCount == 0); @@ -259,10 +259,8 @@ inline void ThreadData::UpdateActivity(bigtime_t active) { fMeasureActiveTime += active; - gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; - + CPUEntry::GetCPU(smp_get_current_cpu())->IncreaseActiveTime(active); fCore->IncreaseActiveTime(active); - }