diff --git a/headers/private/kernel/arch/smp.h b/headers/private/kernel/arch/smp.h index e8a66b97da..f85fcf136f 100644 --- a/headers/private/kernel/arch/smp.h +++ b/headers/private/kernel/arch/smp.h @@ -11,9 +11,6 @@ struct kernel_args; -// must match MAX_BOOT_CPUS in platform_kernel_args.h -#define SMP_MAX_CPUS MAX_BOOT_CPUS - #ifdef __cplusplus extern "C" { #endif diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 28a1ccea66..0452592abb 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -9,8 +9,17 @@ #define KERNEL_SMP_H +#include +#include + #include +#include + + +#define SMP_MAX_CPUS MAX_BOOT_CPUS + + struct kernel_args; @@ -35,6 +44,26 @@ typedef uint32 cpu_mask_t; typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3); +class CPUSet { +public: + CPUSet() { ClearAll(); } + + inline void ClearAll(); + inline void SetAll(); + + inline void SetBit(int32 cpu); + inline void ClearBit(int32 cpu); + + inline bool GetBit(int32 cpu) const; + + inline bool IsEmpty() const; + +private: + static const int kArraySize = ROUNDUP(SMP_MAX_CPUS, 32) / 32; + + uint32 fBitmap[kArraySize]; +}; + #ifdef __cplusplus extern "C" { @@ -47,10 +76,10 @@ status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu); status_t smp_init_post_generic_syscalls(void); bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous); void smp_wake_up_non_boot_cpus(void); -void smp_cpu_rendezvous(uint32 *var); +void smp_cpu_rendezvous(uint32* var); void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags); -void smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, +void smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags); void smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags); @@ -68,6 +97,56 @@ int smp_intercpu_int_handler(int32 cpu); #endif +inline void +CPUSet::ClearAll() +{ + memset(fBitmap, 0, sizeof(fBitmap)); +} + + +inline void +CPUSet::SetAll() +{ + memset(fBitmap, ~uint8(0), sizeof(fBitmap)); +} + + +inline void +CPUSet::SetBit(int32 cpu) +{ + uint32* element = &fBitmap[cpu % kArraySize]; + atomic_or(element, 1u << (cpu / kArraySize)); +} + + +inline void +CPUSet::ClearBit(int32 cpu) +{ + uint32* element = &fBitmap[cpu % kArraySize]; + atomic_and(element, ~uint32(1u << (cpu / kArraySize))); +} + + +inline bool +CPUSet::GetBit(int32 cpu) const +{ + int32* element = (int32*)&fBitmap[cpu % kArraySize]; + return ((uint32)atomic_get(element) & (1u << (cpu / kArraySize))) != 0; +} + + +inline bool +CPUSet::IsEmpty() const +{ + for (int i = 0; i < kArraySize; i++) { + if (fBitmap[i] != 0) + return false; + } + + return true; +} + + // Unless spinlock debug features are enabled, try to inline // {acquire,release}_spinlock(). #if !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION diff --git a/src/system/kernel/arch/x86/arch_thread.cpp b/src/system/kernel/arch/x86/arch_thread.cpp index e7b1298a59..9fbbaa3ad3 100644 --- a/src/system/kernel/arch/x86/arch_thread.cpp +++ b/src/system/kernel/arch/x86/arch_thread.cpp @@ -229,9 +229,8 @@ arch_thread_context_switch(Thread* from, Thread* to) != activePagingStructures) { // update on which CPUs the address space is used int cpu = cpuData->cpu_num; - atomic_and(&activePagingStructures->active_on_cpus, - ~((uint32)1 << cpu)); - atomic_or(&toPagingStructures->active_on_cpus, (uint32)1 << cpu); + activePagingStructures->active_on_cpus.ClearBit(cpu); + toPagingStructures->active_on_cpus.SetBit(cpu); // assign the new paging structures to the CPU toPagingStructures->AddReference(); diff --git a/src/system/kernel/arch/x86/paging/X86PagingStructures.cpp b/src/system/kernel/arch/x86/paging/X86PagingStructures.cpp index 4a2d9ee9ad..2844391341 100644 --- a/src/system/kernel/arch/x86/paging/X86PagingStructures.cpp +++ b/src/system/kernel/arch/x86/paging/X86PagingStructures.cpp @@ -9,8 +9,7 @@ X86PagingStructures::X86PagingStructures() : - ref_count(1), - active_on_cpus(0) + ref_count(1) { } diff --git a/src/system/kernel/arch/x86/paging/X86PagingStructures.h b/src/system/kernel/arch/x86/paging/X86PagingStructures.h index 6fd882b39f..2744b0cef0 100644 --- a/src/system/kernel/arch/x86/paging/X86PagingStructures.h +++ b/src/system/kernel/arch/x86/paging/X86PagingStructures.h @@ -14,11 +14,13 @@ #include +#include + struct X86PagingStructures : DeferredDeletable { phys_addr_t pgdir_phys; int32 ref_count; - int32 active_on_cpus; + CPUSet active_on_cpus; // mask indicating on which CPUs the map is currently used X86PagingStructures(); diff --git a/src/system/kernel/arch/x86/paging/X86VMTranslationMap.cpp b/src/system/kernel/arch/x86/paging/X86VMTranslationMap.cpp index a9768653ef..5ed05001ea 100644 --- a/src/system/kernel/arch/x86/paging/X86VMTranslationMap.cpp +++ b/src/system/kernel/arch/x86/paging/X86VMTranslationMap.cpp @@ -113,9 +113,10 @@ X86VMTranslationMap::Flush() restore_interrupts(state); int cpu = smp_get_current_cpu(); - uint32 cpuMask = PagingStructures()->active_on_cpus - & ~((uint32)1 << cpu); - if (cpuMask != 0) { + CPUSet cpuMask = PagingStructures()->active_on_cpus; + cpuMask.ClearBit(cpu); + + if (!cpuMask.IsEmpty()) { smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES, 0, 0, 0, NULL, SMP_MSG_FLAG_SYNC); } @@ -132,9 +133,10 @@ X86VMTranslationMap::Flush() SMP_MSG_FLAG_SYNC); } else { int cpu = smp_get_current_cpu(); - uint32 cpuMask = PagingStructures()->active_on_cpus - & ~((uint32)1 << cpu); - if (cpuMask != 0) { + CPUSet cpuMask = PagingStructures()->active_on_cpus; + cpuMask.ClearBit(cpu); + + if (!cpuMask.IsEmpty()) { smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST, (addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, SMP_MSG_FLAG_SYNC); diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index e54bc38170..63d280569d 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -80,7 +80,7 @@ struct smp_msg { uint32 flags; int32 ref_count; int32 done; - uint32 proc_bitmap; + CPUSet proc_bitmap; }; enum mailbox_source { @@ -90,7 +90,8 @@ enum mailbox_source { static int32 sBootCPUSpin = 0; -static int32 sEarlyCPUCall = 0; +static int32 sEarlyCPUCallCount; +static CPUSet sEarlyCPUCallSet; static void (*sEarlyCPUCallFunction)(void*, int); void* sEarlyCPUCallCookie; @@ -294,7 +295,13 @@ dump_ici_message(int argc, char** argv) kprintf(" flags: %" B_PRIx32 "\n", message->flags); kprintf(" ref_count: %" B_PRIx32 "\n", message->ref_count); kprintf(" done: %s\n", message->done == 1 ? "true" : "false"); - kprintf(" proc_bitmap: %" B_PRIx32 "\n", message->proc_bitmap); + + kprintf(" proc_bitmap: "); + for (int32 i = 0; i < sNumCPUs; i++) { + if (message->proc_bitmap.GetBit(i)) + kprintf("%s%" B_PRId32, i != 0 ? ", " : "", i); + } + kprintf("\n"); return 0; } @@ -809,14 +816,14 @@ check_for_message(int currentCPU, mailbox_source& sourceMailbox) msg = sBroadcastMessages; while (msg != NULL) { - if (CHECK_BIT(msg->proc_bitmap, currentCPU) != 0) { + if (!msg->proc_bitmap.GetBit(currentCPU)) { // we have handled this one already msg = msg->next; continue; } // mark it so we wont try to process this one again - msg->proc_bitmap = SET_BIT(msg->proc_bitmap, currentCPU); + msg->proc_bitmap.ClearBit(currentCPU); atomic_add(&gCPU[currentCPU].ici_counter, 1); sourceMailbox = MAILBOX_BCAST; @@ -994,7 +1001,8 @@ static void process_early_cpu_call(int32 cpu) { sEarlyCPUCallFunction(sEarlyCPUCallCookie, cpu); - atomic_and(&sEarlyCPUCall, ~(uint32)(1 << cpu)); + sEarlyCPUCallSet.ClearBit(cpu); + atomic_add(&sEarlyCPUCallCount, 1); } @@ -1005,14 +1013,13 @@ call_all_cpus_early(void (*function)(void*, int), void* cookie) sEarlyCPUCallFunction = function; sEarlyCPUCallCookie = cookie; - uint32 cpuMask = (1 << sNumCPUs) - 2; - // all CPUs but the boot cpu - - atomic_set(&sEarlyCPUCall, cpuMask); + atomic_set(&sEarlyCPUCallCount, 1); + sEarlyCPUCallSet.SetAll(); + sEarlyCPUCallSet.ClearBit(0); // wait for all CPUs to finish - while ((atomic_get(&sEarlyCPUCall) & cpuMask) != 0) - cpu_pause(); + while (atomic_get(&sEarlyCPUCallCount) < sNumCPUs) + cpu_wait(&sEarlyCPUCallCount, sNumCPUs); } function(cookie, 0); @@ -1099,24 +1106,16 @@ smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, void -smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, +smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, addr_t data2, addr_t data3, void *dataPointer, uint32 flags) { if (!sICIEnabled) return; - int currentCPU = smp_get_current_cpu(); - cpuMask &= ~((cpu_mask_t)1 << currentCPU) - & (((cpu_mask_t)1 << sNumCPUs) - 1); - if (cpuMask == 0) { - panic("smp_send_multicast_ici(): 0 CPU mask"); - return; - } - // count target CPUs int32 targetCPUs = 0; for (int32 i = 0; i < sNumCPUs; i++) { - if ((cpuMask & (cpu_mask_t)1 << i) != 0) + if (cpuMask.GetBit(i)) targetCPUs++; } @@ -1131,9 +1130,17 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, msg->data_ptr = dataPointer; msg->ref_count = targetCPUs; msg->flags = flags; - msg->proc_bitmap = ~cpuMask; msg->done = 0; + int currentCPU = smp_get_current_cpu(); + msg->proc_bitmap = cpuMask; + msg->proc_bitmap.ClearBit(currentCPU); + if (msg->proc_bitmap.IsEmpty()) { + panic("smp_send_multicast_ici(): 0 CPU mask"); + return; + } + + // stick it in the broadcast mailbox acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); msg->next = sBroadcastMessages; @@ -1142,7 +1149,7 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, atomic_add(&sBroadcastMessageCounter, 1); for (int32 i = 0; i < sNumCPUs; i++) { - if ((cpuMask & (cpu_mask_t)1 << i) == 0) + if (!cpuMask.GetBit(i)) atomic_add(&gCPU[i].ici_counter, 1); } @@ -1193,7 +1200,8 @@ smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3, msg->data_ptr = dataPointer; msg->ref_count = sNumCPUs - 1; msg->flags = flags; - msg->proc_bitmap = SET_BIT(0, currentCPU); + msg->proc_bitmap.SetAll(); + msg->proc_bitmap.ClearBit(currentCPU); msg->done = 0; TRACE("smp_send_broadcast_ici%d: inserting msg %p into broadcast " @@ -1258,7 +1266,8 @@ smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message, msg->data_ptr = dataPointer; msg->ref_count = sNumCPUs - 1; msg->flags = flags; - msg->proc_bitmap = SET_BIT(0, currentCPU); + msg->proc_bitmap.SetAll(); + msg->proc_bitmap.ClearBit(currentCPU); msg->done = 0; TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: inserting msg %p " @@ -1320,7 +1329,7 @@ smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous) smp_cpu_rendezvous(rendezVous); while (atomic_get(&sBootCPUSpin) == 0) { - if ((atomic_get(&sEarlyCPUCall) & (1 << cpu)) != 0) + if (sEarlyCPUCallSet.GetBit(cpu)) process_early_cpu_call(cpu); cpu_pause();