Aren/haiku
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

kernel: Use CPUSet in ICI code instead of cpu_mask_t

Browse Source
This commit is contained in:
Pawel Dziepak 2013-12-06 02:54:34 +01:00
parent 5f3798921f
commit 7629d527c5
7 changed files with 131 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
headers/private/kernel/arch/smp.h
View File

@ -11,9 +11,6 @@
struct kernel_args;
// must match MAX_BOOT_CPUS in platform_kernel_args.h
#define SMP_MAX_CPUS MAX_BOOT_CPUS
#ifdef __cplusplus
extern "C" {
#endif

83
headers/private/kernel/smp.h
View File

@ -9,8 +9,17 @@
#define KERNEL_SMP_H
#include <boot/kernel_args.h>
#include <kernel.h>
#include <KernelExport.h>
#include <string.h>
#define SMP_MAX_CPUS MAX_BOOT_CPUS
struct kernel_args;
@ -35,6 +44,26 @@ typedef uint32 cpu_mask_t;
typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3);
class CPUSet {
public:
CPUSet() { ClearAll(); }
inline void ClearAll();
inline void SetAll();
inline void SetBit(int32 cpu);
inline void ClearBit(int32 cpu);
inline bool GetBit(int32 cpu) const;
inline bool IsEmpty() const;
private:
static const int kArraySize = ROUNDUP(SMP_MAX_CPUS, 32) / 32;
uint32 fBitmap[kArraySize];
};
#ifdef __cplusplus
extern "C" {
@ -47,10 +76,10 @@ status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu);
status_t smp_init_post_generic_syscalls(void);
bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous);
void smp_wake_up_non_boot_cpus(void);
void smp_cpu_rendezvous(uint32 *var);
void smp_cpu_rendezvous(uint32* var);
void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3,
void *data_ptr, uint32 flags);
void smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data,
void smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
void smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
void *data_ptr, uint32 flags);
@ -68,6 +97,56 @@ int smp_intercpu_int_handler(int32 cpu);
#endif
inline void
CPUSet::ClearAll()
{
memset(fBitmap, 0, sizeof(fBitmap));
}
inline void
CPUSet::SetAll()
{
memset(fBitmap, ~uint8(0), sizeof(fBitmap));
}
inline void
CPUSet::SetBit(int32 cpu)
{
uint32* element = &fBitmap[cpu % kArraySize];
atomic_or(element, 1u << (cpu / kArraySize));
}
inline void
CPUSet::ClearBit(int32 cpu)
{
uint32* element = &fBitmap[cpu % kArraySize];
atomic_and(element, ~uint32(1u << (cpu / kArraySize)));
}
inline bool
CPUSet::GetBit(int32 cpu) const
{
int32* element = (int32*)&fBitmap[cpu % kArraySize];
return ((uint32)atomic_get(element) & (1u << (cpu / kArraySize))) != 0;
}
inline bool
CPUSet::IsEmpty() const
{
for (int i = 0; i < kArraySize; i++) {
if (fBitmap[i] != 0)
return false;
}
return true;
}
// Unless spinlock debug features are enabled, try to inline
// {acquire,release}_spinlock().
#if !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION

5
src/system/kernel/arch/x86/arch_thread.cpp
View File

@ -229,9 +229,8 @@ arch_thread_context_switch(Thread* from, Thread* to)
!= activePagingStructures) {
// update on which CPUs the address space is used
int cpu = cpuData->cpu_num;
atomic_and(&activePagingStructures->active_on_cpus,
~((uint32)1 << cpu));
atomic_or(&toPagingStructures->active_on_cpus, (uint32)1 << cpu);
activePagingStructures->active_on_cpus.ClearBit(cpu);
toPagingStructures->active_on_cpus.SetBit(cpu);
// assign the new paging structures to the CPU
toPagingStructures->AddReference();

3
src/system/kernel/arch/x86/paging/X86PagingStructures.cpp
View File

@ -9,8 +9,7 @@
X86PagingStructures::X86PagingStructures()
:
ref_count(1),
active_on_cpus(0)
ref_count(1)
{
}

4
src/system/kernel/arch/x86/paging/X86PagingStructures.h
View File

@ -14,11 +14,13 @@
#include <heap.h>
#include <smp.h>
struct X86PagingStructures : DeferredDeletable {
phys_addr_t pgdir_phys;
int32 ref_count;
int32 active_on_cpus;
CPUSet active_on_cpus;
// mask indicating on which CPUs the map is currently used
X86PagingStructures();

14
src/system/kernel/arch/x86/paging/X86VMTranslationMap.cpp
View File

@ -113,9 +113,10 @@ X86VMTranslationMap::Flush()
restore_interrupts(state);
int cpu = smp_get_current_cpu();
uint32 cpuMask = PagingStructures()->active_on_cpus
& ~((uint32)1 << cpu);
if (cpuMask != 0) {
CPUSet cpuMask = PagingStructures()->active_on_cpus;
cpuMask.ClearBit(cpu);
if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES,
0, 0, 0, NULL, SMP_MSG_FLAG_SYNC);
}
@ -132,9 +133,10 @@ X86VMTranslationMap::Flush()
SMP_MSG_FLAG_SYNC);
} else {
int cpu = smp_get_current_cpu();
uint32 cpuMask = PagingStructures()->active_on_cpus
& ~((uint32)1 << cpu);
if (cpuMask != 0) {
CPUSet cpuMask = PagingStructures()->active_on_cpus;
cpuMask.ClearBit(cpu);
if (!cpuMask.IsEmpty()) {
smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST,
(addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL,
SMP_MSG_FLAG_SYNC);

63
src/system/kernel/smp.cpp
View File

@ -80,7 +80,7 @@ struct smp_msg {
uint32 flags;
int32 ref_count;
int32 done;
uint32 proc_bitmap;
CPUSet proc_bitmap;
};
enum mailbox_source {
@ -90,7 +90,8 @@ enum mailbox_source {
static int32 sBootCPUSpin = 0;
static int32 sEarlyCPUCall = 0;
static int32 sEarlyCPUCallCount;
static CPUSet sEarlyCPUCallSet;
static void (*sEarlyCPUCallFunction)(void*, int);
void* sEarlyCPUCallCookie;
@ -294,7 +295,13 @@ dump_ici_message(int argc, char** argv)
kprintf(" flags: %" B_PRIx32 "\n", message->flags);
kprintf(" ref_count: %" B_PRIx32 "\n", message->ref_count);
kprintf(" done: %s\n", message->done == 1 ? "true" : "false");
kprintf(" proc_bitmap: %" B_PRIx32 "\n", message->proc_bitmap);
kprintf(" proc_bitmap: ");
for (int32 i = 0; i < sNumCPUs; i++) {
if (message->proc_bitmap.GetBit(i))
kprintf("%s%" B_PRId32, i != 0 ? ", " : "", i);
}
kprintf("\n");
return 0;
}
@ -809,14 +816,14 @@ check_for_message(int currentCPU, mailbox_source& sourceMailbox)
msg = sBroadcastMessages;
while (msg != NULL) {
if (CHECK_BIT(msg->proc_bitmap, currentCPU) != 0) {
if (!msg->proc_bitmap.GetBit(currentCPU)) {
// we have handled this one already
msg = msg->next;
continue;
}
// mark it so we wont try to process this one again
msg->proc_bitmap = SET_BIT(msg->proc_bitmap, currentCPU);
msg->proc_bitmap.ClearBit(currentCPU);
atomic_add(&gCPU[currentCPU].ici_counter, 1);
sourceMailbox = MAILBOX_BCAST;
@ -994,7 +1001,8 @@ static void
process_early_cpu_call(int32 cpu)
{
sEarlyCPUCallFunction(sEarlyCPUCallCookie, cpu);
atomic_and(&sEarlyCPUCall, ~(uint32)(1 << cpu));
sEarlyCPUCallSet.ClearBit(cpu);
atomic_add(&sEarlyCPUCallCount, 1);
}
@ -1005,14 +1013,13 @@ call_all_cpus_early(void (*function)(void*, int), void* cookie)
sEarlyCPUCallFunction = function;
sEarlyCPUCallCookie = cookie;
uint32 cpuMask = (1 << sNumCPUs) - 2;
// all CPUs but the boot cpu
atomic_set(&sEarlyCPUCall, cpuMask);
atomic_set(&sEarlyCPUCallCount, 1);
sEarlyCPUCallSet.SetAll();
sEarlyCPUCallSet.ClearBit(0);
// wait for all CPUs to finish
while ((atomic_get(&sEarlyCPUCall) & cpuMask) != 0)
cpu_pause();
while (atomic_get(&sEarlyCPUCallCount) < sNumCPUs)
cpu_wait(&sEarlyCPUCallCount, sNumCPUs);
}
function(cookie, 0);
@ -1099,24 +1106,16 @@ smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2,
void
smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data,
smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
addr_t data2, addr_t data3, void *dataPointer, uint32 flags)
{
if (!sICIEnabled)
return;
int currentCPU = smp_get_current_cpu();
cpuMask &= ~((cpu_mask_t)1 << currentCPU)
& (((cpu_mask_t)1 << sNumCPUs) - 1);
if (cpuMask == 0) {
panic("smp_send_multicast_ici(): 0 CPU mask");
return;
}
// count target CPUs
int32 targetCPUs = 0;
for (int32 i = 0; i < sNumCPUs; i++) {
if ((cpuMask & (cpu_mask_t)1 << i) != 0)
if (cpuMask.GetBit(i))
targetCPUs++;
}
@ -1131,9 +1130,17 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data,
msg->data_ptr = dataPointer;
msg->ref_count = targetCPUs;
msg->flags = flags;
msg->proc_bitmap = ~cpuMask;
msg->done = 0;
int currentCPU = smp_get_current_cpu();
msg->proc_bitmap = cpuMask;
msg->proc_bitmap.ClearBit(currentCPU);
if (msg->proc_bitmap.IsEmpty()) {
panic("smp_send_multicast_ici(): 0 CPU mask");
return;
}
// stick it in the broadcast mailbox
acquire_spinlock_nocheck(&sBroadcastMessageSpinlock);
msg->next = sBroadcastMessages;
@ -1142,7 +1149,7 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data,
atomic_add(&sBroadcastMessageCounter, 1);
for (int32 i = 0; i < sNumCPUs; i++) {
if ((cpuMask & (cpu_mask_t)1 << i) == 0)
if (!cpuMask.GetBit(i))
atomic_add(&gCPU[i].ici_counter, 1);
}
@ -1193,7 +1200,8 @@ smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
msg->data_ptr = dataPointer;
msg->ref_count = sNumCPUs - 1;
msg->flags = flags;
msg->proc_bitmap = SET_BIT(0, currentCPU);
msg->proc_bitmap.SetAll();
msg->proc_bitmap.ClearBit(currentCPU);
msg->done = 0;
TRACE("smp_send_broadcast_ici%d: inserting msg %p into broadcast "
@ -1258,7 +1266,8 @@ smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
msg->data_ptr = dataPointer;
msg->ref_count = sNumCPUs - 1;
msg->flags = flags;
msg->proc_bitmap = SET_BIT(0, currentCPU);
msg->proc_bitmap.SetAll();
msg->proc_bitmap.ClearBit(currentCPU);
msg->done = 0;
TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: inserting msg %p "
@ -1320,7 +1329,7 @@ smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous)
smp_cpu_rendezvous(rendezVous);
while (atomic_get(&sBootCPUSpin) == 0) {
if ((atomic_get(&sEarlyCPUCall) & (1 << cpu)) != 0)
if (sEarlyCPUCallSet.GetBit(cpu))
process_early_cpu_call(cpu);
cpu_pause();