Remove <machine/atomic.h>; use <sys/atomic.h> instead.

Add <powerpc/cpuset.h> (for mpc85xx pmap).
Add some initial MP code for mpc85xx
Rework ipi code to be common across all ppcs
Change PPC to keep curlwp in %r13 while in the kernel.
Move astpending from cpu_info to mdlwp
Improve cpu_need_resched to be more MP friendly.

distrib/sets/lists/comp/ad.powerpc

@@ -1,4 +1,4 @@
-# $NetBSD: ad.powerpc,v 1.49 2011/04/15 19:31:06 joerg Exp $
+# $NetBSD: ad.powerpc,v 1.50 2011/06/05 16:52:22 matt Exp $
 ./usr/bin/elf2aout				comp-sysutil-bin
 ./usr/bin/psim					comp-debug-bin		gdb,gdb=6
 ./usr/include/altivec.h				comp-obsolete		obsolete
@@ -8,7 +8,7 @@
 ./usr/include/powerpc/ansi.h			comp-c-include
 ./usr/include/powerpc/aout_machdep.h		comp-c-include
 ./usr/include/powerpc/asm.h			comp-c-include
-./usr/include/powerpc/atomic.h			comp-c-include
+./usr/include/powerpc/atomic.h			comp-obsolete		obsolete
 ./usr/include/powerpc/bat.h			comp-obsolete		obsolete
 ./usr/include/powerpc/bswap.h			comp-c-include
 ./usr/include/powerpc/bus.h			comp-obsolete		obsolete

distrib/sets/lists/comp/md.amigappc

@@ -1,11 +1,11 @@
-# $NetBSD: md.amigappc,v 1.6 2010/03/02 21:54:06 matt Exp $
+# $NetBSD: md.amigappc,v 1.7 2011/06/05 16:52:22 matt Exp $
 ./usr/include/ieeefp.h				comp-c-include
 ./usr/include/amigappc				comp-c-include
 ./usr/include/amigappc/_G_config.h		comp-obsolete		obsolete
 ./usr/include/amigappc/ansi.h			comp-c-include
 ./usr/include/amigappc/aout_machdep.h		comp-c-include
 ./usr/include/amigappc/asm.h			comp-c-include
-./usr/include/amigappc/atomic.h			comp-c-include
+./usr/include/amigappc/atomic.h			comp-obsolete		obsolete
 #XXX./usr/include/amigappc/autoconf.h		comp-c-include
 ./usr/include/amigappc/bat.h			comp-obsolete		obsolete
 #XXX./usr/include/amigappc/bus.h			comp-c-include

distrib/sets/lists/comp/md.bebox

@@ -1,10 +1,10 @@
-# $NetBSD: md.bebox,v 1.32 2010/03/14 23:19:18 mrg Exp $
+# $NetBSD: md.bebox,v 1.33 2011/06/05 16:52:22 matt Exp $
 ./usr/include/bebox				comp-c-include
 ./usr/include/bebox/_G_config.h			comp-obsolete		obsolete
 ./usr/include/bebox/ansi.h			comp-c-include
 ./usr/include/bebox/aout_machdep.h		comp-c-include
 ./usr/include/bebox/asm.h			comp-c-include
-./usr/include/bebox/atomic.h			comp-c-include
+./usr/include/bebox/atomic.h			comp-obsolete		obsolete
 ./usr/include/bebox/bat.h			comp-obsolete		obsolete
 ./usr/include/bebox/bootinfo.h			comp-c-include
 ./usr/include/bebox/bswap.h			comp-c-include

distrib/sets/lists/comp/md.evbppc

@@ -1,10 +1,10 @@
-# $NetBSD: md.evbppc,v 1.10 2010/03/02 21:54:06 matt Exp $
+# $NetBSD: md.evbppc,v 1.11 2011/06/05 16:52:22 matt Exp $
 ./usr/include/evbppc				comp-c-include
 ./usr/include/evbppc/_G_config.h		comp-obsolete		obsolete
 ./usr/include/evbppc/ansi.h			comp-c-include
 ./usr/include/evbppc/aout_machdep.h		comp-c-include
 ./usr/include/evbppc/asm.h			comp-c-include
-./usr/include/evbppc/atomic.h			comp-c-include
+./usr/include/evbppc/atomic.h			comp-obsolete		obsolete
 ./usr/include/evbppc/bswap.h			comp-c-include
 ./usr/include/evbppc/bus.h			comp-obsolete		obsolete
 ./usr/include/evbppc/cdefs.h			comp-c-include

distrib/sets/lists/comp/md.ibmnws

@@ -1,10 +1,10 @@
-# $NetBSD: md.ibmnws,v 1.8 2010/03/02 21:54:06 matt Exp $
+# $NetBSD: md.ibmnws,v 1.9 2011/06/05 16:52:22 matt Exp $
 ./usr/include/ibmnws				comp-c-include
 ./usr/include/ibmnws/_G_config.h		comp-obsolete		obsolete
 ./usr/include/ibmnws/ansi.h			comp-c-include
 ./usr/include/ibmnws/aout_machdep.h		comp-c-include
 ./usr/include/ibmnws/asm.h			comp-c-include
-./usr/include/ibmnws/atomic.h			comp-c-include
+./usr/include/ibmnws/atomic.h			comp-obsolete		obsolete
 ./usr/include/ibmnws/bat.h			comp-obsolete		obsolete
 ./usr/include/ibmnws/bswap.h			comp-c-include
 ./usr/include/ibmnws/bus.h			comp-obsolete		obsolete

distrib/sets/lists/comp/md.macppc

@@ -1,4 +1,4 @@
-# $NetBSD: md.macppc,v 1.33 2010/03/02 21:54:06 matt Exp $
+# $NetBSD: md.macppc,v 1.34 2011/06/05 16:52:22 matt Exp $
 ./usr/include/ieeefp.h				comp-c-include
 ./usr/include/macppc				comp-c-include
 ./usr/include/macppc/_G_config.h		comp-obsolete		obsolete
@@ -7,7 +7,7 @@
 ./usr/include/macppc/aout_machdep.h		comp-c-include
 ./usr/include/macppc/apmvar.h			comp-c-include
 ./usr/include/macppc/asm.h			comp-c-include
-./usr/include/macppc/atomic.h			comp-c-include
+./usr/include/macppc/atomic.h			comp-obsolete		obsolete
 ./usr/include/macppc/autoconf.h			comp-c-include
 ./usr/include/macppc/bat.h			comp-obsolete		obsolete
 ./usr/include/macppc/bswap.h			comp-c-include

distrib/sets/lists/comp/md.mvmeppc

@@ -1,11 +1,11 @@
-# $NetBSD: md.mvmeppc,v 1.12 2010/03/02 21:54:06 matt Exp $
+# $NetBSD: md.mvmeppc,v 1.13 2011/06/05 16:52:22 matt Exp $
 ./usr/include/ieeefp.h				comp-c-include
 ./usr/include/mvmeppc				comp-c-include
 ./usr/include/mvmeppc/_G_config.h		comp-obsolete		obsolete
 ./usr/include/mvmeppc/ansi.h			comp-c-include
 ./usr/include/mvmeppc/aout_machdep.h		comp-c-include
 ./usr/include/mvmeppc/asm.h			comp-c-include
-./usr/include/mvmeppc/atomic.h			comp-c-include
+./usr/include/mvmeppc/atomic.h			comp-obsolete		obsolete
 ./usr/include/mvmeppc/bat.h			comp-obsolete		obsolete
 ./usr/include/mvmeppc/bootinfo.h		comp-c-include
 ./usr/include/mvmeppc/bswap.h			comp-c-include

distrib/sets/lists/comp/md.ofppc

@@ -1,11 +1,11 @@
-# $NetBSD: md.ofppc,v 1.11 2010/11/15 12:26:36 njoly Exp $
+# $NetBSD: md.ofppc,v 1.12 2011/06/05 16:52:22 matt Exp $
 ./usr/include/ieeefp.h				comp-c-include
 ./usr/include/ofppc				comp-c-include
 ./usr/include/ofppc/_G_config.h			comp-obsolete		obsolete
 ./usr/include/ofppc/ansi.h			comp-c-include
 ./usr/include/ofppc/aout_machdep.h		comp-c-include
 ./usr/include/ofppc/asm.h			comp-c-include
-./usr/include/ofppc/atomic.h			comp-c-include
+./usr/include/ofppc/atomic.h			comp-obsolete		obsolete
 ./usr/include/ofppc/bat.h			comp-obsolete		obsolete
 ./usr/include/ofppc/bswap.h			comp-c-include
 ./usr/include/ofppc/cdefs.h			comp-c-include

distrib/sets/lists/comp/md.prep

@@ -1,11 +1,11 @@
-# $NetBSD: md.prep,v 1.25 2010/03/14 23:19:18 mrg Exp $
+# $NetBSD: md.prep,v 1.26 2011/06/05 16:52:22 matt Exp $
 ./usr/include/ieeefp.h				comp-c-include
 ./usr/include/prep				comp-c-include
 ./usr/include/prep/_G_config.h			comp-obsolete		obsolete
 ./usr/include/prep/ansi.h			comp-c-include
 ./usr/include/prep/aout_machdep.h		comp-c-include
 ./usr/include/prep/asm.h			comp-c-include
-./usr/include/prep/atomic.h			comp-c-include
+./usr/include/prep/atomic.h			comp-obsolete		obsolete
 ./usr/include/prep/bat.h			comp-obsolete		obsolete
 ./usr/include/prep/bootinfo.h			comp-c-include
 ./usr/include/prep/bswap.h			comp-c-include

distrib/sets/lists/comp/md.rs6000

@@ -1,10 +1,10 @@
-# $NetBSD: md.rs6000,v 1.3 2010/03/14 23:19:18 mrg Exp $
+# $NetBSD: md.rs6000,v 1.4 2011/06/05 16:52:22 matt Exp $
 ./usr/include/ieeefp.h				comp-c-include
 ./usr/include/rs6000				comp-c-include
 ./usr/include/rs6000/ansi.h			comp-c-include
 ./usr/include/rs6000/aout_machdep.h		comp-c-include
 ./usr/include/rs6000/asm.h			comp-c-include
-./usr/include/rs6000/atomic.h			comp-c-include
+./usr/include/rs6000/atomic.h			comp-obsolete		obsolete
 ./usr/include/rs6000/bootinfo.h			comp-c-include
 ./usr/include/rs6000/bswap.h			comp-c-include
 ./usr/include/rs6000/cdefs.h			comp-c-include

distrib/sets/lists/comp/md.sandpoint

@@ -1,11 +1,11 @@
-# $NetBSD: md.sandpoint,v 1.14 2010/03/02 21:54:06 matt Exp $
+# $NetBSD: md.sandpoint,v 1.15 2011/06/05 16:52:22 matt Exp $
 ./usr/include/ieeefp.h				comp-c-include
 ./usr/include/sandpoint				comp-c-include
 ./usr/include/sandpoint/_G_config.h		comp-obsolete		obsolete
 ./usr/include/sandpoint/ansi.h			comp-c-include
 ./usr/include/sandpoint/aout_machdep.h		comp-c-include
 ./usr/include/sandpoint/asm.h			comp-c-include
-./usr/include/sandpoint/atomic.h		comp-c-include
+./usr/include/sandpoint/atomic.h		comp-obsolete		obsolete
 ./usr/include/sandpoint/bootinfo.h		comp-c-include
 ./usr/include/sandpoint/bswap.h			comp-c-include
 ./usr/include/sandpoint/bus.h			comp-obsolete		obsolete

sys/arch/bebox/bebox/pic_bebox.c

@@ -1,4 +1,4 @@
-/* $NetBSD: pic_bebox.c,v 1.6 2008/04/28 20:23:15 martin Exp $ */
+/* $NetBSD: pic_bebox.c,v 1.7 2011/06/05 16:52:23 matt Exp $ */
 
 /*-
  * Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pic_bebox.c,v 1.6 2008/04/28 20:23:15 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pic_bebox.c,v 1.7 2011/06/05 16:52:23 matt Exp $");
 
 #include <sys/param.h>
 #include <sys/malloc.h>
@@ -38,7 +38,6 @@ __KERNEL_RCSID(0, "$NetBSD: pic_bebox.c,v 1.6 2008/04/28 20:23:15 martin Exp $")
 
 #include <uvm/uvm_extern.h>
 
-#include <machine/atomic.h>
 #include <machine/pio.h>
 
 #include <arch/powerpc/pic/picvar.h>

sys/arch/bebox/include/atomic.h

@@ -1,3 +0,0 @@
-/*	$NetBSD: atomic.h,v 1.2 2005/12/11 12:17:03 christos Exp $	*/
-
-#include <powerpc/atomic.h>

sys/arch/evbppc/conf/INSTALL_PMPPC

@@ -1,8 +1,8 @@
-# 	$NetBSD: INSTALL_PMPPC,v 1.2 2011/01/18 01:07:06 matt Exp $
+# 	$NetBSD: INSTALL_PMPPC,v 1.3 2011/06/05 16:52:23 matt Exp $
 
 include "arch/evbppc/conf/PMPPC"
 
-#ident 		"INSTALL_PMPPC-$Revision: 1.2 $"
+#ident 		"INSTALL_PMPPC-$Revision: 1.3 $"
 
 # DEBUG options turned on:
 
@@ -10,7 +10,7 @@ options		PIPE_SOCKETPAIR		# smaller, but slower pipe(2)
 
 pseudo-device	md
 
-makeoptions	NEEDS_MDSETIMAGE="yes"
+makeoptions	NEED_MDSETIMAGE="yes"
 
 no config	netbsd
 config		netbsd root on md0a
@@ -23,4 +23,3 @@ options 	MEMORY_DISK_SERVER=0	# no userspace memory disk support
 ## the `ramdisk' image (which is built in distrib/evbppc/ramdisk/ramdisk).
 options 	MEMORY_DISK_ROOT_SIZE=6144	# size of memory disk, in blocks
 options 	MEMORY_DISK_RBFLAGS=RB_SINGLE	# boot in single-user mode
-

sys/arch/evbppc/include/atomic.h

@@ -1,3 +0,0 @@
-/*	$NetBSD: atomic.h,v 1.2 2005/12/11 12:17:12 christos Exp $	*/
-
-#include <powerpc/atomic.h>

sys/arch/evbppc/mpc85xx/machdep.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: machdep.c,v 1.6 2011/05/28 05:21:40 matt Exp $	*/
+/*	$NetBSD: machdep.c,v 1.7 2011/06/05 16:52:23 matt Exp $	*/
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -709,10 +709,30 @@ e500_cpu_attach(device_t self, u_int instance)
 	e500_tlb_print(self, "tlb0", mfspr(SPR_TLB0CFG));
 	e500_tlb_print(self, "tlb1", mfspr(SPR_TLB1CFG));
 
-	intr_cpu_init(ci);
+	intr_cpu_attach(ci);
 	cpu_evcnt_attach(ci);
+
+	if (ci == curcpu())
+		intr_cpu_hatch(ci);
 }
 
+void
+e500_ipi_halt(void)
+{
+	register_t msr, hid0;
+
+	msr = wrtee(0);
+
+	hid0 = mfspr(SPR_HID0);
+	hid0 = (hid0 & ~HID0_TBEN) | HID0_DOZE;
+	mtspr(SPR_HID0, hid0);
+
+	msr = (msr & ~(PSL_EE|PSL_CE|PSL_ME)) | PSL_WE;
+	mtmsr(msr);
+	for (;;);	/* loop forever */
+}
+
+
 static void
 calltozero(void)
 {

sys/arch/evbppc/mpc85xx/mpc85xx_start.S

@@ -1,4 +1,4 @@
-/*	$NetBSD: mpc85xx_start.S,v 1.2 2011/01/18 01:10:25 matt Exp $	*/
+/*	$NetBSD: mpc85xx_start.S,v 1.3 2011/06/05 16:52:23 matt Exp $	*/
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -37,7 +37,7 @@
 #include <sys/cdefs.h>
 #include <powerpc/asm.h>
 
-RCSID("$NetBSD: mpc85xx_start.S,v 1.2 2011/01/18 01:10:25 matt Exp $")
+RCSID("$NetBSD: mpc85xx_start.S,v 1.3 2011/06/05 16:52:23 matt Exp $")
 
 #include "opt_altivec.h"
 #include "opt_ddb.h"
@@ -276,6 +276,9 @@ loop:	b	loop			/* XXX not reached */
 #ifdef PPC_HAVE_SPE
 #include <powerpc/booke/spe_subr.S>
 #endif
+#if defined(MULTIPROCESSOR)
+#include <powerpc/booke/e500_mpsubr.S>
+#endif
 
 #if 0
 	.p2align 5

sys/arch/ibmnws/include/atomic.h

@@ -1,3 +0,0 @@
-/*	$NetBSD: atomic.h,v 1.2 2005/12/11 12:17:50 christos Exp $	*/
-
-#include <powerpc/atomic.h>

sys/arch/macppc/include/atomic.h

@@ -1,3 +0,0 @@
-/*	$NetBSD: atomic.h,v 1.2 2005/12/11 12:18:06 christos Exp $	*/
-
-#include <powerpc/atomic.h>

sys/arch/macppc/include/intr.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: intr.h,v 1.27 2008/04/28 20:23:27 martin Exp $	*/
+/*	$NetBSD: intr.h,v 1.28 2011/06/05 16:52:23 matt Exp $	*/
 
 /*-
  * Copyright (c) 1998 The NetBSD Foundation, Inc.
@@ -44,14 +44,19 @@
 
 #ifdef MULTIPROCESSOR
 struct cpu_info;
+
+#include <powerpc/pic/ipivar.h>
 #endif /* MULTIPROCESSOR */
 
 #endif /* _LOCORE */
 
+#ifdef _KERNEL
+
 /* probe for a PIC and set it up, return TRUE on success */
 int init_ohare(void);
 int init_heathrow(void);
 int init_grandcentral(void);
 void setup_hammerhead_ipi(void);
+#endif
 
 #endif /* _MACPPC_INTR_H_ */

sys/arch/macppc/macppc/cpu.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: cpu.c,v 1.54 2010/12/20 00:25:37 matt Exp $	*/
+/*	$NetBSD: cpu.c,v 1.55 2011/06/05 16:52:24 matt Exp $	*/
 
 /*-
  * Copyright (c) 2001 Tsubai Masanari.
@@ -33,7 +33,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.54 2010/12/20 00:25:37 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.55 2011/06/05 16:52:24 matt Exp $");
 
 #include "opt_ppcparam.h"
 #include "opt_multiprocessor.h"
@@ -50,7 +50,6 @@ __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.54 2010/12/20 00:25:37 matt Exp $");
 #include <powerpc/oea/hid.h>
 #include <powerpc/oea/bat.h>
 #include <powerpc/openpic.h>
-#include <powerpc/atomic.h>
 #include <powerpc/spr.h>
 #include <powerpc/oea/spr.h>
 #ifdef ALTIVEC
@@ -243,7 +242,7 @@ md_setup_trampoline(volatile struct cpu_hatch_data *h, struct cpu_info *ci)
 #endif /* OPENPIC */
 		/* Start secondary CPU and stop timebase. */
 		out32(0xf2800000, (int)cpu_spinup_trampoline);
-		ppc_send_ipi(1, PPC_IPI_NOMESG);
+		cpu_send_ipi(1, IPI_NOMESG);
 #ifdef OPENPIC
 	}
 #endif
@@ -297,7 +296,7 @@ md_start_timebase(volatile struct cpu_hatch_data *h)
 
 		/* Start timebase. */
 		out32(0xf2800000, 0x100);
-		ppc_send_ipi(1, PPC_IPI_NOMESG);
+		cpu_send_ipi(1, IPI_NOMESG);
 #ifdef OPENPIC
 	}
 #endif

sys/arch/macppc/macppc/ipi_hammerhead.c

@@ -1,4 +1,4 @@
-/* $NetBSD: ipi_hammerhead.c,v 1.4 2010/12/20 00:25:37 matt Exp $ */
+/* $NetBSD: ipi_hammerhead.c,v 1.5 2011/06/05 16:52:24 matt Exp $ */
 /*-
  * Copyright (c) 2007 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -29,15 +29,15 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipi_hammerhead.c,v 1.4 2010/12/20 00:25:37 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipi_hammerhead.c,v 1.5 2011/06/05 16:52:24 matt Exp $");
 
 #include "opt_multiprocessor.h"
 #include <sys/param.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
+#include <sys/atomic.h>
 
 #include <machine/pio.h>
-#include <powerpc/atomic.h>
 
 #include <arch/powerpc/pic/picvar.h>
 #include <arch/powerpc/pic/ipivar.h>
@@ -45,8 +45,7 @@ __KERNEL_RCSID(0, "$NetBSD: ipi_hammerhead.c,v 1.4 2010/12/20 00:25:37 matt Exp
 #ifdef MULTIPROCESSOR
 
 extern struct ipi_ops ipiops;
-extern volatile u_long IPI[CPU_MAXNUM];
+static void hh_send_ipi(cpuid_t, uint32_t);
-static void hh_send_ipi(int, u_long);
 static void hh_establish_ipi(int, int, void *);
 
 #define HH_INTR_SECONDARY	0xf80000c0
@@ -62,30 +61,24 @@ setup_hammerhead_ipi(void)
 }
 
 static void
-hh_send_ipi(int target, u_long mesg)
+hh_send_ipi(cpuid_t target, uint32_t mesg)
 {
 	int cpu_id = target;
 
-	if (target == IPI_T_ALL) {
+	if (target == IPI_DST_ALL) {
-		atomic_setbits_ulong(&IPI[0], mesg);
+		atomic_or_32(&cpu_info[0].ci_pending_ipis, mesg);
-		atomic_setbits_ulong(&IPI[1], mesg);
+		atomic_or_32(&cpu_info[1].ci_pending_ipis, mesg);
 		in32(HH_INTR_PRIMARY);
 		out32(HH_INTR_SECONDARY, ~0);
 		out32(HH_INTR_SECONDARY, 0);
 		return;
 	}
 
-	if (target == IPI_T_NOTME) {
+	if (target == IPI_DST_NOTME) {
-		switch (cpu_number()) {
+		cpu_id = cpu_number() ^ 1;
-		case 0:
-			cpu_id = 1;
-			break;
-		case 1:
-			cpu_id = 0;
-		}
 	}
 
-	atomic_setbits_ulong(&IPI[cpu_id], mesg);
+	atomic_or_32(&cpu_info[cpu_id].ci_pending_ipis, mesg);
 	switch (cpu_id) {
 	case 0:
 		in32(HH_INTR_PRIMARY);
@@ -100,8 +93,7 @@ hh_send_ipi(int target, u_long mesg)
 static void
 hh_establish_ipi(int type, int level, void *ih_args)
 {
-	intr_establish(ipiops.ppc_ipi_vector, type, level, ppcipi_intr,
+	intr_establish(ipiops.ppc_ipi_vector, type, level, ipi_intr, ih_args);
-	    ih_args);
 }
 
 #endif /*MULTIPROCESSOR*/

sys/arch/macppc/macppc/machdep.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: machdep.c,v 1.155 2010/12/20 00:25:37 matt Exp $	*/
+/*	$NetBSD: machdep.c,v 1.156 2011/06/05 16:52:24 matt Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996 Wolfgang Solfrank.
@@ -32,7 +32,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.155 2010/12/20 00:25:37 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.156 2011/06/05 16:52:24 matt Exp $");
 
 #include "opt_compat_netbsd.h"
 #include "opt_ddb.h"
@@ -180,7 +180,7 @@ cpu_reboot(int howto, char *what)
 
 #ifdef MULTIPROCESSOR
 	/* Halt other CPU */
-	ppc_send_ipi(IPI_T_NOTME, PPC_IPI_HALT);
+	cpu_send_ipi(IPI_DST_NOTME, IPI_HALT);
 	delay(100000);	/* XXX */
 #endif
 

sys/arch/mvmeppc/include/atomic.h

@@ -1,3 +0,0 @@
-/*	$NetBSD: atomic.h,v 1.2 2005/12/11 12:18:19 christos Exp $	*/
-
-#include <powerpc/atomic.h>

sys/arch/ofppc/include/atomic.h

@@ -1,3 +0,0 @@
-/*	$NetBSD: atomic.h,v 1.2 2005/12/11 12:18:29 christos Exp $	*/
-
-#include <powerpc/atomic.h>

sys/arch/ofppc/include/intr.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: intr.h,v 1.14 2010/11/13 14:07:07 uebayasi Exp $	*/
+/*	$NetBSD: intr.h,v 1.15 2011/06/05 16:52:24 matt Exp $	*/
 
 /*-
  * Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -32,9 +32,16 @@
 #ifndef	_OFPPC_INTR_H_
 #define	_OFPPC_INTR_H_
 
+#ifdef _KERNEL_OPT
+#include "opt_multiprocessor.h"
+#endif
+
 #include <powerpc/intr.h>
 
 #ifndef _LOCORE
+#ifdef MULTIPROCESSOR
+#include <powerpc/pic/ipivar.h>
+#endif
 #include <machine/cpu.h>
 
 #define ICU_LEN		32

sys/arch/ofppc/ofppc/cpu.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: cpu.c,v 1.13 2010/12/20 00:25:41 matt Exp $	*/
+/*	$NetBSD: cpu.c,v 1.14 2011/06/05 16:52:24 matt Exp $	*/
 
 /*-
  * Copyright (c) 2000, 2001 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.13 2010/12/20 00:25:41 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.14 2011/06/05 16:52:24 matt Exp $");
 
 #include "opt_ppcparam.h"
 #include "opt_multiprocessor.h"
@@ -44,7 +44,6 @@ __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.13 2010/12/20 00:25:41 matt Exp $");
 #include <dev/ofw/openfirm.h>
 
 #include <powerpc/openpic.h>
-#include <powerpc/atomic.h>
 #include <powerpc/spr.h>
 #include <powerpc/oea/spr.h>
 #include <powerpc/oea/hid.h>

sys/arch/powerpc/booke/booke_machdep.c

@@ -116,35 +116,32 @@ static const struct intrsw null_intrsw = {
 
 const struct intrsw *powerpc_intrsw = &null_intrsw;
 struct cpu_md_ops cpu_md_ops;
-extern struct cpu_info cpu_info[1];
 
-#if 0
+struct cpu_softc cpu_softc[] = {
-pt_entry_t ptp0[NPTEPG] = {
-	[(0x20000 & SEGOFSET) >> PGSHIFT] = 0x00020000|PTE_xR|PTE_xX|PTE_M,
-};
-
-struct pmap_segtab pmap_kern_segtab = {
-	.seg_tab[0x20000 >> SEGSHIFT] = ptp0,
-};
-#endif
-
-struct cpu_softc cpu_softc[1] = {
 	[0] = {
-		.cpu_ci = cpu_info,
+		.cpu_ci = &cpu_info[0],
 	},
+#ifdef MULTIPROCESSOR
+	[CPU_MAXNUM-1] = {
+		.cpu_ci = &cpu_info[CPU_MAXNUM-1],
+	},
+#endif
 };
-struct cpu_info cpu_info[1] = {
+struct cpu_info cpu_info[] = {
 	[0] = {
 		.ci_curlwp = &lwp0,
 		.ci_tlb_info = &pmap_tlb0_info,
-		.ci_softc = cpu_softc,
+		.ci_softc = &cpu_softc[0],
 		.ci_cpl = IPL_HIGH,
-		.ci_fpulwp = &lwp0,
-		.ci_veclwp = &lwp0,
-#if 0
-		.ci_pmap_kern_segtab = &pmap_kern_segtab,
-#endif
 	},
+#ifdef MULTIPROCESSOR
+	[CPU_MAXNUM-1] = {
+		.ci_curlwp = NULL,
+		.ci_tlb_info = &pmap_tlb0_info,
+		.ci_softc = &cpu_softc[CPU_MAXNUM-1],
+		.ci_cpl = IPL_HIGH,
+	},
+#endif
 };
 
 /*

sys/arch/powerpc/booke/booke_pmap.c

@@ -33,11 +33,11 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-#define _PMAP_PRIVATE
+#define __PMAP_PRIVATE
 
 #include <sys/cdefs.h>
 
-__KERNEL_RCSID(0, "$NetBSD: booke_pmap.c,v 1.3 2011/02/17 13:55:44 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: booke_pmap.c,v 1.4 2011/06/05 16:52:24 matt Exp $");
 
 #include <sys/param.h>
 #include <sys/kcore.h>
@@ -87,8 +87,15 @@ pmap_procwr(struct proc *p, vaddr_t va, size_t len)
 }
 
 void
-pmap_md_page_syncicache(struct vm_page *pg)
+pmap_md_page_syncicache(struct vm_page *pg, __cpuset_t onproc)
 {
+	/*
+	 * If onproc is empty, we could do a
+	 * pmap_page_protect(pg, VM_PROT_NONE) and remove all
+	 * mappings of the page and clear its execness.  Then
+	 * the next time page is faulted, it will get icache
+	 * synched.  But this is easier. :)
+	 */
 	paddr_t pa = VM_PAGE_TO_PHYS(pg);
 	dcache_wb_page(pa);
 	icache_inv_page(pa);

sys/arch/powerpc/booke/booke_stubs.c

@@ -38,7 +38,7 @@
 
 #include <sys/cdefs.h>
 
-__KERNEL_RCSID(0, "$NetBSD: booke_stubs.c,v 1.4 2011/02/18 21:08:18 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: booke_stubs.c,v 1.5 2011/06/05 16:52:24 matt Exp $");
 
 #include <sys/param.h>
 #include <sys/cpu.h>
@@ -248,12 +248,20 @@ softint_trigger(uintptr_t machdep)
 	(*powerpc_intrsw->intrsw_softint_trigger)(machdep);
 }
 
-void intr_cpu_init(struct cpu_info *) __stub;
+void intr_cpu_attach(struct cpu_info *) __stub;
 
 void
-intr_cpu_init(struct cpu_info *ci)
+intr_cpu_attach(struct cpu_info *ci)
 {
-	(*powerpc_intrsw->intrsw_cpu_init)(ci);
+	(*powerpc_intrsw->intrsw_cpu_attach)(ci);
+}
+
+void intr_cpu_hatch(struct cpu_info *) __stub;
+
+void
+intr_cpu_hatch(struct cpu_info *ci)
+{
+	(*powerpc_intrsw->intrsw_cpu_hatch)(ci);
 }
 
 void intr_init(void) __stub;

sys/arch/powerpc/booke/e500_intr.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: e500_intr.c,v 1.7 2011/05/27 15:19:43 matt Exp $	*/
+/*	$NetBSD: e500_intr.c,v 1.8 2011/06/05 16:52:24 matt Exp $	*/
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -45,6 +45,8 @@
 #include <sys/kmem.h>
 #include <sys/atomic.h>
 #include <sys/bus.h>
+#include <sys/xcall.h>
+#include <sys/bitops.h>
 
 #include <uvm/uvm_extern.h>
 
@@ -61,9 +63,16 @@
 
 #define	IST_PERCPU_P(ist)	((ist) >= IST_TIMER)
 
+#ifdef __HAVE_PREEMPTION
+#define	IPL_PREEMPT_SOFTMASK	(1 << IPL_NONE)
+#else
+#define	IPL_PREEMPT_SOFTMASK	0
+#endif
+
 #define	IPL_SOFTMASK \
 	    ((1 << IPL_SOFTSERIAL) | (1 << IPL_SOFTNET   )	\
-	    |(1 << IPL_SOFTBIO   ) | (1 << IPL_SOFTCLOCK ))
+	    |(1 << IPL_SOFTBIO   ) | (1 << IPL_SOFTCLOCK )	\
+	    |IPL_PREEMPT_SOFTMASK)
 
 #define SOFTINT2IPL_MAP \
 	    ((IPL_SOFTSERIAL << (4*SOFTINT_SERIAL))	\
@@ -352,7 +361,9 @@ static const struct intr_source *e500_intr_last_source;
 
 static void 	*e500_intr_establish(int, int, int, int (*)(void *), void *);
 static void 	e500_intr_disestablish(void *);
-static void 	e500_intr_cpu_init(struct cpu_info *ci);
+static void 	e500_intr_cpu_attach(struct cpu_info *ci);
+static void 	e500_intr_cpu_hatch(struct cpu_info *ci);
+static void	e500_intr_cpu_send_ipi(cpuid_t, uintptr_t);
 static void 	e500_intr_init(void);
 static const char *e500_intr_string(int, int);
 static void 	e500_critintr(struct trapframe *tf);
@@ -372,7 +383,9 @@ const struct intrsw e500_intrsw = {
 	.intrsw_establish = e500_intr_establish,
 	.intrsw_disestablish = e500_intr_disestablish,
 	.intrsw_init = e500_intr_init,
-	.intrsw_cpu_init = e500_intr_cpu_init,
+	.intrsw_cpu_attach = e500_intr_cpu_attach,
+	.intrsw_cpu_hatch = e500_intr_cpu_hatch,
+	.intrsw_cpu_send_ipi = e500_intr_cpu_send_ipi,
 	.intrsw_string = e500_intr_string,
 
 	.intrsw_critintr = e500_critintr,
@@ -455,7 +468,8 @@ e500_softint_deliver(struct cpu_info *ci, struct cpu_softc *cpu,
 }
 
 static inline void
-e500_softint(struct cpu_info *ci, struct cpu_softc *cpu, int old_ipl)
+e500_softint(struct cpu_info *ci, struct cpu_softc *cpu, int old_ipl,
+	vaddr_t pc)
 {
 	const u_int softint_mask = (IPL_SOFTMASK << old_ipl) & IPL_SOFTMASK;
 	u_int softints;
@@ -487,6 +501,13 @@ e500_softint(struct cpu_info *ci, struct cpu_softc *cpu, int old_ipl)
 			    SOFTINT_CLOCK);
 			continue;
 		}
+#ifdef __HAVE_PREEMPTION
+		KASSERT(old_ipl == IPL_NONE);
+		if (softints & (1 << IPL_NONE)) {
+			ci->ci_data.cpu_softints ^= (1 << IPL_NONE);
+			kpreempt(pc);
+		}
+#endif
 	}
 }
 #endif /* __HAVE_FAST_SOFTINTS */
@@ -526,7 +547,8 @@ e500_spl0(void)
 #ifdef __HAVE_FAST_SOFTINTS
 	if (__predict_false(ci->ci_data.cpu_softints != 0)) {
 		e500_splset(ci, IPL_HIGH);
-		e500_softint(ci, ci->ci_softc, IPL_NONE);
+		e500_softint(ci, ci->ci_softc, IPL_NONE,
+		    (vaddr_t)__builtin_return_address(0));
 	}
 #endif /* __HAVE_FAST_SOFTINTS */
 	e500_splset(ci, IPL_NONE);
@@ -558,7 +580,8 @@ e500_splx(int ipl)
 	const u_int softints = (ci->ci_data.cpu_softints << ipl) & IPL_SOFTMASK;
 	if (__predict_false(softints != 0)) {
 		e500_splset(ci, IPL_HIGH);
-		e500_softint(ci, ci->ci_softc, ipl);
+		e500_softint(ci, ci->ci_softc, ipl,
+		    (vaddr_t)__builtin_return_address(0));
 	}
 #endif /* __HAVE_FAST_SOFTINTS */
 	e500_splset(ci, ipl);
@@ -652,7 +675,7 @@ e500_intr_irq_info_get(struct cpu_info *ci, u_int irq, int ipl, int ist,
 	}
 
 	ii->irq_vector = irq + info->ii_ist_vectors[ist];
-	if (IST_PERCPU_P(ist))
+	if (IST_PERCPU_P(ist) && ist != IST_IPI)
 		ii->irq_vector += ci->ci_cpuid * info->ii_percpu_sources;
 
 	switch (ist) {
@@ -764,7 +787,9 @@ e500_intr_cpu_establish(struct cpu_info *ci, int irq, int ipl, int ist,
 	 * All interrupts go to the primary except per-cpu interrupts which get
 	 * routed to the appropriate cpu.
 	 */
-	uint32_t dr = IST_PERCPU_P(ist) ? 1 << ci->ci_cpuid : 1;
+	uint32_t dr = openpic_read(cpu, ii.irq_dr);
+
+	dr |= 1 << (IST_PERCPU_P(ist) ? ci->ci_cpuid : 0);
 
 	/*
 	 * Update the vector/priority and destination registers keeping the
@@ -997,7 +1022,8 @@ e500_extintr(struct trapframe *tf)
 	if (__predict_false(softints != 0)) {
 		KASSERT(old_ipl < IPL_VM);
 		e500_splset(ci, IPL_HIGH);	/* pop to high */
-		e500_softint(ci, cpu, old_ipl);	/* deal with them */
+		e500_softint(ci, cpu, old_ipl,	/* deal with them */
+		    tf->tf_srr0);
 		e500_splset(ci, old_ipl);	/* and drop back */
 	}
 #endif /* __HAVE_FAST_SOFTINTS */
@@ -1094,7 +1120,7 @@ e500_intr_init(void)
 }
 
 static void
-e500_intr_cpu_init(struct cpu_info *ci)
+e500_intr_cpu_attach(struct cpu_info *ci)
 {
 	struct cpu_softc * const cpu = ci->ci_softc;
 	const char * const xname = device_xname(ci->ci_dev);
@@ -1154,14 +1180,96 @@ e500_intr_cpu_init(struct cpu_info *ci)
 		evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR,
 		    NULL, xname, e500_mi_intr_names[j].in_name);
 	}
+}
 
+static void
+e500_intr_cpu_send_ipi(cpuid_t target, uint32_t ipimsg)
+{
+	struct cpu_info * const ci = curcpu();
+	struct cpu_softc * const cpu = ci->ci_softc;
+	uint32_t dstmask;
+
+	if (target >= ncpu) {
+		CPU_INFO_ITERATOR cii;
+		struct cpu_info *dst_ci;
+
+		KASSERT(target == IPI_DST_NOTME || target == IPI_DST_ALL);
+
+		dstmask = 0;
+		for (CPU_INFO_FOREACH(cii, dst_ci)) {
+			if (target == IPI_DST_ALL || ci != dst_ci) {
+				dstmask |= 1 << cpu_index(ci);
+				if (ipimsg)
+					atomic_or_32(&dst_ci->ci_pending_ipis,
+					    ipimsg);
+			}
+		}
+	} else {
+		struct cpu_info * const dst_ci = cpu_lookup(target);
+		KASSERT(target == cpu_index(dst_ci));
+		dstmask = (1 << target);
+		if (ipimsg)
+			atomic_or_32(&dst_ci->ci_pending_ipis, ipimsg);
+	}
+
+	openpic_write(cpu, OPENPIC_IPIDR(0), dstmask);
+}
+
+typedef void (*ipifunc_t)(void);
+
+#ifdef __HAVE_PREEEMPTION
+static void
+e500_ipi_kpreempt(void)
+{
+	e500_softint_trigger(1 << IPL_NONE);
+}
+#endif
+
+static const ipifunc_t e500_ipifuncs[] = {
+	[ilog2(IPI_XCALL)] =	xc_ipi_handler,
+	[ilog2(IPI_HALT)] =	e500_ipi_halt,
+#ifdef __HAVE_PREEMPTION
+	[ilog2(IPI_KPREEMPT)] =	e500_ipi_kpreempt,
+#endif
+	[ilog2(IPI_TLB1SYNC)] =	e500_tlb1_sync,
+};
+
+static int
+e500_ipi_intr(void *v)
+{
+	struct cpu_info * const ci = curcpu();
+
+	ci->ci_ev_ipi.ev_count++;
+
+	uint32_t pending_ipis = atomic_swap_32(&ci->ci_pending_ipis, 0);
+	for (u_int ipi = 31; pending_ipis != 0; ipi--, pending_ipis <<= 1) {
+		const u_int bits = __builtin_clz(pending_ipis);
+		ipi -= bits;
+		pending_ipis <<= bits;
+		KASSERT(e500_ipifuncs[ipi] != NULL);
+		(*e500_ipifuncs[ipi])();
+	}
+	
+	return 1;
+}
+
+static void
+e500_intr_cpu_hatch(struct cpu_info *ci)
+{
 	/*
-	 * Establish interrupt for this CPU.
+	 * Establish clock interrupt for this CPU.
 	 */
 	if (e500_intr_cpu_establish(ci, E500_CLOCK_TIMER, IPL_CLOCK, IST_TIMER,
 	    e500_clock_intr, NULL) == NULL)
 		panic("%s: failed to establish clock interrupt!", __func__);
 
+	/*
+	 * Establish the IPI interrupts for this CPU.
+	 */
+	if (e500_intr_cpu_establish(ci, 0, IPL_VM, IST_IPI, e500_ipi_intr,
+	    NULL) == NULL)
+		panic("%s: failed to establish ipi interrupt!", __func__);
+
 	/*
 	 * Enable watchdog interrupts.
 	 */

sys/arch/powerpc/booke/e500_mpsubr.S

@@ -0,0 +1,113 @@
+
+/*
+ * r3 = fdt pointer (ignored)
+ * r4 = 0
+ * r5 = 0
+ * r6 = EPAPR magic (0x45505150)
+ * r7 = TLB1[0] entry size (64MB)
+ * r8 = 0
+ * r9 = 0
+ */
+_ENTRY(e500_spinup_trampoline)
+
+	/*
+	 * First thing we need to do is to set SPRG0 with our cpu_info
+	 * and get our initial stack pointer (this must be within the
+	 * bounds of the TLB1[0] entry U-boot setup for us).
+	 *
+	 * cpu_hatch will return a new SP to use.
+	 *
+	 * All the caller-saved register are ours to use.  So we will.
+	 */
+	lis	%r20, _C_LABEL(cpu_hatch_data)@ha
+	addi	%r20, %r20, _C_LABEL(cpu_hatch_data)@l
+
+	lwz	%r21, HATCH_CI(%r20)		/* get cpu_info */
+	mtsprg0	%r21				/* save cpu_info */
+	lwz	%r1, HATCH_SP(%r20)		/* get hatch SP */
+	
+	/*
+	 * We have to setup the IVOR SPRs since the ones u-boot setup
+	 * don't work for us.
+	 */
+	bl	_C_LABEL(exception_init)	/* setup IVORs */
+
+	/*
+	 * U-boot has mapped the bottom 64MB in TLB1[0].  We are going to need
+	 * change this entry and it's not safe to do so while running out of it.
+	 * So we copy TLB1[0] to TLB1[1] but set it for AS1.  We then switch
+	 * to AS1 and reload TLB1[0] with its correct value, and we switch
+	 * back to AS0.  Then we can load the rest of the TLB1 entries.
+	 */
+
+	/*
+	 * Fetch TLB1[0]
+	 */
+	lis	%r16, (MASX_TLBSEL_MAKE(1)|MAS0_ESEL_MAKE(0))@h
+	mtspr	SPR_MAS0, %r16
+	tlbre
+
+	/*
+	 * Copy TLB1[0] to TLB[1] and set it to use AS1
+	 */
+	mfspr	%r3, SPR_MAS0
+	addis	%r3, %r3, MAS0_ESEL@h		/* advance to next TLB entry */
+	mtspr	SPR_MAS0, %r3			/* place into SPR */
+	mfspr	%r4, SPR_MAS1
+	ori	%r4, %r4, MAS1_TS@l		/* Make it use AS1 */
+	mtspr	SPR_MAS1, %r4
+	tlbwe					/* write the TLB entry */
+
+	/*
+	 * Let's find out what TLB1[0] entry we are supposed to use.
+	 */
+	li	%r3, 0
+	bl	_C_LABEL(e500_tlb1_fetch)
+	lwz	%r28, 0(%r3)			/* load the saved TLB1 entry */
+	mtspr	SPR_MAS0, %r28			/* place into SPRs */
+	mtspr	SPR_MAS1, %r29
+	mtspr	SPR_MAS2, %r30
+	mtspr	SPR_MAS3, %r31
+
+	/*
+	 * Now to switch to running in AS1
+	 */
+	mfmsr	%r3
+	ori	%r4,%r3,(PSL_DS|PSL_IS)@l
+	mtsrr1	%r4
+
+	bl	1f
+1:	mflr	%r11
+	addi	%r4,%r11,.Las1start-1b
+	addi	%r5,%r11,.Las1end-1b
+	mtsrr0	%r4
+	rfi			/* switch to AS1, context synchronizing */
+
+.Las1start:
+	/*
+	 * We are now running in AS1, update TLB1[0]
+	 */
+	tlbwe
+
+	mtsrr0	%r5
+	mtsrr1	%r3
+	rfi			/* switch back to AS0, context synchronizing */
+
+.Las1end:
+	/*
+	 * We now have our TLB1[0] in place.  Now we need to load the rest of
+	 * TLB1 with our entries.  After this is done, we should have access
+	 * to everything.
+	 */
+	bl	_C_LABEL(e500_tlb1_sync)
+
+	/*
+	 * We've gotten the low level stuff done.
+	 * Now to do more advanced stuff.
+	 */
+	bl	_C_LABEL(cpu_hatch)	
+	mr	%r1, %r3			/* our new SP */
+
+	wrteei	1				/* allow interrupts */
+
+	b	_C_LABEL(idle_loop)

sys/arch/powerpc/booke/e500_tlb.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: e500_tlb.c,v 1.2 2011/01/18 01:02:52 matt Exp $	*/
+/*	$NetBSD: e500_tlb.c,v 1.3 2011/06/05 16:52:24 matt Exp $	*/
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -36,7 +36,7 @@
 
 #include <sys/cdefs.h>
 
-__KERNEL_RCSID(0, "$NetBSD: e500_tlb.c,v 1.2 2011/01/18 01:02:52 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: e500_tlb.c,v 1.3 2011/06/05 16:52:24 matt Exp $");
 
 #include <sys/param.h>
 
@@ -273,6 +273,38 @@ tlb_to_hwtlb(const struct e500_tlb tlb)
 	return hwtlb;
 }
 
+void *
+e500_tlb1_fetch(size_t slot)
+{
+	struct e500_tlb1 * const tlb1 = &e500_tlb1;
+
+	return &tlb1->tlb1_entries[slot].e_hwtlb;
+}
+
+void
+e500_tlb1_sync(void)
+{
+	struct e500_tlb1 * const tlb1 = &e500_tlb1;
+	for (u_int slot = 1; slot < tlb1->tlb1_numentries; slot++) {
+		const struct e500_hwtlb * const new_hwtlb =
+		    &tlb1->tlb1_entries[slot].e_hwtlb;
+		const struct e500_hwtlb old_hwtlb =
+		    hwtlb_read(MAS0_TLBSEL_TLB1, slot);
+#define CHANGED(n,o,f)	((n)->f != (o).f)
+		bool mas1_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas1);
+		bool mas2_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas2);
+		bool mas3_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas3);
+#undef CHANGED
+		bool new_valid_p = (new_hwtlb->hwtlb_mas1 & MAS1_V) != 0;
+		bool old_valid_p = (old_hwtlb.hwtlb_mas1 & MAS1_V) != 0;
+		if ((new_valid_p || old_valid_p)
+		    && (mas1_changed_p
+			|| (new_valid_p
+			    && (mas2_changed_p || mas3_changed_p))))
+			hwtlb_write(*new_hwtlb, true);
+	}
+}
+
 static int
 e500_alloc_tlb1_entry(void)
 {

sys/arch/powerpc/booke/genassym.cf

@@ -1,4 +1,4 @@
-#	$NetBSD: genassym.cf,v 1.2 2011/01/18 01:02:52 matt Exp $
+#	$NetBSD: genassym.cf,v 1.3 2011/06/05 16:52:24 matt Exp $
 
 #-
 # Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
@@ -154,6 +154,7 @@ define	PCB_USPRG0	offsetof(struct pcb, pcb_usprg0)
 define	L_PCB		offsetof(struct lwp, l_addr)
 define	L_CPU		offsetof(struct lwp, l_cpu)
 define	L_CTXSWTCH	offsetof(struct lwp, l_ctxswtch)
+define	L_MD_ASTPENDING	offsetof(struct lwp, l_md.md_astpending)
 define	L_MD_UTF	offsetof(struct lwp, l_md.md_utf)
 define	L_PROC		offsetof(struct lwp, l_proc)
 
@@ -163,11 +164,11 @@ define	CI_SIZE		sizeof(struct cpu_info)
 define	CI_CURLWP	offsetof(struct cpu_info, ci_curlwp)
 define	CI_CURPCB	offsetof(struct cpu_info, ci_curpcb)
 define	CI_CURPM	offsetof(struct cpu_info, ci_curpm)
-define	CI_ASTPENDING	offsetof(struct cpu_info, ci_astpending)
 define	CI_WANT_RESCHED	offsetof(struct cpu_info, ci_want_resched)
 define	CI_CPL		offsetof(struct cpu_info, ci_cpl)
 define	CI_IDEPTH	offsetof(struct cpu_info, ci_idepth)
 define	CI_IDLESPIN	offsetof(struct cpu_info, ci_idlespin)
+define	CI_IDLELWP	offsetof(struct cpu_info, ci_data.cpu_idlelwp)
 define	CI_MTX_COUNT	offsetof(struct cpu_info, ci_mtx_count)
 define	CI_MTX_OLDSPL	offsetof(struct cpu_info, ci_mtx_oldspl)
 define	CI_SAVELIFO	offsetof(struct cpu_info, ci_savelifo)

sys/arch/powerpc/booke/trap.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: trap.c,v 1.6 2011/05/02 02:01:32 matt Exp $	*/
+/*	$NetBSD: trap.c,v 1.7 2011/06/05 16:52:24 matt Exp $	*/
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -39,7 +39,7 @@
 
 #include <sys/cdefs.h>
 
-__KERNEL_RCSID(1, "$NetBSD: trap.c,v 1.6 2011/05/02 02:01:32 matt Exp $");
+__KERNEL_RCSID(1, "$NetBSD: trap.c,v 1.7 2011/06/05 16:52:24 matt Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -772,7 +772,7 @@ trap(enum ppc_booke_exceptions trap_code, struct trapframe *tf)
 		break;
 	case T_AST:
 		KASSERT(usertrap);
-		ci->ci_astpending = 0;		/* we are about to do it */
+		l->l_md.md_astpending = 0;	/* we are about to do it */
 		ci->ci_data.cpu_nsoft++;
 		if (l->l_pflag & LP_OWEUPC) {
 			l->l_pflag &= ~LP_OWEUPC;

sys/arch/powerpc/booke/trap_subr.S

@@ -1,4 +1,4 @@
-/*	$NetBSD: trap_subr.S,v 1.2 2011/01/18 01:02:52 matt Exp $	*/
+/*	$NetBSD: trap_subr.S,v 1.3 2011/06/05 16:52:24 matt Exp $	*/
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -34,7 +34,7 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-RCSID("$NetBSD: trap_subr.S,v 1.2 2011/01/18 01:02:52 matt Exp $")
+RCSID("$NetBSD: trap_subr.S,v 1.3 2011/06/05 16:52:24 matt Exp $")
 
 	.globl	_C_LABEL(critintr_call)
 	.globl	_C_LABEL(extintr_call)
@@ -202,6 +202,7 @@ RCSID("$NetBSD: trap_subr.S,v 1.2 2011/01/18 01:02:52 matt Exp $")
 	stmw	%r3, FRAME_3(%r1);	/* save r2-r31 */		\
 	/* Now everything has been saved */				\
 	mr	%r31, %r0;		/* move SRR1 back to r31 */	\
+	mfsprg2	%r13;			/* put curlwp in r13 */		\
 	FRAME_SAVE_SPEFSCR;						\
 	li	%r7, exc;		/* load EXC_* */		\
 	stw	%r7, FRAME_EXC(%r1);	/* save into trapframe */	\
@@ -293,6 +294,7 @@ RCSID("$NetBSD: trap_subr.S,v 1.2 2011/01/18 01:02:52 matt Exp $")
 	addi	%r2,%r2,-4*(32-start);	/* find our save area */	\
 	lmw	%r##start,0(%r2);	/* get start-r31 */		\
 	mtsprg3	%r2;			/* save updated pointer */	\
+	mfsprg2	%r13;			/* put curlwp into r13 */	\
 	li	%r7, exc;		/* load EXC_* */		\
 	stw	%r7, FRAME_EXC(%r1);	/* save into trapframe */	\
 	addi	%r3, %r1, FRAME_TF	/* only argument is trapframe */
@@ -385,8 +387,7 @@ _C_LABEL(trapexit):
 #	tweqi	%r0, 0
 	andi.	%r4, %r31, PSL_PR		/* lets look at PSL_PR */
 	beq	trapdone			/* if clear, skip to exit  */
-	GET_CPUINFO(%r3)			/* get curcpu() */
+	lwz	%r4, L_MD_ASTPENDING(%r13)	/* get ast pending */
-	lwz	%r4, CI_ASTPENDING(%r3)		/* get ast pending */
 	cmplwi	%r4, 0				/* is there an ast pending */
 	beq+	trapdone			/*    nope, proceed to exit */
 	li	%r6, EXC_AST			/* yes. */
@@ -420,7 +421,7 @@ _C_LABEL(intrcall):
 #	tweqi	%r0, 0
 	andi.	%r4, %r5, PSL_PR		/* lets look at PSL_PR */
 	beq	intrexit			/* if clear, skip to exit  */
-	lwz	%r4, CI_ASTPENDING(%r6)		/* get ast pending */
+	lwz	%r4, L_MD_ASTPENDING(%r13)	/* get ast pending */
 	cmplwi	%r4, 0				/* is there an ast pending */
 	beq+	intrexit			/*    nope, proceed to exit */
 	stmw	%r14, FRAME_14(%r1)		/* save rest of registers */
@@ -467,15 +468,14 @@ _C_LABEL(system_call_vector):
 	FRAME_ENTER(EXC_SC, %r3)
 
 	wrteei	1				/* enable interrupts */
-	mfspr	%r6, SPR_SPRG2			/* get curlwp */
+	lwz	%r7, L_PROC(%r13)		/* get proc for lwp */
-	lwz	%r7, L_PROC(%r6)		/* get proc for lwp */
 	lwz	%r8, P_MD_SYSCALL(%r7)		/* get syscall */
-	mtctr	%r8				/* need to call indirect */
+	mtlr	%r8				/* need to call indirect */
-	bctrl					/* syscall(tf) */
+	blrl					/* syscall(tf) */
 _C_LABEL(sctrapexit):
 	wrteei	0				/* disable interrupts */
-	GET_CPUINFO(%r3)			/* get curcpu() */
+	mfsprg2	%r3				/* get curlwp */
-	lwz	%r4, CI_ASTPENDING(%r3)		/* get ast pending */
+	lwz	%r4, L_MD_ASTPENDING(%r13)	/* get ast pending */
 	cmplwi	%r4, 0				/* is there an ast pending */
 	beq+	trapdone			/*    nope, proceed to exit */
 	li	%r0, EXC_AST			/* yes. */
@@ -859,6 +859,10 @@ _C_LABEL(exception_init):
 	ori	%r5,%r6,_C_LABEL(perfmon_vector)@l
 	mtspr	SPR_IVOR35, %r5
 
+	mfpir	%r5		/* get Process ID register */
+	cmplwi	%r5,0
+	bnelr			/* return if non-0 (non-primary) */
+
 	lis	%r5,_C_LABEL(critintr_call)@ha
 	lwzu	%r6,_C_LABEL(critintr_call)@l(%r5)
 	lwz	%r7,INTRSW_CRITINTR(%r3)

sys/arch/powerpc/conf/files.powerpc

@@ -1,7 +1,7 @@
-#	$NetBSD: files.powerpc,v 1.76 2011/05/02 02:01:32 matt Exp $
+#	$NetBSD: files.powerpc,v 1.77 2011/06/05 16:52:24 matt Exp $
 
 defflag	opt_altivec.h	ALTIVEC K_ALTIVEC PPC_HAVE_SPE
-defflag	opt_openpic.h	OPENPIC OPENPIC_SERIAL_MODE
+defflag	opt_openpic.h	OPENPIC OPENPIC_SERIAL_MODE OPENPIC_DISTRIBUTE
 defparam opt_ppcparam.h	L2CR_CONFIG L3CR_CONFIG INTSTK CLOCKBASE
 defflag	opt_ppcarch.h	PPC_OEA PPC_OEA601 PPC_OEA64 PPC_OEA64_BRIDGE PPC_MPC8XX PPC_IBM4XX PPC_IBM403 PPC_BOOKE
 defflag opt_pmap.h	PMAPDEBUG PMAPCHECK PMAPCOUNTERS

sys/arch/powerpc/ibm4xx/genassym.cf

@@ -1,4 +1,4 @@
-#	$NetBSD: genassym.cf,v 1.18 2011/01/18 01:02:54 matt Exp $
+#	$NetBSD: genassym.cf,v 1.19 2011/06/05 16:52:25 matt Exp $
 
 #
 # Copyright (C) 1995, 1996 Wolfgang Solfrank.
@@ -132,6 +132,7 @@ define	PM_CTX		offsetof(struct pmap, pm_ctx)
 define	MSR_PR		31-ilog2(PSL_PR)
 
 define	L_CPU		offsetof(struct lwp, l_cpu)
+define	L_MD_ASTPENDING	offsetof(struct lwp, l_md.md_astpending)
 define	L_MD_UTF	offsetof(struct lwp, l_md.md_utf)
 define	L_PCB		offsetof(struct lwp, l_addr)
 define	L_PROC		offsetof(struct lwp, l_proc)
@@ -142,7 +143,6 @@ define	CI_SIZE		sizeof(struct cpu_info)
 define	CI_CURLWP	offsetof(struct cpu_info, ci_curlwp)
 define	CI_CURPCB	offsetof(struct cpu_info, ci_curpcb)
 define	CI_CURPM	offsetof(struct cpu_info, ci_curpm)
-define	CI_ASTPENDING	offsetof(struct cpu_info, ci_astpending)
 define	CI_WANT_RESCHED	offsetof(struct cpu_info, ci_want_resched)
 define	CI_CPL		offsetof(struct cpu_info, ci_cpl)
 define	CI_IDEPTH	offsetof(struct cpu_info, ci_idepth)

sys/arch/powerpc/ibm4xx/trap.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: trap.c,v 1.58 2011/01/18 01:02:54 matt Exp $	*/
+/*	$NetBSD: trap.c,v 1.59 2011/06/05 16:52:25 matt Exp $	*/
 
 /*
  * Copyright 2001 Wasabi Systems, Inc.
@@ -67,7 +67,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.58 2011/01/18 01:02:54 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.59 2011/06/05 16:52:25 matt Exp $");
 
 #include "opt_altivec.h"
 #include "opt_ddb.h"
@@ -295,7 +295,7 @@ trap(struct trapframe *tf)
 		break;
 
 	case EXC_AST|EXC_USER:
-		curcpu()->ci_astpending = 0;	/* we are about to do it */
+		l->l_md.md_astpending = 0;	/* we are about to do it */
 		//curcpu()->ci_data.cpu_nast++;
 		if (l->l_pflag & LP_OWEUPC) {
 			l->l_pflag &= ~LP_OWEUPC;

sys/arch/powerpc/ibm4xx/trap_subr.S

@@ -1,4 +1,4 @@
-/*	$NetBSD: trap_subr.S,v 1.14 2011/05/02 06:37:47 kiyohara Exp $	*/
+/*	$NetBSD: trap_subr.S,v 1.15 2011/06/05 16:52:25 matt Exp $	*/
 
 /*
  * Copyright 2001 Wasabi Systems, Inc.
@@ -92,7 +92,7 @@
 	stmw	%r30,(savearea+CPUSAVE_SRR0)(%r1); /* save srr0/srr1 */	\
 	mfsprg1	%r1;			/* restore SP */ 	\
 	mtcr	%r31;						\
-	bc	%cr4,MSR_PR,1f;	/* branch if MSR[PR] is clear */	\
+	bf	MSR_PR,1f;		/* branch if MSR[PR] is clear */ \
 	GET_PCB(%r1);						\
 	addi	%r1,%r1,USPACE-CALLFRAMELEN; /* stack is top of user struct */ \
 1:
@@ -111,7 +111,7 @@
 	stmw	%r30,(savearea+CPUSAVE_SRR0)(%r1); /* save srr0/srr1 */	\
 	mfsprg1	%r1;			/* restore SP */ 	\
 	mtcr	%r31;						\
-	bc	%cr4,MSR_PR,1f;	/* branch if MSR[PR] is clear */	\
+	bf	MSR_PR,1f;		/* branch if MSR[PR] is clear */ \
 	GET_PCB(%r1);						\
 	addi	%r1,%r1,USPACE-CALLFRAMELEN; /* stack is top of user struct */ \
 1:
@@ -127,7 +127,7 @@
 	stmw	%r30,(savearea+CPUSAVE_SRR0)(%r1); /* save srr0/srr1 */	\
 	mfsprg1	%r1;			/* restore SP */ 	\
 	mtcr	%r31;						\
-	bc	%cr4,MSR_PR,1f;	/* branch if MSR[PR] is clear */	\
+	bf	MSR_PR,1f;		/* branch if MSR[PR] is clear */ \
 	GET_PCB(%r1);						\
 	addi	%r1,%r1,USPACE-CALLFRAMELEN; /* stack is top of user struct */ \
 1:
@@ -273,6 +273,7 @@ _C_LABEL(ipkdbsize) = .-_C_LABEL(ipkdblow)
 	lmw	%r28,(savearea+CPUSAVE_R28)(%r2);			\
 	stmw	%r3,FRAME_R3(%r1);					\
 	lmw	%r28,(savearea+CPUSAVE_DEAR)(%r2);			\
+	lwz	%r13,CI_CURLWP(%r2);					\
 	mfxer	%r3;							\
 	mfctr	%r4;							\
 	mflr	%r5;							\
@@ -301,6 +302,7 @@ _C_LABEL(ipkdbsize) = .-_C_LABEL(ipkdblow)
 	mtctr	%r9;							\
 	mtsrr0	%r10;							\
 	mtsrr1	%r11;							\
+	lwz	%r13,FRAME_R13(%r1);					\
 	lwz	%r12,FRAME_R12(%r1);					\
 	lwz	%r11,FRAME_R11(%r1);					\
 	lwz	%r10,FRAME_R10(%r1);					\
@@ -335,10 +337,9 @@ _C_LABEL(trapexit):
 
 	/* Test AST pending: */
 	mtcr	%r31
-	bc	%cr4,MSR_PR,trapleave_to_kernel /* branch if MSR[PR] is false */
+	bf	MSR_PR,trapleave_to_kernel /* branch if MSR[PR] is false */
 
-	GET_CPUINFO(%r3)
+	lwz	%r4,L_MD_ASTPENDING(%r13)
-	lwz	%r4,CI_ASTPENDING(%r3)
 	andi.	%r4,%r4,1
 	beq	trapleave_to_user
 
@@ -347,7 +348,7 @@ _C_LABEL(trapexit):
 	b	trapagain
 
 trapleave_to_kernel:
-	lmw	%r13, FRAME_R13(%r1)	/* restore callee registers */
+	lmw	%r14, FRAME_R14(%r1)	/* restore callee registers */
 
 intrleave_to_kernel:
 	FRAME_RESTORE		/* old SP is now in sprg1 */
@@ -410,9 +411,7 @@ s_sctrap:
 	wrteei	1			/* Enable interrupts */
 /* Call the appropriate syscall handler: */
 	addi	%r3,%r1,FRAME_TF
-	GET_CPUINFO(%r4)
+	lwz	%r4,L_PROC(%r13)
-	lwz	%r4,CI_CURLWP(%r4)
-	lwz	%r4,L_PROC(%r4)
 	lwz	%r4,P_MD_SYSCALL(%r4)
 	mtctr	%r4
 	bctrl
@@ -444,6 +443,7 @@ _C_LABEL(sctrapexit):
 	stmw	%r28,FRAME_LR(%r1);	/* save LR, CR, XER, CTR */	\
 	GET_CPUINFO(%r5);						\
 	lmw	%r28,(tempsave+CPUSAVE_R28)(%r5); /* restore r28-r31 */	\
+	lwz	%r13,CI_CURLWP(%r5);					\
 	lwz	%r5,CI_IDEPTH(%r5);					\
 	mfsrr0	%r4;							\
 	mfsrr1	%r3;							\
@@ -469,12 +469,12 @@ intr_exit:
 	addi	%r4,%r4,-1		/* adjust reentrancy count */
 	stw	%r4,CI_IDEPTH(%r5)
 
-	lwz	%r5,FRAME_SRR1(%r1)
+	lwz	%r4,FRAME_SRR1(%r1)
 /* Returning to user mode? */
-	mtcr	%r5			/* saved SRR1 */
+	mtcr	%r4			/* saved SRR1 */
-	bc	%cr4,MSR_PR,intrleave_to_kernel /* branch if MSR[PR] is false */
+	bf	MSR_PR,intrleave_to_kernel /* branch if MSR[PR] is false */
 
-	lwz	%r4,CI_ASTPENDING(%r5)	/* Test AST pending */
+	lwz	%r4,L_MD_ASTPENDING(%r13)/* Test AST pending */
 	andi.	%r4,%r4,1
 	beq	intrleave_to_user
 

sys/arch/powerpc/include/Makefile

@@ -1,11 +1,11 @@
-#	$NetBSD: Makefile,v 1.35 2010/03/02 21:52:32 matt Exp $
+#	$NetBSD: Makefile,v 1.36 2011/06/05 16:52:25 matt Exp $
 
 .if !defined(INCSDIR)
 INCSDIR?= /usr/include/powerpc
 INCS+=	spr.h
 .endif
 
-INCS+=	ansi.h aout_machdep.h asm.h atomic.h \
+INCS+=	ansi.h aout_machdep.h asm.h \
 	bswap.h \
 	cdefs.h cpu.h \
 	elf_machdep.h endian.h endian_machdep.h \

sys/arch/powerpc/include/asm.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: asm.h,v 1.36 2011/02/07 06:37:01 matt Exp $	*/
+/*	$NetBSD: asm.h,v 1.37 2011/06/05 16:52:25 matt Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996 Wolfgang Solfrank.
@@ -211,12 +211,12 @@ y:	.quad	.y,.TOC.@tocbase,0;	\
 	li	tmp2,-1;						\
 	stint	tmp2,CI_IDEPTH(tmp1);					\
 	li	tmp2,0;							\
-	lis	tmp1,_C_LABEL(lwp0)@h;					\
+	lis	%r13,_C_LABEL(lwp0)@h;					\
-	ori	tmp1,tmp1,_C_LABEL(lwp0)@l;				\
+	ori	%r13,%r13,_C_LABEL(lwp0)@l;				\
-	stptr	er,L_PCB(tmp1);		/* XXXuvm_lwp_getuarea */	\
+	stptr	er,L_PCB(%r13);		/* XXXuvm_lwp_getuarea */	\
 	addi	er,er,USPACE;		/* stackpointer for lwp0 */	\
 	addi	sp,er,-FRAMELEN-CALLFRAMELEN;	/* stackpointer for lwp0 */ \
-	stptr	sp,L_MD_UTF(tmp1);	/* save in lwp0.l_md.md_utf */	\
+	stptr	sp,L_MD_UTF(%r13);	/* save in lwp0.l_md.md_utf */	\
 		/* er = end of mem reserved for kernel */		\
 	li	tmp2,0;							\
 	stptr	tmp2,-CALLFRAMELEN(er);	/* end of stack chain */	\

sys/arch/powerpc/include/atomic.h

@@ -1,129 +0,0 @@
-/* $NetBSD: atomic.h,v 1.4 2005/12/28 19:09:29 perry Exp $ */
-
-/*-
- */
-
-/*
- * Misc. `atomic' operations.
- */
-
-#ifndef _POWERPC_ATOMIC_H_
-#define	_POWERPC_ATOMIC_H_
-
-/*
- * atomic_setbits_ulong:
- *
- *	Atomically set bits in a `unsigned long'.
- */
-static __inline void
-atomic_setbits_ulong(volatile unsigned long *ulp, unsigned long v)
-{
-	unsigned long tmp;
-
-	__asm volatile(
-"# BEGIN atomic_setbits_ulong	\n"
-"1:	lwarx	%0,0,%2		\n"
-"	or	%0,%1,%0	\n"
-"	stwcx.	%0,0,%2		\n"
-"	bne-	1b		\n"
-"	sync			\n"
-"# END atomic_setbits_ulong	\n"
-	: "=&r" (tmp)
-	: "r" (v), "r" (ulp)
-	: "memory");
-}
-
-/*
- * atomic_clearbits_ulong:
- *
- *	Atomically clear bits in a `unsigned long'.
- */
-static __inline void
-atomic_clearbits_ulong(volatile unsigned long *ulp, unsigned long v)
-{
-	unsigned long tmp;
-
-	__asm volatile(
-"# BEGIN atomic_clearbits_ulong	\n"
-"1:	lwarx	%0,0,%2		\n"
-"	and	%0,%1,%0	\n"
-"	stwcx.	%0,0,%2		\n"
-"	bne-	1b		\n"
-"	sync			\n"
-"# END atomic_clearbits_ulong	\n"
-	: "=&r" (tmp)
-	: "r" (~v), "r" (ulp)
-	: "memory");
-}
-
-/*
- * atomic_add_ulong:
- *
- *	Atomically add a value to a `unsigned long'.
- */
-static __inline void
-atomic_add_ulong(volatile unsigned long *ulp, unsigned long v)
-{
-	unsigned long tmp;
-
-	__asm volatile(
-"# BEGIN atomic_add_ulong	\n"
-"1:	lwarx	%0,0,%2		\n"
-"	add	%0,%1,%0	\n"
-"	stwcx.	%0,0,%2		\n"
-"	bne-	1b		\n"
-"	sync			\n"
-"# END atomic_add_ulong		\n"
-	: "=&r" (tmp)
-	: "r" (v), "r" (ulp)
-	: "memory");
-}
-
-/*
- * atomic_sub_ulong:
- *
- *	Atomically subtract a value from a `unsigned long'.
- */
-static __inline void
-atomic_sub_ulong(volatile unsigned long *ulp, unsigned long v)
-{
-	unsigned long tmp;
-
-	__asm volatile(
-"# BEGIN atomic_sub_ulong	\n"
-"1:	lwarx	%0,0,%2		\n"
-"	sub	%0,%0,%1	\n"
-"	stwcx.	%0,0,%2		\n"
-"	bne-	1b		\n"
-"	sync			\n"
-"# END atomic_sub_ulong		\n"
-	: "=&r" (tmp)
-	: "r" (v), "r" (ulp)
-	: "memory");
-}
-
-/*
- * atomic_loadlatch_ulong:
- *
- *	Atomically load and latch a `unsigned long' value.
- */
-static __inline unsigned long
-atomic_loadlatch_ulong(volatile unsigned long *ulp, unsigned long v)
-{
-	unsigned long tmp;
-
-	__asm volatile(
-"# BEGIN atomic_loadlatch_ulong	\n"
-"1:	lwarx	%0,0,%2		\n"
-"	stwcx.	%1,0,%2		\n"
-"	bne-	1b		\n"
-"	sync			\n"
-"# END atomic_loadlatch_ulong	\n"
-	: "=r" (tmp)
-	: "r" (v), "r" (ulp)
-	: "memory");
-
-	return (tmp);
-}
-
-#endif /* _POWERPC_ATOMIC_H_ */

sys/arch/powerpc/include/booke/cpuvar.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: cpuvar.h,v 1.5 2011/02/17 13:53:32 matt Exp $	*/
+/*	$NetBSD: cpuvar.h,v 1.6 2011/06/05 16:52:25 matt Exp $	*/
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -166,6 +166,8 @@ void	booke_fixup_stubs(void);
 void	booke_cpu_startup(const char *);	/* model name */
 struct powerpc_bus_dma_tag booke_bus_dma_tag;
 
+extern struct cpu_info cpu_info[];
+
 void	cpu_evcnt_attach(struct cpu_info *);
 uint32_t cpu_read_4(bus_size_t);
 uint8_t	cpu_read_1(bus_size_t);

sys/arch/powerpc/include/booke/e500var.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: e500var.h,v 1.2 2011/01/18 01:02:54 matt Exp $	*/
+/*	$NetBSD: e500var.h,v 1.3 2011/06/05 16:52:25 matt Exp $	*/
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -81,8 +81,28 @@ uint16_t e500_get_svr(void);
 int	e500_cpunode_submatch(device_t, cfdata_t, const char *, void *);
 #endif
 
+/*
+ * Used by MP hatch code to fetch the TLB1 entries so they be setup on the
+ * just hatched CPU.
+ */
+void   *e500_tlb1_fetch(size_t);
+void	e500_tlb1_sync(void);
+void	e500_ipi_halt(void);
+
 void	pq3gpio_attach(device_t, device_t, void *);
 
+/*
+ * For a lack of a better place, define this u-boot structure here.
+ */
+
+struct uboot_spinup_entry {
+	uint64_t entry_addr;
+	uint64_t entry_r3;
+	uint32_t entry__rsvd;
+	uint32_t entry_pir;
+	uint64_t entry_r6;
+};
+
 #endif /* _KERNEL */
 
 #endif /* !_POWERPC_BOOKE_E500VAR_H_ */

sys/arch/powerpc/include/booke/intr.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: intr.h,v 1.3 2011/02/08 06:28:56 matt Exp $	*/
+/*	$NetBSD: intr.h,v 1.4 2011/06/05 16:52:25 matt Exp $	*/
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -64,21 +64,29 @@
 #define IST_MAX		(NIPL+10)
 #endif
 
-#define	IPI_DST_ALL	-2
+#define	IPI_DST_ALL	((cpuid_t)-2)
-#define	IPI_DST_NOTME	-1
+#define	IPI_DST_NOTME	((cpuid_t)-1)
+
+#define IPI_NOMESG	0x0000
+#define IPI_HALT	0x0001
+#define IPI_XCALL	0x0002
+#define	IPI_KPREEMPT	0x0004
+#define IPI_TLB1SYNC	0x0008
 
 #define	__HAVE_FAST_SOFTINTS	1
+#define	SOFTINT_KPREEMPT	SOFTINT_COUNT
 
 #ifndef _LOCORE
 
 void 	*intr_establish(int, int, int, int (*)(void *), void *);
 void 	intr_disestablish(void *);
-void	intr_cpu_init(struct cpu_info *);
+void	intr_cpu_attach(struct cpu_info *);
+void	intr_cpu_hatch(struct cpu_info *);
 void	intr_init(void);
 const char *
 	intr_string(int, int);
 
-void	cpu_send_ipi(cpuid_t, uintptr_t);
+void	cpu_send_ipi(cpuid_t, uint32_t);
 
 void	spl0(void);
 int 	splraise(int);
@@ -103,7 +111,9 @@ typedef struct {
 struct intrsw {
 	void *(*intrsw_establish)(int, int, int, int (*)(void *), void *);
 	void (*intrsw_disestablish)(void *);
-	void (*intrsw_cpu_init)(struct cpu_info *);
+	void (*intrsw_cpu_attach)(struct cpu_info *);
+	void (*intrsw_cpu_hatch)(struct cpu_info *);
+	void (*intrsw_cpu_send_ipi)(cpuid_t, uint32_t);
 	void (*intrsw_init)(void);
 	void (*intrsw_critintr)(struct trapframe *);
 	void (*intrsw_decrintr)(struct trapframe *);
@@ -114,7 +124,6 @@ struct intrsw {
 	void (*intrsw_spl0)(void);
 	void (*intrsw_splx)(int);
 	const char *(*intrsw_string)(int, int);
-	void (*intrsw_send_ipi)(cpuid_t, uintptr_t);
 #ifdef __HAVE_FAST_SOFTINTS
 	void (*intrsw_softint_init_md)(struct lwp *, u_int, uintptr_t *);
 	void (*intrsw_softint_trigger)(uintptr_t);

sys/arch/powerpc/include/booke/pmap.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: pmap.h,v 1.4 2011/02/17 13:55:45 matt Exp $	*/
+/*	$NetBSD: pmap.h,v 1.5 2011/06/05 16:52:25 matt Exp $	*/
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -43,8 +43,9 @@
 #include <sys/cpu.h>
 #include <sys/kcore.h>
 #include <uvm/uvm_page.h>
-#ifdef _PMAP_PRIVATE
+#ifdef __PMAP_PRIVATE
 #include <powerpc/booke/cpuvar.h>
+#include <powerpc/cpuset.h>
 #endif
 
 #define	PMAP_MD_NOCACHE		0x01000000
@@ -78,7 +79,7 @@ paddr_t	pmap_md_direct_mapped_vaddr_to_paddr(vaddr_t);
 vaddr_t	pmap_md_direct_map_paddr(paddr_t);
 void	pmap_md_init(void);
 
-void	pmap_md_page_syncicache(struct vm_page *);
+void	pmap_md_page_syncicache(struct vm_page *, __cpuset_t);
 void	pmap_bootstrap(vaddr_t, vaddr_t, const phys_ram_seg_t *, size_t);
 bool	pmap_extract(struct pmap *, vaddr_t, paddr_t *);
 

sys/arch/powerpc/include/booke/spr.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: spr.h,v 1.6 2011/04/29 21:43:51 matt Exp $	*/
+/*	$NetBSD: spr.h,v 1.7 2011/06/05 16:52:25 matt Exp $	*/
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -483,6 +483,8 @@
 #define   HID0_DCFA		  0x00000040 /* Data Cache Flush Assist */
 #define   HID0_NOOPTI		  0x00000001 /* NO-OP Touch Instructions */
 #define	SPR_HID1		1009
+#define   HID1_ASTME		  0x00004000 /* Address Streaming Enable */
+#define   HID1_ABE		  0x00001000 /* Address Broadcast Enable */
 #define	SPR_L1CSR0		1010	/* E... L1 Cache Control and Status Register 0 (Data) */
 #define	SPR_L1CSR1		1011	/* E... L1 Cache Control and Status Register 1 (Instruction) */
 #define   L1CSR_CPE		  0x00010000 /* 15: Cache Parity Error */

sys/arch/powerpc/include/booke/vmparam.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: vmparam.h,v 1.3 2011/01/18 01:02:54 matt Exp $	*/
+/*	$NetBSD: vmparam.h,v 1.4 2011/06/05 16:52:25 matt Exp $	*/
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -37,6 +37,8 @@
 #ifndef _POWERPC_BOOKE_VMPARAM_H_
 #define _POWERPC_BOOKE_VMPARAM_H_
 
+#include <sys/mutex.h>
+
 /*
  * Most of the definitions in this can be overriden by a machine-specific
  * vmparam.h if required.  Otherwise a port can just include this file

sys/arch/powerpc/include/cpu.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: cpu.h,v 1.76 2011/05/04 04:33:37 macallan Exp $	*/
+/*	$NetBSD: cpu.h,v 1.77 2011/06/05 16:52:25 matt Exp $	*/
 
 /*
  * Copyright (C) 1999 Wolfgang Solfrank.
@@ -68,11 +68,8 @@ struct cpu_info {
 
 	struct pcb *ci_curpcb;
 	struct pmap *ci_curpm;
-	struct lwp * volatile ci_fpulwp;
+	int ci_cpuid;			/* from SPR_PIR */
-	struct lwp * volatile ci_veclwp;
-	int ci_cpuid;
 
-	volatile int ci_astpending;
 	int ci_want_resched;
 	volatile uint64_t ci_lastintr;
 	volatile u_long ci_lasttb;
@@ -83,6 +80,7 @@ struct cpu_info {
 #ifndef PPC_BOOKE
 	volatile imask_t ci_ipending;
 #endif
+	volatile uint32_t ci_pending_ipis;
 	int ci_mtx_oldspl;
 	int ci_mtx_count;
 #ifdef PPC_IBM4XX
@@ -199,6 +197,7 @@ void	cpu_boot_secondary_processors(void);
 
 extern struct cpu_info cpu_info[];
 
+static __inline struct cpu_info * curcpu(void) __pure;
 static __inline struct cpu_info *
 curcpu(void)
 {
@@ -208,7 +207,8 @@ curcpu(void)
 	return ci;
 }
 
-#define curlwp			(curcpu()->ci_curlwp)
+register struct lwp *powerpc_curlwp __asm("r13");
+#define	curlwp			powerpc_curlwp
 #define curpcb			(curcpu()->ci_curpcb)
 #define curpm			(curcpu()->ci_curpm)
 
@@ -397,10 +397,10 @@ void cpu_spinup_trampoline(void);
 
 #define	DELAY(n)		delay(n)
 
-#define	cpu_need_resched(ci, v)	(ci->ci_want_resched = ci->ci_astpending = 1)
+void	cpu_need_resched(struct cpu_info *, int);
-#define	cpu_did_resched(l)	((void)(curcpu()->ci_want_resched = 0))
+void	cpu_signotify(struct lwp *);
-#define	cpu_need_proftick(l)	((l)->l_pflag |= LP_OWEUPC, curcpu()->ci_astpending = 1)
+void	cpu_need_proftick(struct lwp *);
-#define	cpu_signotify(l)	(curcpu()->ci_astpending = 1)	/* XXXSMP */
+#define	cpu_did_resched(l)			((l)->l_md.md_astpending = 0)
 
 #if !defined(PPC_IBM4XX) && !defined(PPC_BOOKE)
 void oea_init(void (*)(void));

sys/arch/powerpc/include/cpuset.h

@@ -0,0 +1,57 @@
+/*	$NetBSD: cpuset.h,v 1.1 2011/06/05 16:52:25 matt Exp $	*/
+
+/*-
+ * Copyright (c) 2004 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _POWERPC_CPUSET_H_
+#define	_POWERPC_CPUSET_H_
+
+#include <sys/atomic.h>
+
+#define	CPUSET_SINGLE(cpu)		((__cpuset_t)1U << (cpu))
+
+#define	CPUSET_ADD(set, cpu)		atomic_or_32(&(set), CPUSET_SINGLE(cpu))
+#define	CPUSET_DEL(set, cpu)		atomic_and_32(&(set), ~CPUSET_SINGLE(cpu))
+#define	CPUSET_ADDSET(set1, set2)	atomic_or_32(&(set1), (set2))
+#define	CPUSET_DELSET(set1, set2)	atomic_and_32(&(set1), ~(set2))
+
+#define	CPUSET_EXCEPT(set, cpu)		((set) & ~CPUSET_SINGLE(cpu))
+
+#define	CPUSET_HAS_P(set, cpu)		((set) & CPUSET_SINGLE(cpu))
+#define	CPUSET_INTERSECTS_P(set1, set2)	((set1) & (set2))
+#define	CPUSET_NEXT(set)		(ffs(set) - 1)
+
+#define CPUSET_NULLSET			((__cpuset_t)0)
+#define	CPUSET_EMPTY_P(set)		((set) == (__cpuset_t)0)
+#define	CPUSET_EQUAL_P(set1, set2)	((set1) == (set2))
+#define	CPUSET_CLEAR(set)		((set) = (__cpuset_t)0)
+#define	CPUSET_ASSIGN(set1, set2)	((set1) = (set2))
+#define	CPUSET_MERGE(set1, set2)	((set1) |= (set2))
+#define	CPUSET_REMOVE(set1, set2)	((set1) & ~(set2))
+#define	CPUSET_SUBSET(set1, set2)	((set1) & (set2))
+#define	CPUSET_INVERT(set)		(~(set))
+
+#endif /* _POWERPC_CPUSET_H_ */

sys/arch/powerpc/include/proc.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: proc.h,v 1.11 2011/05/02 02:01:33 matt Exp $	*/
+/*	$NetBSD: proc.h,v 1.12 2011/06/05 16:52:25 matt Exp $	*/
 
 /*-
  * Copyright (C) 1995, 1996 Wolfgang Solfrank.
@@ -39,6 +39,7 @@
  */
 struct mdlwp {
 	volatile int md_flags;
+	volatile int md_astpending;
 	struct trapframe *md_utf;		/* user trampframe */
 };
 #define MDLWP_USEDFPU	__BIT(PCU_FPU)	/* this thread has used the FPU */

sys/arch/powerpc/include/types.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: types.h,v 1.40 2011/05/02 02:01:33 matt Exp $	*/
+/*	$NetBSD: types.h,v 1.41 2011/06/05 16:52:25 matt Exp $	*/
 
 /*-
  * Copyright (C) 1995 Wolfgang Solfrank.
@@ -67,6 +67,7 @@ typedef struct label_t {
 #endif
 
 typedef volatile int __cpu_simple_lock_t;
+typedef volatile __uint32_t __cpuset_t;
 
 #define __SIMPLELOCK_LOCKED	1
 #define __SIMPLELOCK_UNLOCKED	0

sys/arch/powerpc/oea/cpu_subr.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: cpu_subr.c,v 1.62 2011/02/12 01:02:12 matt Exp $	*/
+/*	$NetBSD: cpu_subr.c,v 1.63 2011/06/05 16:52:25 matt Exp $	*/
 
 /*-
  * Copyright (c) 2001 Matt Thomas.
@@ -34,7 +34,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.62 2011/02/12 01:02:12 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.63 2011/06/05 16:52:25 matt Exp $");
 
 #include "opt_ppcparam.h"
 #include "opt_multiprocessor.h"
@@ -232,8 +232,6 @@ static const struct cputab models[] = {
 struct cpu_info cpu_info[CPU_MAXNUM] = {
     [0] = {
 	.ci_curlwp = &lwp0,
-	.ci_fpulwp = &lwp0,
-	.ci_veclwp = &lwp0,
     },
 };
 volatile struct cpu_hatch_data *cpu_hatch_data;
@@ -247,8 +245,6 @@ extern struct bat battable[];
 struct cpu_info cpu_info[1] = {
     [0] = {
 	.ci_curlwp = &lwp0,
-	.ci_fpulwp = &lwp0,
-	.ci_veclwp = &lwp0,
     },
 };
 #endif /*MULTIPROCESSOR*/

sys/arch/powerpc/oea/genassym.cf

@@ -1,4 +1,4 @@
-#	$NetBSD: genassym.cf,v 1.17 2011/01/18 01:02:55 matt Exp $
+#	$NetBSD: genassym.cf,v 1.18 2011/06/05 16:52:25 matt Exp $
 
 #
 # Copyright (C) 1995, 1996 Wolfgang Solfrank.
@@ -41,6 +41,7 @@ include <sys/time.h>
 include <sys/mutex.h>
 include <sys/rwlock.h>
 include <sys/proc.h>
+include <sys/bitops.h>
 
 include <uvm/uvm_extern.h>
 
@@ -213,6 +214,7 @@ define	VREG_VSCR	offsetof(struct vreg, vscr)
 
 define	L_PCB		offsetof(struct lwp, l_addr)
 define	L_CPU		offsetof(struct lwp, l_cpu)
+define	L_MD_ASTPENDING	offsetof(struct lwp, l_md.md_astpending)
 define	L_MD_UTF	offsetof(struct lwp, l_md.md_utf)
 define	L_PROC		offsetof(struct lwp, l_proc)
 
@@ -222,7 +224,6 @@ define	CI_SIZE		sizeof(struct cpu_info)
 define	CI_CURLWP	offsetof(struct cpu_info, ci_curlwp)
 define	CI_CURPCB	offsetof(struct cpu_info, ci_curpcb)
 define	CI_CURPM	offsetof(struct cpu_info, ci_curpm)
-define	CI_ASTPENDING	offsetof(struct cpu_info, ci_astpending)
 define	CI_WANT_RESCHED	offsetof(struct cpu_info, ci_want_resched)
 define	CI_CPL		offsetof(struct cpu_info, ci_cpl)
 define	CI_IDEPTH	offsetof(struct cpu_info, ci_idepth)
@@ -294,3 +295,5 @@ define	RW_OWNER	offsetof(struct krwlock, rw_owner)
 define	RW_WRITE_LOCKED	RW_WRITE_LOCKED
 define	RW_READ_INCR	RW_READ_INCR
 define	RW_READER	RW_READER
+
+define	MSR_PR		31-ilog2(PSL_PR)

sys/arch/powerpc/pic/files.pic

@@ -1,5 +1,5 @@
 # 
-#	$NetBSD: files.pic,v 1.4 2009/08/19 06:22:54 nisimura Exp $
+#	$NetBSD: files.pic,v 1.5 2011/06/05 16:52:26 matt Exp $
 #
 # generic PIC abstraction
 
@@ -17,7 +17,6 @@ defflag opt_interrupt.h PIC_DISTOPENPIC:	pic_distopenpic
 defflag opt_interrupt.h PIC_PREPIVR:	pic_prepivr
 defflag opt_interrupt.h PIC_I8259:	pic_i8259
 defflag opt_interrupt.h PIC_MPCSOC:	pic_mpcsoc
-defflag opt_interrupt.h OPENPIC_DISTRIBUTE
 
 file arch/powerpc/pic/pic_openpic.c	pic_openpic	needs-flag
 file arch/powerpc/pic/pic_distopenpic.c	pic_distopenpic	needs-flag

sys/arch/powerpc/pic/intr.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: intr.c,v 1.10 2010/12/20 00:25:41 matt Exp $ */
+/*	$NetBSD: intr.c,v 1.11 2011/06/05 16:52:26 matt Exp $ */
 
 /*-
  * Copyright (c) 2007 Michael Lorenz
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.10 2010/12/20 00:25:41 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.11 2011/06/05 16:52:26 matt Exp $");
 
 #include "opt_multiprocessor.h"
 
@@ -590,7 +590,7 @@ start:
 #ifdef MULTIPROCESSOR
 	/* THIS IS WRONG XXX */
 	while (realirq == ipiops.ppc_ipi_vector) {
-		ppcipi_intr(NULL);
+		ipi_intr(NULL);
 		pic->pic_ack_irq(pic, realirq);
 		realirq = pic->pic_get_irq(pic, PIC_GET_RECHECK);
 	}

sys/arch/powerpc/pic/ipi.c

@@ -1,4 +1,4 @@
-/* $NetBSD: ipi.c,v 1.8 2011/05/02 02:01:33 matt Exp $ */
+/* $NetBSD: ipi.c,v 1.9 2011/06/05 16:52:26 matt Exp $ */
 /*-
  * Copyright (c) 2007 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -29,7 +29,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.8 2011/05/02 02:01:33 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.9 2011/06/05 16:52:26 matt Exp $");
 
 #include "opt_multiprocessor.h"
 #include "opt_pic.h"
@@ -39,10 +39,8 @@ __KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.8 2011/05/02 02:01:33 matt Exp $");
 #include <sys/param.h>
 #include <sys/kernel.h>
 #include <sys/xcall.h>
-
+#include <sys/atomic.h>
-#include <powerpc/atomic.h>
+#include <sys/cpu.h>
-#include <powerpc/fpu.h>
-#include <powerpc/altivec.h>
 
 #include <arch/powerpc/pic/picvar.h>
 #include <arch/powerpc/pic/ipivar.h>
@@ -51,27 +49,27 @@ __KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.8 2011/05/02 02:01:33 matt Exp $");
 #ifdef MULTIPROCESSOR
 
 struct ipi_ops ipiops;
-volatile u_long IPI[CPU_MAXNUM];
 
 /* Process an actual IPI */
 
 int
-ppcipi_intr(void *v)
+ipi_intr(void *v)
 {
-	int cpu_id = curcpu()->ci_index;
+	struct cpu_info * const ci = curcpu();
+	int cpu_id = cpu_index(ci);
 	int msr;
-	u_long ipi;
+	uint32_t ipi;
 
-	curcpu()->ci_ev_ipi.ev_count++;
+	ci->ci_ev_ipi.ev_count++;
-	ipi = atomic_loadlatch_ulong(&IPI[cpu_id], 0);
+	ipi = atomic_swap_32(&ci->ci_pending_ipis, 0);
 
-	if (ipi == PPC_IPI_NOMESG)
+	if (ipi == IPI_NOMESG)
 		return 1;
 
-	if (ipi & PPC_IPI_XCALL)
+	if (ipi & IPI_XCALL)
 		xc_ipi_handler();
 
-	if (ipi & PPC_IPI_HALT) {
+	if (ipi & IPI_HALT) {
 		aprint_normal("halting CPU %d\n", cpu_id);
 		msr = (mfmsr() & ~PSL_EE) | PSL_POW;
 		for (;;) {
@@ -82,25 +80,4 @@ ppcipi_intr(void *v)
 
 	return 1;
 }
-
-/*
- * MD support for xcall(9) interface.
- */
-
-void
-xc_send_ipi(struct cpu_info *ci)
-{
-
-	KASSERT(kpreempt_disabled());
-	KASSERT(curcpu() != ci);
-
-	if (ci) {
-		/* Unicast: remote CPU. */
-		ppc_send_ipi(ci->ci_cpuid, PPC_IPI_XCALL);
-	} else {
-		/* Broadcast: all, but local CPU (caller will handle it). */
-		ppc_send_ipi(IPI_T_NOTME, PPC_IPI_XCALL);
-	}
-}
-
 #endif /*MULTIPROCESSOR*/

sys/arch/powerpc/pic/ipi_openpic.c

@@ -1,4 +1,4 @@
-/* $NetBSD: ipi_openpic.c,v 1.4 2008/04/28 20:23:32 martin Exp $ */
+/* $NetBSD: ipi_openpic.c,v 1.5 2011/06/05 16:52:26 matt Exp $ */
 /*-
  * Copyright (c) 2007 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -29,18 +29,19 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipi_openpic.c,v 1.4 2008/04/28 20:23:32 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipi_openpic.c,v 1.5 2011/06/05 16:52:26 matt Exp $");
 
 #include "opt_multiprocessor.h"
 #include <sys/param.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
+#include <sys/atomic.h>
+#include <sys/cpu.h>
 
 #include <uvm/uvm_extern.h>
 
 #include <machine/pio.h>
 #include <powerpc/openpic.h>
-#include <powerpc/atomic.h>
 
 #include <arch/powerpc/pic/picvar.h>
 #include <arch/powerpc/pic/ipivar.h>
@@ -48,8 +49,7 @@ __KERNEL_RCSID(0, "$NetBSD: ipi_openpic.c,v 1.4 2008/04/28 20:23:32 martin Exp $
 #ifdef MULTIPROCESSOR
 
 extern struct ipi_ops ipiops;
-extern volatile u_long IPI[CPU_MAXNUM];
+static void openpic_send_ipi(cpuid_t, uint32_t);
-static void openpic_send_ipi(int, u_long);
 static void openpic_establish_ipi(int, int, void *);
 
 void
@@ -74,29 +74,31 @@ setup_openpic_ipi(void)
 }
 
 static void
-openpic_send_ipi(int target, u_long mesg)
+openpic_send_ipi(cpuid_t target, uint32_t mesg)
 {
-	int cpumask = 0, i;
+	struct cpu_info * const ci = curcpu();
+	uint32_t cpumask = 0;
 
-	switch(target) {
+	switch (target) {
-		case IPI_T_ALL:
+		case IPI_DST_ALL:
-			for (i = 0; i < ncpu; i++) {
+		case IPI_DST_NOTME:
-				cpumask |= 1 << i;
+			for (u_int i = 0; i < ncpu; i++) {
-				atomic_setbits_ulong(&IPI[i], mesg);
+				struct cpu_info * const dst_ci = cpu_lookup(i);
+				if (target == IPI_DST_ALL || dst_ci != ci) {
+					cpumask |= 1 << cpu_index(dst_ci);
+					atomic_or_32(&dst_ci->ci_pending_ipis,
+					    mesg);
+				}
 			}
 			break;
-		case IPI_T_NOTME:
+		default: {
-			for (i = 0; i < ncpu; i++) {
+			struct cpu_info * const dst_ci = cpu_lookup(target);
-				if (i != curcpu()->ci_index)
+			cpumask = 1 << cpu_index(dst_ci);
-					cpumask |= 1 << i;
+			atomic_or_32(&dst_ci->ci_pending_ipis, mesg);
-				atomic_setbits_ulong(&IPI[i], mesg);
-			}
 			break;
-		default:
+		}
-			cpumask = 1 << target;
-			atomic_setbits_ulong(&IPI[target], mesg);
 	}
-	openpic_write(OPENPIC_IPI(curcpu()->ci_index, 1), cpumask);
+	openpic_write(OPENPIC_IPI(cpu_index(ci), 1), cpumask);
 }
 
 static void

sys/arch/powerpc/pic/ipivar.h

@@ -1,4 +1,4 @@
-/* $NetBSD: ipivar.h,v 1.4 2010/06/22 18:29:02 rmind Exp $ */
+/* $NetBSD: ipivar.h,v 1.5 2011/06/05 16:52:26 matt Exp $ */
 /*-
  * Copyright (c) 2007 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -29,44 +29,40 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipivar.h,v 1.4 2010/06/22 18:29:02 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipivar.h,v 1.5 2011/06/05 16:52:26 matt Exp $");
 
 #ifndef _IPI_VAR_H_
 #define _IPI_VAR_H_
 
-#include <machine/intr.h>
-
 struct ipi_ops {
-	void (*ppc_send_ipi)(int, u_long);
+	void (*ppc_send_ipi)(cpuid_t, uint32_t);
 	/* type, level, arg */
 	void (*ppc_establish_ipi)(int, int, void *);
 	int ppc_ipi_vector;
 };
 
 /* target macros, 0+ are real cpu numbers */
-#define IPI_T_ALL	-2
+#define IPI_DST_ALL	((cpuid_t) -2)
-#define IPI_T_NOTME	-1
+#define IPI_DST_NOTME	((cpuid_t) -1)
 
-#define PPC_IPI_NOMESG		0x0000
+#define IPI_NOMESG		0x0000
-#define PPC_IPI_HALT		0x0001
+#define IPI_HALT		0x0001
-#define PPC_IPI_FLUSH_FPU	0x0002
+#define IPI_XCALL		0x0002
-#define PPC_IPI_FLUSH_VEC	0x0004
+#define IPI_KPREEMPT		0x0004
-#define PPC_IPI_XCALL		0x0008
 
 /* OpenPIC */
 void setup_openpic_ipi(void);
 
 /* IPI Handler */
-int ppcipi_intr(void *);
+int ipi_intr(void *);
 
 /* convenience */
 extern struct ipi_ops ipiops;
 
 static inline void
-ppc_send_ipi(int cpuid, u_long msg)
+cpu_send_ipi(cpuid_t cpuid, uint32_t msg)
 {
-	ipiops.ppc_send_ipi(cpuid, msg);
+	(*ipiops.ppc_send_ipi)(cpuid, msg);
 }
 
-
 #endif /*_IPI_VAR_H_*/

sys/arch/powerpc/pic/openpic_common.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: openpic_common.c,v 1.3 2008/04/29 06:53:02 martin Exp $ */
+/*	$NetBSD: openpic_common.c,v 1.4 2011/06/05 16:52:26 matt Exp $ */
 
 /*-
  * Copyright (c) 2007 Michael Lorenz
@@ -27,7 +27,10 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: openpic_common.c,v 1.3 2008/04/29 06:53:02 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: openpic_common.c,v 1.4 2011/06/05 16:52:26 matt Exp $");
+
+#include "opt_openpic.h"
+#include "opt_interrupt.h"
 
 #include <sys/param.h>
 #include <sys/malloc.h>
@@ -40,8 +43,6 @@ __KERNEL_RCSID(0, "$NetBSD: openpic_common.c,v 1.3 2008/04/29 06:53:02 martin Ex
 
 #include <arch/powerpc/pic/picvar.h>
 
-#include "opt_interrupt.h"
-
 volatile unsigned char *openpic_base;
 
 void

sys/arch/powerpc/pic/pic_distopenpic.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: pic_distopenpic.c,v 1.4 2008/05/10 15:31:05 martin Exp $ */
+/*	$NetBSD: pic_distopenpic.c,v 1.5 2011/06/05 16:52:26 matt Exp $ */
 
 /*-
  * Copyright (c) 2008 Tim Rightnour
@@ -30,7 +30,10 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pic_distopenpic.c,v 1.4 2008/05/10 15:31:05 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pic_distopenpic.c,v 1.5 2011/06/05 16:52:26 matt Exp $");
+
+#include "opt_openpic.h"
+#include "opt_interrupt.h"
 
 #include <sys/param.h>
 #include <sys/malloc.h>
@@ -43,8 +46,6 @@ __KERNEL_RCSID(0, "$NetBSD: pic_distopenpic.c,v 1.4 2008/05/10 15:31:05 martin E
 
 #include <arch/powerpc/pic/picvar.h>
 
-#include "opt_interrupt.h"
-
 /* distributed stuff */
 static int opic_isu_from_irq(struct openpic_ops *, int, int *);
 static u_int distopic_read(struct openpic_ops *, int, int);

sys/arch/powerpc/powerpc/lock_stubs.S

@@ -1,4 +1,4 @@
-/*	$NetBSD: lock_stubs.S,v 1.7 2011/01/17 08:23:56 matt Exp $	*/
+/*	$NetBSD: lock_stubs.S,v 1.8 2011/06/05 16:52:26 matt Exp $	*/
 
 /*-
  * Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -69,13 +69,11 @@ ENTRY(_lock_cas)
  * void mutex_enter(kmutex_t *);
  */
 ENTRY(mutex_enter)
-	GET_CPUINFO(%r8)
-	ldptr	%r9,CI_CURLWP(%r8)
 1:
 	lptrarx	%r10,0,%r3
 	cmpwi	%r10,0
 	bne-	2f
-	stptrcx. %r9,0,%r3
+	stptrcx. %r13,0,%r3
 	bne-	1b
 	ISYNC
 	blr
@@ -86,13 +84,11 @@ ENTRY(mutex_enter)
  * void mutex_exit(kmutex_t *);
  */
 ENTRY(mutex_exit)
-	GET_CPUINFO(%r8)
-	ldptr	%r9,CI_CURLWP(%r8)
 	SYNC
 	li	%r7,0
 1:
 	lptrarx	%r10,0,%r3
-	cmpw	%r10,%r9
+	cmpw	%r10,%r13
 	bne-	2f
 	stptrcx. %r7,0,%r3
 	bne-	1b
@@ -124,9 +120,7 @@ ENTRY(rw_enter)
 	b	2f
 1:
 	li	%r9,0
-	GET_CPUINFO(%r8)
+	ori	%r7,%r13,RW_WRITE_LOCKED
-	ldptr	%r7,CI_CURLWP(%r8)
-	ori	%r7,%r7,RW_WRITE_LOCKED
 
 2:	lptrarx	%r10,0,%r3
 	cmpw	%r10,%r9
@@ -151,9 +145,7 @@ ENTRY(rw_tryenter)
 	b	2f
 
 1:	li	%r9,0
-	GET_CPUINFO(%r8)
+	ori	%r7,%r13,RW_WRITE_LOCKED
-	ldptr	%r7,CI_CURLWP(%r8)
-	ori	%r7,%r7,RW_WRITE_LOCKED
 
 2:	lptrarx	%r10,0,%r3
 	cmpw	%r10,%r9
@@ -184,9 +176,7 @@ ENTRY(rw_exit)
 	b	2f
 1:
 	li	%r7,0
-	GET_CPUINFO(%r8)
+	ori	%r9,%r13,RW_WRITE_LOCKED
-	ldptr	%r9,CI_CURLWP(%r8)
-	ori	%r9,%r9,RW_WRITE_LOCKED
 
 2:	lptrarx	%r10,0,%r3
 	cmpw	%r10,%r9

sys/arch/powerpc/powerpc/locore_subr.S

@@ -1,4 +1,4 @@
-/*	$NetBSD: locore_subr.S,v 1.41 2011/01/18 01:02:55 matt Exp $	*/
+/*	$NetBSD: locore_subr.S,v 1.42 2011/06/05 16:52:26 matt Exp $	*/
 
 /*
  * Copyright (c) 2001 Wasabi Systems, Inc.
@@ -98,7 +98,7 @@
 	streg	%r10,(SFRAME_USER_SR)(t0);	/* USER_SR */	\
 	streg	%r11,(SFRAME_CR)(t0);		/* CR */	\
 	streg	%r12,(SFRAME_R2)(t0);		/* R2 */	\
-	streg	%r13,(SFRAME_R13)(t0);		/* volatile */	\
+     /* streg	%r13,(SFRAME_R13)(t0); */	/* volatile */	\
 	streg	%r14,(SFRAME_R14)(t0);				\
 	streg	%r15,(SFRAME_R15)(t0);				\
 	streg	%r16,(SFRAME_R16)(t0);				\
@@ -120,7 +120,7 @@
 	ldreg	%r10,(SFRAME_USER_SR)(t0);	/* USER_SR */	\
 	ldreg	%r11,(SFRAME_CR)(t0);		/* CR */	\
 	ldreg	%r12,(SFRAME_R2)(t0);		/* R2 */	\
-	ldreg	%r13,(SFRAME_R13)(t0);		/* volatile */	\
+     /* ldreg	%r13,(SFRAME_R13)(t0); */	/* volatile */	\
 	ldreg	%r14,(SFRAME_R14)(t0);				\
 	ldreg	%r15,(SFRAME_R15)(t0);				\
 	ldreg	%r16,(SFRAME_R16)(t0);				\
@@ -217,6 +217,7 @@ switchto_restore:
 
 	GET_CPUINFO(%r7)
 	stptr	%r31,CI_CURLWP(%r7)
+	mr	%r13,%r31
 #ifdef PPC_BOOKE
 	mtsprg2	%r31			/* save curlwp in sprg2 */
 #endif
@@ -381,6 +382,7 @@ _ENTRY(softint_fast_dispatch)
 	 */
 
 	stptr	%r3, CI_CURLWP(%r7)
+	mr	%r13, %r3
 #ifdef PPC_BOOKE
 	mtsprg2	%r3
 #endif
@@ -414,6 +416,7 @@ _ENTRY(softint_fast_dispatch)
 
 	GET_CPUINFO(%r7)
 	stptr	%r30, CI_CURLWP(%r7)
+	mr	%r13, %r30
 #ifdef PPC_BOOKE
 	mtsprg2	%r30
 #endif
@@ -484,6 +487,7 @@ _ENTRY(cpu_lwp_bootstrap)
 	 * r3 (old lwp) and r4 (new lwp) are setup in cpu_switchto.
 	 */
 	bl	_C_LABEL(lwp_startup)
+
 	/*
 	 * Fall through into setfunc_trampoline
 	 */
@@ -504,8 +508,13 @@ _ENTRY(setfunc_trampoline)
 	tweqi	%r0,0
 #endif
 	li	%r4, 1			/* make sure userret gets called */
+#if 0
 	GET_CPUINFO(%r3)
-	stint	%r4, CI_ASTPENDING(%r3)
+	ldptr	%r3, CI_CURLWP(%r3)
+	stint	%r4, L_MD_ASTPENDING(%r3)
+#else
+	stint	%r4, L_MD_ASTPENDING(%r13)
+#endif
 	b	trapexit
 
 #if defined(MULTIPROCESSOR) && (defined(PPC_OEA) || defined (PPC_OEA64_BRIDGE))

sys/arch/powerpc/powerpc/ofwreal.S

@@ -1,4 +1,4 @@
-/*	$NetBSD: ofwreal.S,v 1.12 2011/01/17 08:23:56 matt Exp $	*/
+/*	$NetBSD: ofwreal.S,v 1.13 2011/06/05 16:52:26 matt Exp $	*/
 
 /*
  * Copyright (C) 1996 Wolfgang Solfrank.
@@ -95,8 +95,10 @@ ENTRY(ofwr_init)
 fwentry:
 	mflr	%r0			/* save return address */
 	stw	%r0,4(%r1)
-	stwu	%r1,-16(%r1)		/* setup stack frame */
+	stwu	%r1,-32(%r1)		/* setup stack frame */
 	stw	%r3,8(%r1)		/* save arg */
+	stw	%r2,16(%r1)		/* save r2 */
+	stw	%r13,20(%r1)		/* save r13 */
 
 	lis	%r3,clsave@ha		/* save mmu values of client */
 	addi	%r3,%r3,clsave@l
@@ -133,6 +135,8 @@ fwentry:
 	addi	%r3,%r3,clsave@l
 	bl	restoremmu
 
+	lwz	%r13,20(%r1)		/* restore saved value */
+	lwz	%r2,16(%r1)		/* restore saved value */
 	lwz	%r3,8(%r1)		/* restore return value */
 	lwz	%r1,0(%r1)		/* and return */
 	lwz	%r0,4(%r1)

sys/arch/powerpc/powerpc/powerpc_machdep.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: powerpc_machdep.c,v 1.48 2011/05/02 02:01:33 matt Exp $	*/
+/*	$NetBSD: powerpc_machdep.c,v 1.49 2011/06/05 16:52:26 matt Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996 Wolfgang Solfrank.
@@ -32,7 +32,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: powerpc_machdep.c,v 1.48 2011/05/02 02:01:33 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: powerpc_machdep.c,v 1.49 2011/06/05 16:52:26 matt Exp $");
 
 #include "opt_altivec.h"
 #include "opt_modular.h"
@@ -53,6 +53,8 @@ __KERNEL_RCSID(0, "$NetBSD: powerpc_machdep.c,v 1.48 2011/05/02 02:01:33 matt Ex
 #include <sys/module.h>
 #include <sys/device.h>
 #include <sys/pcu.h>
+#include <sys/atomic.h>
+#include <sys/xcall.h>
 
 #include <powerpc/pcb.h>
 #include <powerpc/fpu.h>
@@ -333,6 +335,95 @@ cpu_idle(void)
 	(*curcpu()->ci_idlespin)();
 }
 
+void
+cpu_need_resched(struct cpu_info *ci, int flags)
+{
+	struct lwp * const l = ci->ci_data.cpu_onproc;
+#if defined(MULTIPROCESSOR)
+	struct cpu_info * const cur_ci = curcpu();
+#endif
+
+	KASSERT(kpreempt_disabled());
+
+#ifdef MULTIPROCESSOR
+	atomic_or_uint(&ci->ci_want_resched, flags);
+#else
+	ci->ci_want_resched |= flags;
+#endif
+
+	if (__predict_false((l->l_pflag & LP_INTR) != 0)) {
+		/*
+		 * No point doing anything, it will switch soon.
+		 * Also here to prevent an assertion failure in
+		 * kpreempt() due to preemption being set on a
+		 * soft interrupt LWP.
+		 */
+		return;
+	}
+
+	if (__predict_false(l == ci->ci_data.cpu_idlelwp)) {
+#if defined(MULTIPROCESSOR)
+		/*
+		 * If the other CPU is idling, it must be waiting for an
+		 * interrupt.  So give it one.
+		 */
+		if (__predict_false(ci != cur_ci))
+			cpu_send_ipi(cpu_index(ci), IPI_NOMESG);
+#endif
+		return;
+	}
+
+#ifdef __HAVE_PREEMPTION
+	if (flags & RESCHED_KPREEMPT) {
+		atomic_or_uint(&l->l_dopreempt, DOPREEMPT_ACTIVE);
+		if (ci == cur_ci) {
+			softint_trigger(SOFTINT_KPREEMPT);
+                } else {
+                        cpu_send_ipi(cpu_index(ci), IPI_KPREEMPT);
+                }
+		return;
+	}
+#endif
+	l->l_md.md_astpending = 1;		/* force call to ast() */
+#if defined(MULTIPROCESSOR)
+	if (ci != cur_ci && (flags & RESCHED_IMMED)) {
+		cpu_send_ipi(cpu_index(ci), IPI_NOMESG);
+	} 
+#endif
+}
+
+void
+cpu_need_proftick(lwp_t *l)
+{
+	l->l_pflag |= LP_OWEUPC;
+	l->l_md.md_astpending = 1;
+}
+
+void
+cpu_signotify(lwp_t *l)
+{
+	l->l_md.md_astpending = 1;
+}
+
+#ifdef MULTIPROCESSOR
+/*
+ * MD support for xcall(9) interface.
+ */
+
+void
+xc_send_ipi(struct cpu_info *ci)
+{
+	KASSERT(kpreempt_disabled());
+	KASSERT(curcpu() != ci);
+
+	cpuid_t target = (ci != NULL ? cpu_index(ci) : IPI_DST_NOTME);
+
+	/* Unicast: remote CPU. */
+	/* Broadcast: all, but local CPU (caller will handle it). */
+	cpu_send_ipi(target, IPI_XCALL);
+}
+#endif /* MULTIPROCESSOR */
+
 #ifdef MODULAR
 /*
  * Push any modules loaded by the boot loader.

sys/arch/powerpc/powerpc/trap.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: trap.c,v 1.140 2011/05/02 02:01:33 matt Exp $	*/
+/*	$NetBSD: trap.c,v 1.141 2011/06/05 16:52:26 matt Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996 Wolfgang Solfrank.
@@ -32,7 +32,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.140 2011/05/02 02:01:33 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.141 2011/06/05 16:52:26 matt Exp $");
 
 #include "opt_altivec.h"
 #include "opt_ddb.h"
@@ -347,7 +347,7 @@ trap(struct trapframe *tf)
 		break;
 
 	case EXC_AST|EXC_USER:
-		ci->ci_astpending = 0;		/* we are about to do it */
+		l->l_md.md_astpending = 0;	/* we are about to do it */
 		//ci->ci_data.cpu_nast++;
 		if (l->l_pflag & LP_OWEUPC) {
 			l->l_pflag &= ~LP_OWEUPC;

sys/arch/powerpc/powerpc/trap_subr.S

@@ -1,4 +1,4 @@
-/*	$NetBSD: trap_subr.S,v 1.68 2011/04/07 02:05:02 matt Exp $	*/
+/*	$NetBSD: trap_subr.S,v 1.69 2011/06/05 16:52:26 matt Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996 Wolfgang Solfrank.
@@ -236,7 +236,7 @@ _C_LABEL(dsitrap):
 	mtsprg2	%r30			/* in SPRG2 */
 	mfsrr1	%r31			/* test kernel mode */
 	mtcr	%r31
-	bt	17,1f			/* branch if PSL_PR is set */
+	bt	MSR_PR,1f		/* branch if PSL_PR is set */
 	mfdar	%r31			/* get fault address */
 	rlwinm	%r31,%r31,7,25,28	/* get segment * 8 */
 
@@ -304,7 +304,7 @@ _C_LABEL(dsi601trap):
 	mtsprg2	%r30			/* in SPRG2 */
 	mfsrr1	%r31			/* test kernel mode */
 	mtcr	%r31
-	bt	17,1f			/* branch if PSL_PR is set */
+	bt	MSR_PR,1f		/* branch if PSL_PR is set */
 	mfdar	%r31			/* get fault address */
 	rlwinm	%r31,%r31,12,20,28	/* get "segment" battable offset */
 
@@ -371,7 +371,7 @@ _C_LABEL(extint):
 	mtcr	%r31
 	mr	%r30,%r1
 	mfsprg1	%r1			/* get old SP */
-	bf	17,1f			/* branch if PSL_PR is true */
+	bf	MSR_PR,1f		/* branch if PSL_PR is true */
 	ldptr	%r1,CI_CURPCB(%r30)	/* get kernel stack */
 	addi	%r1,%r1,USPACE-CALLFRAMELEN /* stack is top of user struct */
 	RESTORE_KERN_SRS(%r30, %r31)
@@ -404,7 +404,7 @@ _C_LABEL(decrint):
 	mtcr	%r31
 	mr	%r30,%r1
 	mfsprg1	%r1			/* yes, get old SP */
-	bf	17,1f			/* branch if PSL_PR is true */
+	bf	MSR_PR,1f		/* branch if PSL_PR is true */
 	ldptr	%r1,CI_CURPCB(%r30)	/* get kernel stack */
 	addi	%r1,%r1,USPACE-CALLFRAMELEN /* stack is top of user struct */
 	RESTORE_KERN_SRS(%r30, %r31)
@@ -641,7 +641,7 @@ _C_LABEL(ddblow):
 	mfsrr1	%r1
 	mtcr	%r1
 	GET_CPUINFO(%r1)
-	bf	17,1f			/* branch if privileged */
+	bf	MSR_PR,1f		/* branch if privileged */
 	streg	%r28,(CI_TEMPSAVE+CPUSAVE_R28)(%r1)	/* free r28 */
 	mfsprg2	%r28
 	streg	%r28,(CI_TEMPSAVE+CPUSAVE_R29)(%r1)	/* free r29 */
@@ -779,10 +779,11 @@ _C_LABEL(ipkdbsize) = .-_C_LABEL(ipkdblow)
 	ldreg	%r29,(savearea+CPUSAVE_DSISR)(%r2); /* get saved DSISR */ \
 	ldreg	%r30,(savearea+CPUSAVE_SRR0)(%r2); /* get saved SRR0 */	\
 	ldreg	%r31,(savearea+CPUSAVE_SRR1)(%r2); /* get saved SRR1 */	\
-	mfxer	%r3;							\
+	ldptr	%r13,CI_CURLWP(%r2);	/* get curlwp */	\
-	mfctr	%r4;							\
+	mfxer	%r3;						\
-	mflr	%r5;							\
+	mfctr	%r4;						\
-	andi.	%r5,%r5,0xff00;						\
+	mflr	%r5;						\
+	andi.	%r5,%r5,0xff00;					\
 	stint	%r3,FRAME_XER(%r1);				\
 	streg	%r4,FRAME_CTR(%r1);				\
 	streg	%r30,FRAME_SRR0(%r1);			\
@@ -811,8 +812,7 @@ _C_LABEL(ipkdbsize) = .-_C_LABEL(ipkdblow)
 	ldreg	%r17,FRAME_R17(%r1);	/* restore r17 */	\
 	ldreg	%r16,FRAME_R16(%r1);	/* restore r16 */	\
 	ldreg	%r15,FRAME_R15(%r1);	/* restore r15 */	\
-	ldreg	%r14,FRAME_R14(%r1);	/* restore r14 */	\
+	ldreg	%r14,FRAME_R14(%r1);	/* restore r14 */
-	ldreg	%r13,FRAME_R13(%r1);	/* restore r13 */
 
 #define	FRAME_LEAVE(savearea)					\
 /* Now restore regs: */						\
@@ -831,6 +831,7 @@ _C_LABEL(ipkdbsize) = .-_C_LABEL(ipkdblow)
 	mtxer	%r5;							\
 	mtlr	%r6;							\
 	mtsprg1	%r7;			/* save cr */			\
+	ldreg	%r13,FRAME_R13(%r1);	/* restore r13 */	\
 	ldreg	%r12,FRAME_R12(%r1);	/* restore r12 */	\
 	ldreg	%r11,FRAME_R11(%r1);	/* restore r11 */	\
 	ldreg	%r10,FRAME_R10(%r1);	/* restore r10 */	\
@@ -856,7 +857,7 @@ _C_LABEL(ipkdbsize) = .-_C_LABEL(ipkdblow)
 	GET_CPUINFO(%r2);						\
 	ldreg	%r3,(savearea+CPUSAVE_SRR1)(%r2);			\
 	mtcr	%r3;							\
-	bf	17,1f;			/* branch if PSL_PR is false */	\
+	bf	MSR_PR,1f;		/* branch if PSL_PR is false */	\
 /* Restore user SRs */							\
 	RESTORE_USER_SRS(%r2,%r3);					\
 1:	mfsprg1	%r2;			/* restore cr */		\
@@ -890,7 +891,7 @@ disitrap:
 #ifdef DDB
 	mfsrr1	%r31
 	mtcr	%r31
-	bt	17,trapstart		/* branch is user mode */
+	bt	MSR_PR,trapstart	/* branch is user mode */
 	mfsprg1	%r31			/* get old SP */
 #if 0
 	subf	%r30,%r30,%r31		/* subtract DAR from it */
@@ -926,7 +927,7 @@ realtrap:
 	mfsprg1	%r1			/* restore SP (might have been
 					   overwritten) */
 s_trap:
-	bf	17,k_trap		/* branch if PSL_PR is false */
+	bf	MSR_PR,k_trap		/* branch if PSL_PR is false */
 	GET_CPUINFO(%r1)		/* get cpu_info for this cpu */
 u_trap:
 	ldptr	%r1,CI_CURPCB(%r1)
@@ -960,9 +961,14 @@ trapexit:
 	mtmsr	%r3
 /* Test AST pending: */
 	mtcr	%r31
-	bf	17,trapleave			/* branch if PSL_PR is false */
+	bf	MSR_PR,trapleave	/* branch if PSL_PR is false */
+#if 0
 	GET_CPUINFO(%r3)
-	ldint	%r4,CI_ASTPENDING(%r3)
+	ldptr	%r3,CI_CURLWP(%r3)
+	ldint	%r4,L_MD_ASTPENDING(%r3)
+#else
+	ldint	%r4,L_MD_ASTPENDING(%r13)
+#endif
 	andi.	%r4,%r4,1
 	beq	trapleave
 
@@ -1013,9 +1019,13 @@ s_sctrap:
 	isync
 	addi	%r3,%r1,FRAME_TF
 /* Call the appropriate syscall handler: */
+#if 0
 	GET_CPUINFO(%r4)
 	ldptr	%r4,CI_CURLWP(%r4)
 	ldptr	%r4,L_PROC(%r4)
+#else
+	ldptr	%r4,L_PROC(%r13)
+#endif
 	ldptr	%r4,P_MD_SYSCALL(%r4)
 	mtctr	%r4
 	bctrl
@@ -1069,8 +1079,10 @@ _C_LABEL(sctrapexit):
 	streg	%r10,FRAME_R10(%r1);	/* save r10 */		\
 	streg	%r11,FRAME_R11(%r1);	/* save r11 */		\
 	streg	%r12,FRAME_R12(%r1);	/* save r12 */		\
+	streg	%r13,FRAME_R13(%r1);	/* save r13 */		\
 	ldreg	%r11,(savearea+CPUSAVE_SRR0)(%r2); /* get saved SRR0 */	\
 	ldreg	%r12,(savearea+CPUSAVE_SRR1)(%r2); /* get saved SRR1 */	\
+	ldptr	%r13,CI_CURLWP(%r2);	/* get curlwp */	\
 	ldint	%r3,CI_IDEPTH(%r2);					\
 	addi	%r4,%r3,1;						\
 	stint	%r4,CI_IDEPTH(%r2);					\
@@ -1111,11 +1123,16 @@ intr_exit:
 	stint	%r4,CI_IDEPTH(%r5)
 
 /* Returning to user mode? */
-	ldreg	%r5,FRAME_SRR1(%r1)
+	ldreg	%r4,FRAME_SRR1(%r1)
-	mtcr	%r5			/* saved SRR1 */
+	mtcr	%r4			/* saved SRR1 */
-	bf	17,intrleave		/* branch if PSL_PR is false */
+	bf	MSR_PR,intrleave	/* branch if PSL_PR is false */
 
-	ldint	%r3,CI_ASTPENDING(%r5)	/* Test AST pending */
+#if 0
+	ldptr	%r3,CI_CURLWP(%r5)
+	ldint	%r3,L_MD_ASTPENDING(%r3) /* Test AST pending */
+#else
+	ldint	%r3,L_MD_ASTPENDING(%r13) /* Test AST pending */
+#endif
 	andi.	%r3,%r3,1
 	beq	intrleave		/* common frame exit */
 
@@ -1124,7 +1141,6 @@ intr_exit:
  * process the AST is finish filling the trapframe with the rest of the fixed
  * registers and let trap deal with it.
  */
-	streg	%r13,FRAME_R13(%r1)
 	streg	%r14,FRAME_R14(%r1)
 	streg	%r15,FRAME_R15(%r1)
 	streg	%r16,FRAME_R16(%r1)

sys/arch/prep/include/atomic.h

@@ -1,3 +0,0 @@
-/*	$NetBSD: atomic.h,v 1.2 2005/12/11 12:18:47 christos Exp $	*/
-
-#include <powerpc/atomic.h>

sys/arch/rs6000/include/atomic.h

@@ -1,3 +0,0 @@
-/*	$NetBSD: atomic.h,v 1.1 2007/12/17 19:09:07 garbled Exp $	*/
-
-#include <powerpc/atomic.h>

sys/arch/sandpoint/include/atomic.h

@@ -1,3 +0,0 @@
-/*	$NetBSD: atomic.h,v 1.2 2005/12/11 12:18:51 christos Exp $	*/
-
-#include <powerpc/atomic.h>

sys/common/pmap/tlb/pmap.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: pmap.c,v 1.3 2011/01/18 01:20:06 matt Exp $	*/
+/*	$NetBSD: pmap.c,v 1.4 2011/06/05 16:52:27 matt Exp $	*/
 
 /*-
  * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
@@ -67,7 +67,7 @@
 
 #include <sys/cdefs.h>
 
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.3 2011/01/18 01:20:06 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.4 2011/06/05 16:52:27 matt Exp $");
 
 /*
  *	Manages physical address maps.
@@ -98,6 +98,8 @@ __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.3 2011/01/18 01:20:06 matt Exp $");
 #include "opt_sysv.h"
 #include "opt_multiprocessor.h"
 
+#define __PMAP_PRIVATE
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
@@ -317,27 +319,34 @@ pmap_page_set_attributes(struct vm_page *pg, u_int set_attributes)
 #endif
 }
 
-static inline void
+static void
 pmap_page_syncicache(struct vm_page *pg)
 {
-#ifdef MULTIPROCESSOR
+#ifndef MULTIPROCESSOR
+	struct pmap * const curpmap = curcpu()->ci_curpm;
+#endif
 	pv_entry_t pv = &VM_PAGE_TO_MD(pg)->mdpg_first;
-	uint32_t onproc = 0;
+	__cpuset_t onproc = CPUSET_NULLSET;
 	(void)VM_PAGE_PVLIST_LOCK(pg, false);
 	if (pv->pv_pmap != NULL) {
 		for (; pv != NULL; pv = pv->pv_next) {
-			onproc |= pv->pv_pmap->pm_onproc;
+#ifdef MULTIPROCESSOR
-			if (onproc == cpus_running)
+			CPUSET_MERGE(onproc, pv->pv_pmap->pm_onproc);
+			if (CPUSET_EQUAL_P(onproc, cpus_running)) {
 				break;
+			}
+#else
+			if (pv->pv_pmap == curpmap) {
+				onproc = CPUSET_SINGLE(0);
+				break;
+			}
+#endif
 		}
 	}
 	VM_PAGE_PVLIST_UNLOCK(pg);
 	kpreempt_disable();
-	pmap_tlb_syncicache(VM_PAGE_TO_MD(pg)->mdpg_first.pv_va, onproc);
+	pmap_md_page_syncicache(pg, onproc);
 	kpreempt_enable();
-#else
-	pmap_md_page_syncicache(pg);
-#endif
 }
 
 /*
@@ -1582,7 +1591,7 @@ pmap_pvlist_lock_init(size_t cache_line_size)
 	struct pmap_pvlist_info * const pli = &pmap_pvlist_info;
 	const vaddr_t lock_page = uvm_pageboot_alloc(PAGE_SIZE);
 	vaddr_t lock_va = lock_page;
-	if (sizeof(kmutex_t) > dcache_line_size) {
+	if (sizeof(kmutex_t) > cache_line_size) {
 		cache_line_size = roundup2(sizeof(kmutex_t), cache_line_size);
 	}
 	const size_t nlocks = PAGE_SIZE / cache_line_size;

sys/common/pmap/tlb/pmap.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: pmap.h,v 1.2 2011/01/18 01:11:50 matt Exp $	*/
+/*	$NetBSD: pmap.h,v 1.3 2011/06/05 16:52:27 matt Exp $	*/
 
 /*
  * Copyright (c) 1992, 1993
@@ -126,8 +126,8 @@ struct pmap_asid_info {
  */
 struct pmap {
 #ifdef MULTIPROCESSOR
-	volatile uint32_t	pm_active;	/* pmap was active on ... */
+	__cpuset_t		pm_active;	/* pmap was active on ... */
-	volatile uint32_t	pm_onproc;	/* pmap is active on ... */
+	__cpuset_t		pm_onproc;	/* pmap is active on ... */
 	volatile u_int		pm_shootdown_pending;
 #endif
 	struct pmap_segtab	*pm_segtab;	/* pointers to pages of PTEs */
@@ -161,7 +161,7 @@ struct pmap_tlb_info {
 #ifdef MULTIPROCESSOR
 	pmap_t ti_victim;
 	uint32_t ti_synci_page_bitmap;	/* page indices needing a syncicache */
-	uint32_t ti_cpu_mask;		/* bitmask of CPUs sharing this TLB */
+	__cpuset_t ti_cpu_mask;		/* bitmask of CPUs sharing this TLB */
 	enum tlb_invalidate_op ti_tlbinvop;
 	u_int ti_index;
 #define tlbinfo_index(ti)	((ti)->ti_index)

sys/common/pmap/tlb/pmap_tlb.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: pmap_tlb.c,v 1.3 2011/02/17 13:55:45 matt Exp $	*/
+/*	$NetBSD: pmap_tlb.c,v 1.4 2011/06/05 16:52:27 matt Exp $	*/
 
 /*-
  * Copyright (c) 2010 The NetBSD Foundation, Inc.
@@ -31,7 +31,7 @@
 
 #include <sys/cdefs.h>
 
-__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.3 2011/02/17 13:55:45 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.4 2011/06/05 16:52:27 matt Exp $");
 
 /*
  * Manages address spaces in a TLB.
@@ -122,7 +122,7 @@ __KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.3 2011/02/17 13:55:45 matt Exp $");
  * a lot of overhead for not much gain.
  */
 
-#define _PMAP_PRIVATE
+#define __PMAP_PRIVATE
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -223,7 +223,7 @@ pmap_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai,
 	 * The bits in pm_active belonging to this TLB can only be changed
 	 * while this TLB's lock is held.
 	 */
-	atomic_and_32(&pm->pm_active, ~ti->ti_cpu_mask);
+	CPUSET_DELSET(pm->pm_active, ti->ti_cpu_mask);
 #endif /* MULTIPROCESSOR */
 }
 
@@ -278,15 +278,15 @@ pmap_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci)
 	KASSERT(cold);
 
 	TLBINFO_LOCK(ti);
-	uint32_t cpu_mask = 1 << cpu_index(ci);
+	const __cpuset_t cpu_mask = CPUSET_SINGLE(cpu_index(ci));
-	ti->ti_cpu_mask |= cpu_mask;
+	CPUSET_ADDSET(ti->ti_cpu_mask, cpu_mask);
 	ci->ci_tlb_info = ti;
 	ci->ci_ksp_tlb_slot = ti->ti_wired++;
 	/*
 	 * Mark the kernel as active and "onproc" for this cpu.
 	 */
-	pmap_kernel()->pm_active |= cpu_mask;
+	CPUSET_ADDSET(pmap_kernel()->pm_active, cpu_mask);
-	pmap_kernel()->pm_onproc |= cpu_mask;
+	CPUSET_ADDSET(pmap_kernel()->pm_onproc, cpu_mask);
 	TLBINFO_UNLOCK(ti);
 }
 #endif /* MULTIPROCESSOR */
@@ -367,7 +367,7 @@ pmap_tlb_asid_reinitialize(struct pmap_tlb_info *ti, enum tlb_invalidate_op op)
 		KASSERT(pm != pmap_kernel());
 		KASSERT(pai->pai_asid > KERNEL_PID);
 #ifdef MULTIPROCESSOR
-		if (pm->pm_onproc & ti->ti_cpu_mask) {
+		if (!CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_onproc, ti->ti_cpu_mask)) {
 			if (!TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
 				TLBINFO_ASID_MARK_USED(ti, pai->pai_asid);
 				ti->ti_asids_free--;
@@ -406,7 +406,7 @@ pmap_tlb_shootdown_process(void)
 		 */
 		struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti);
 		KASSERT(ti->ti_victim != pmap_kernel());
-		if (ti->ti_victim->pm_onproc & ti->ti_cpu_mask) {
+		if (!CPU_EMPTY(CPUSET_SUBSET(ti->ti_victim->pm_onproc, ti->ti_cpu_mask)) {
 			/*
 			 * The victim is an active pmap so we will just
 			 * invalidate its TLB entries.
@@ -420,7 +420,7 @@ pmap_tlb_shootdown_process(void)
 			 * next called for this pmap, it will allocate a new
 			 * ASID.
 			 */
-			KASSERT((pm->pm_onproc & ti->ti_cpu_mask) == 0);
+			KASSERT(!CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_onproc, ti->ti_cpu_mask)));
 			pmap_pai_reset(ti, pai, PAI_PMAP(pai, ti));
 		}
 		break;
@@ -485,7 +485,8 @@ pmap_tlb_shootdown_bystanders(pmap_t pm)
 	/*
 	 * We don't need to deal our own TLB.
 	 */
-	uint32_t pm_active = pm->pm_active & ~curcpu()->ci_tlb_info->ti_cpu_mask;
+	__cpuset_t pm_active =
+	   CPUSET_EXCLUDE(pm->pm_active, curcpu()->ci_tlb_info->ti_cpu_mask);
 	const bool kernel_p = (pm == pmap_kernel());
 	bool ipi_sent = false;
 
@@ -494,19 +495,20 @@ pmap_tlb_shootdown_bystanders(pmap_t pm)
 	 * have been made so they will already be cognizant of them.
 	 */
 
-	for (size_t i = 0; pm_active != 0; i++) {
+	for (size_t i = 0; !CPUSET_EMPTY_P(pm_active); i++) {
 		KASSERT(i < pmap_ntlbs);
 		struct pmap_tlb_info * const ti = pmap_tlbs[i];
 		KASSERT(tlbinfo_index(ti) == i);
 		/*
 		 * Skip this TLB if there are no active mappings for it.
 		 */
-		if ((pm_active & ti->ti_cpu_mask) == 0)
+		if (CPUSET_EMPTY_P(CPUSET_SUBSET(pm_active, ti->ti_cpu_mask)))
 			continue;
 		struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
-		pm_active &= ~ti->ti_cpu_mask;
+		CPUSET_DELSET(pm_active, ti->ti_cpu_mask);
 		TLBINFO_LOCK(ti);
-		const uint32_t onproc = (pm->pm_onproc & ti->ti_cpu_mask);
+		const __cpuset onproc = CPUSET_SUBSET(pm->pm_onproc,
+		    ti->ti_cpu_mask);
 		if (onproc != 0) {
 			if (kernel_p) {
 				ti->ti_tlbinvop =
@@ -539,13 +541,13 @@ pmap_tlb_shootdown_bystanders(pmap_t pm)
 			 * change now that we have released the lock but we
 			 * can tolerate spurious shootdowns.
 			 */
-			KASSERT(onproc != 0);
+			KASSERT(!CPU_EMPTY_P(onproc));
-			u_int j = ffs(onproc) - 1;
+			u_int j = CPUSET_NEXT(onproc);
 			cpu_send_ipi(cpu_lookup(j), IPI_SHOOTDOWN);
 			ipi_sent = true;
 			continue;
 		}
-		if (pm->pm_active & ti->ti_cpu_mask) {
+		if (!CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_active, ti->ti_cpu_mask) {
 			/*
 			 * If this pmap has an ASID assigned but it's not
 			 * currently running, nuke its ASID.  Next time the
@@ -611,8 +613,8 @@ pmap_tlb_asid_alloc(struct pmap_tlb_info *ti, pmap_t pm,
 	KASSERT(pai->pai_asid == 0);
 	KASSERT(pai->pai_link.le_prev == NULL);
 #ifdef MULTIPROCESSOR
-	KASSERT((pm->pm_onproc & ti->ti_cpu_mask) == 0);
+	KASSERT(CPU_EMPTY_P(CPUSET_INTESECTION(pm->pm_onproc, ti->ti_cpu_mask)));
-	KASSERT((pm->pm_active & ti->ti_cpu_mask) == 0);
+	KASSERT(CPU_EMPTY_P(CPUSET_INTESECTION(pm->pm_active, ti->ti_cpu_mask)));
 #endif
 	KASSERT(ti->ti_asids_free > 0);
 	KASSERT(ti->ti_asid_hint <= ti->ti_asid_max);
@@ -716,7 +718,7 @@ pmap_tlb_asid_acquire(pmap_t pm, struct lwp *l)
 		 * The bits in pm_onproc belonging to this TLB can only
 		 * be changed while this TLBs lock is held.
 		 */
-		atomic_or_32(&pm->pm_onproc, 1 << cpu_index(ci));
+		CPUSET_ADD(&pm->pm_onproc, cpu_index(ci));
 #endif
 		tlb_set_asid(pai->pai_asid);
 	}
@@ -748,7 +750,7 @@ pmap_tlb_asid_deactivate(pmap_t pm)
 		 * The bits in pm_onproc belonging to this TLB can only
 		 * be changed while this TLBs lock is held.
 		 */
-		atomic_and_32(&pm->pm_onproc, ~(1 << cpu_index(ci)));
+		CPUSET_DEL(pm->pm_onproc, cpu_index(ci));
 		TLBINFO_UNLOCK(ti);
 	}
 #endif
@@ -759,11 +761,11 @@ pmap_tlb_asid_release_all(struct pmap *pm)
 {
 #if defined(MULTIPROCESSOR)
 	KASSERT(pm != pmap_kernel());
-	KASSERT(pm->pm_onproc == 0);
+	KASSERT(CPUSET_EMPTY(pm->pm_onproc));
-	for (u_int i = 0; pm->pm_active != 0; i++) {
+	for (u_int i = 0; !CPUSET_EMPTY(pm->pm_active); i++) {
 		KASSERT(i < pmap_ntlbs);
 		struct pmap_tlb_info * const ti = pmap_tlbs[i];
-		if (pm->pm_active & ti->ti_cpu_mask) {
+		if (!CPU_EMPTY_P(CPUSET_INTESECTION(pm->pm_active, ti->ti_cpu_mask))) {
 			struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
 			TLBINFO_LOCK(ti);
 			KASSERT(ti->ti_victim != pm);