Add the DTrace hooks to the kernel (KDTRACE_HOOKS config option).

DTrace adds a pointer to the lwp and proc structures which it uses to
manage its state.  These are opaque from the kernel perspective to keep
the kernel free of CDDL code. The state arenas are kmem_alloced and freed
as proccesses and threads are created and destoyed.

Also add a check for trap06 (privileged/illegal instruction) so that
DTrace can check for D scripts that may have triggered the trap so it
can clean up after them and resume normal operation.

Ok with core@.

sys/arch/i386/i386/trap.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: trap.c,v 1.253 2010/01/17 22:21:18 dsl Exp $	*/
+/*	$NetBSD: trap.c,v 1.254 2010/02/21 02:11:40 darran Exp $	*/
 
 /*-
  * Copyright (c) 1998, 2000, 2005, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -68,7 +68,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.253 2010/01/17 22:21:18 dsl Exp $");
+__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.254 2010/02/21 02:11:40 darran Exp $");
 
 #include "opt_ddb.h"
 #include "opt_kgdb.h"
@@ -120,6 +120,20 @@ __KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.253 2010/01/17 22:21:18 dsl Exp $");
 
 #include "npx.h"
 
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+
+/*
+ * This is a hook which is initialised by the dtrace module
+ * to handle traps which might occur during DTrace probe
+ * execution.
+ */
+dtrace_trap_func_t	dtrace_trap_func = NULL;
+
+dtrace_doubletrap_func_t	dtrace_doubletrap_func = NULL;
+#endif
+
+
 static inline int xmm_si_code(struct lwp *);
 void trap(struct trapframe *);
 void trap_tss(struct i386tss *, int, int);
@@ -338,6 +352,27 @@ trap(struct trapframe *frame)
 		LWP_CACHE_CREDS(l, p);
 	}
 
+#ifdef KDTRACE_HOOKS
+	/*
+	 * A trap can occur while DTrace executes a probe. Before
+	 * executing the probe, DTrace blocks re-scheduling and sets
+	 * a flag in it's per-cpu flags to indicate that it doesn't
+	 * want to fault. On returning from the the probe, the no-fault
+	 * flag is cleared and finally re-scheduling is enabled.
+	 *
+	 * If the DTrace kernel module has registered a trap handler,
+	 * call it and if it returns non-zero, assume that it has
+	 * handled the trap and modified the trap frame so that this
+	 * function can return normally.
+	 */
+	if ((type == T_PROTFLT || type == T_PAGEFLT) &&
+	    dtrace_trap_func != NULL) {
+		if ((*dtrace_trap_func)(frame, type)) {
+			return;
+		}
+	}
+#endif
+
 	switch (type) {
 
 	case T_ASTFLT:

sys/arch/i386/i386/vector.S

@@ -1,4 +1,4 @@
-/*	$NetBSD: vector.S,v 1.51 2010/01/17 22:21:18 dsl Exp $	*/
+/*	$NetBSD: vector.S,v 1.52 2010/02/21 02:11:40 darran Exp $	*/
 
 /*
  * Copyright 2002 (c) Wasabi Systems, Inc.
@@ -65,7 +65,7 @@
  */
 
 #include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: vector.S,v 1.51 2010/01/17 22:21:18 dsl Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vector.S,v 1.52 2010/02/21 02:11:40 darran Exp $");
 
 #include "opt_ddb.h"
 #include "opt_multiprocessor.h"
@@ -129,6 +129,23 @@ __KERNEL_RCSID(0, "$NetBSD: vector.S,v 1.51 2010/01/17 22:21:18 dsl Exp $");
 	shl	$24,%eax				;\
 	orl	%edx,%eax
 
+#ifdef KDTRACE_HOOKS
+	.bss
+	.globl	dtrace_invop_jump_addr
+	.align	4
+	.type	dtrace_invop_jump_addr, @object
+        .size	dtrace_invop_jump_addr, 4
+dtrace_invop_jump_addr:
+	.zero	4
+	.globl	dtrace_invop_calltrap_addr
+	.align	4
+	.type	dtrace_invop_calltrap_addr, @object
+        .size	dtrace_invop_calltrap_addr, 4
+dtrace_invop_calltrap_addr:
+	.zero	8
+	.text
+#endif
+
 #ifndef XEN
 #if NLAPIC > 0
 #ifdef MULTIPROCESSOR
@@ -885,8 +902,47 @@ IDTVEC(trap04)
 	ZTRAP(T_OFLOW)
 IDTVEC(trap05)
 	ZTRAP(T_BOUND)
+/*
+ * Privileged instruction fault.
+ */
+#ifdef KDTRACE_HOOKS
+	SUPERALIGN_TEXT
+IDTVEC(trap06)
+	/* Check if there is no DTrace hook registered. */
+	cmpl	$0,dtrace_invop_jump_addr
+	je	norm_ill
+
+	/* Check if this is a user fault. */
+	/* XXX this was 0x0020 in FreeBSD */
+	cmpl	$GSEL(GCODE_SEL, SEL_KPL), 4(%esp)   /* Check code segment. */
+
+	/* If so, just handle it as a normal trap. */
+	jne	norm_ill
+              
+	/*
+	 * This is a kernel instruction fault that might have been caused
+	 * by a DTrace provider.
+	 */
+	pushal				/* Push all registers onto the stack. */
+
+	/*
+	 * Set our jump address for the jump back in the event that
+	 * the exception wasn't caused by DTrace at all.
+	 */
+	movl	$norm_ill, dtrace_invop_calltrap_addr
+
+	/* Jump to the code hooked in by DTrace. */
+	jmpl	*dtrace_invop_jump_addr
+
+	/*
+	 * Process the instruction fault in the normal way.
+	 */
+norm_ill:
+	ZTRAP(T_PRIVINFLT)
+#else
 IDTVEC(trap06)
 	ZTRAP(T_PRIVINFLT)
+#endif
 IDTVEC(trap07)
 #if NNPX > 0
 	pushl	$0			# dummy error code

sys/kern/kern_lwp.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: kern_lwp.c,v 1.137 2009/12/17 01:25:10 rmind Exp $	*/
+/*	$NetBSD: kern_lwp.c,v 1.138 2010/02/21 02:11:40 darran Exp $	*/
 
 /*-
  * Copyright (c) 2001, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -209,7 +209,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.137 2009/12/17 01:25:10 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.138 2010/02/21 02:11:40 darran Exp $");
 
 #include "opt_ddb.h"
 #include "opt_lockdebug.h"
@@ -236,6 +236,10 @@ __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.137 2009/12/17 01:25:10 rmind Exp $")
 #include <sys/atomic.h>
 #include <sys/filedesc.h>
 
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+#endif
+
 #include <uvm/uvm_extern.h>
 #include <uvm/uvm_object.h>
 
@@ -620,6 +624,9 @@ lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, int flags,
 	l2->l_cpu = l1->l_cpu;
 	kpreempt_enable();
 
+#ifdef KDTRACE_HOOKS
+	kdtrace_thread_ctor(NULL, l2);
+#endif
 	lwp_initspecific(l2);
 	sched_lwp_fork(l1, l2);
 	lwp_update_creds(l2);
@@ -956,6 +963,9 @@ lwp_free(struct lwp *l, bool recycle, bool last)
 
 	KASSERT(SLIST_EMPTY(&l->l_pi_lenders));
 	KASSERT(l->l_inheritedprio == -1);
+#ifdef KDTRACE_HOOKS
+	kdtrace_thread_dtor(NULL, l);
+#endif
 	if (!recycle)
 		pool_cache_put(lwp_cache, l);
 }

sys/kern/kern_proc.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: kern_proc.c,v 1.159 2009/12/17 01:25:10 rmind Exp $	*/
+/*	$NetBSD: kern_proc.c,v 1.160 2010/02/21 02:11:40 darran Exp $	*/
 
 /*-
  * Copyright (c) 1999, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_proc.c,v 1.159 2009/12/17 01:25:10 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_proc.c,v 1.160 2010/02/21 02:11:40 darran Exp $");
 
 #include "opt_kstack.h"
 #include "opt_maxuprc.h"
@@ -94,6 +94,10 @@ __KERNEL_RCSID(0, "$NetBSD: kern_proc.c,v 1.159 2009/12/17 01:25:10 rmind Exp $"
 #include <sys/atomic.h>
 #include <sys/kmem.h>
 
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+#endif
+
 #include <uvm/uvm.h>
 #include <uvm/uvm_extern.h>
 
@@ -442,6 +446,10 @@ proc0_init(void)
 	mutex_init(&p->p_sigacts->sa_mutex, MUTEX_DEFAULT, IPL_SCHED);
 	siginit(p);
 
+#ifdef KDTRACE_HOOKS
+	kdtrace_proc_ctor(NULL, p);
+#endif
+
 	proc_initspecific(p);
 	lwp_initspecific(l);
 
@@ -710,6 +718,10 @@ proc_alloc(void)
 	pt->pt_proc = p;
 	pid_alloc_cnt++;
 
+#ifdef KDTRACE_HOOKS
+	kdtrace_proc_ctor(NULL, p);
+#endif
+
 	mutex_exit(proc_lock);
 
 	return p;
@@ -753,6 +765,9 @@ void
 proc_free_mem(struct proc *p)
 {
 
+#ifdef KDTRACE_HOOKS
+	kdtrace_proc_dtor(NULL, p);
+#endif
 	pool_cache_put(proc_cache, p);
 }
 

sys/kern/kern_synch.c

@@ -1,4 +1,4 @@
-/*	$NetBSD: kern_synch.c,v 1.275 2010/02/18 20:58:23 skrll Exp $	*/
+/*	$NetBSD: kern_synch.c,v 1.276 2010/02/21 02:11:40 darran Exp $	*/
 
 /*-
  * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009
@@ -69,7 +69,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.275 2010/02/18 20:58:23 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.276 2010/02/21 02:11:40 darran Exp $");
 
 #include "opt_kstack.h"
 #include "opt_perfctrs.h"
@@ -102,6 +102,12 @@ __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.275 2010/02/18 20:58:23 skrll Exp $
 
 #include <dev/lockstat.h>
 
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+int                             dtrace_vtime_active;
+dtrace_vtime_switch_func_t      dtrace_vtime_switch_func;
+#endif
+
 static void	sched_unsleep(struct lwp *, bool);
 static void	sched_changepri(struct lwp *, pri_t);
 static void	sched_lendpri(struct lwp *, pri_t);
@@ -762,6 +768,17 @@ mi_switch(lwp_t *l)
 				SPINLOCK_BACKOFF(count);
 		}
 
+#ifdef KDTRACE_HOOKS
+		/*
+		 * If DTrace has set the active vtime enum to anything
+		 * other than INACTIVE (0), then it should have set the
+		 * function to call.
+		 */
+		if (dtrace_vtime_active) {
+			(*dtrace_vtime_switch_func)(newl);
+		}
+#endif
+
 		/* Switch to the new LWP.. */
 		prevlwp = cpu_switchto(l, newl, returning);
 		ci = curcpu();
@@ -903,6 +920,17 @@ lwp_exit_switchaway(lwp_t *l)
 			SPINLOCK_BACKOFF(count);
 	}
 
+#ifdef KDTRACE_HOOKS
+	    /*
+	     * If DTrace has set the active vtime enum to anything
+	     * other than INACTIVE (0), then it should have set the
+	     * function to call.
+	     */
+	    if (dtrace_vtime_active) {
+		    (*dtrace_vtime_switch_func)(newl);
+	    }
+#endif
+
 	/* Switch to the new LWP.. */
 	(void)cpu_switchto(NULL, newl, false);
 

sys/sys/dtrace_bsd.h

@@ -0,0 +1,170 @@
+/*	$NetBSD: dtrace_bsd.h,v 1.1 2010/02/21 02:11:39 darran Exp $	*/
+
+/*-
+ * Copyright (c) 2007-2008 John Birrell (jb@freebsd.org)
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: src/sys/sys/dtrace_bsd.h,v 1.3.2.1 2009/08/03 08:13:06 kensmith Exp $
+ *
+ * This file contains BSD shims for Sun's DTrace code.
+ */
+
+#ifndef _SYS_DTRACE_BSD_H
+#define	_SYS_DTRACE_BSD_H
+
+/* Forward definitions: */
+struct mbuf;
+struct trapframe;
+struct lwp;
+struct vattr;
+struct vnode;
+struct ucred;
+
+/*
+ * Cyclic clock function type definition used to hook the cyclic
+ * subsystem into the appropriate timer interrupt.
+ */
+typedef	void (*cyclic_clock_func_t)(struct trapframe *);
+
+/*
+ * These external variables are actually machine-dependent, so
+ * they might not actually exist.
+ *
+ * Defining them here avoids a proliferation of header files.
+ */
+extern cyclic_clock_func_t     lapic_cyclic_clock_func[];
+
+/*
+ * The dtrace module handles traps that occur during a DTrace probe.
+ * This type definition is used in the trap handler to provide a
+ * hook for the dtrace module to register it's handler with.
+ */
+typedef int (*dtrace_trap_func_t)(struct trapframe *, u_int);
+
+int	dtrace_trap(struct trapframe *, u_int);
+
+extern dtrace_trap_func_t	dtrace_trap_func;
+
+/* Used by the machine dependent trap() code. */
+typedef	int (*dtrace_invop_func_t)(uintptr_t, uintptr_t *, uintptr_t);
+typedef void (*dtrace_doubletrap_func_t)(void);
+
+/* Global variables in trap.c */
+extern	dtrace_invop_func_t	dtrace_invop_func;
+extern	dtrace_doubletrap_func_t	dtrace_doubletrap_func;
+
+/* Virtual time hook function type. */
+typedef	void (*dtrace_vtime_switch_func_t)(struct lwp *);
+
+extern int			dtrace_vtime_active;
+extern dtrace_vtime_switch_func_t	dtrace_vtime_switch_func;
+
+/* The fasttrap module hooks into the fork, exit and exit. */
+typedef void (*dtrace_fork_func_t)(struct proc *, struct proc *);
+typedef void (*dtrace_execexit_func_t)(struct proc *);
+
+/* Global variable in kern_fork.c */
+extern dtrace_fork_func_t	dtrace_fasttrap_fork;
+
+/* Global variable in kern_exec.c */
+extern dtrace_execexit_func_t	dtrace_fasttrap_exec;
+
+/* Global variable in kern_exit.c */
+extern dtrace_execexit_func_t	dtrace_fasttrap_exit;
+
+/* The dtmalloc provider hooks into malloc. */
+typedef	void (*dtrace_malloc_probe_func_t)(u_int32_t, uintptr_t arg0,
+    uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4);
+
+extern dtrace_malloc_probe_func_t   dtrace_malloc_probe;
+
+/* dtnfsclient NFSv3 access cache provider hooks. */
+typedef void (*dtrace_nfsclient_accesscache_flush_probe_func_t)(uint32_t,
+    struct vnode *);
+extern dtrace_nfsclient_accesscache_flush_probe_func_t
+    dtrace_nfsclient_accesscache_flush_done_probe;
+
+typedef void (*dtrace_nfsclient_accesscache_get_probe_func_t)(uint32_t,
+    struct vnode *, uid_t, uint32_t);
+extern dtrace_nfsclient_accesscache_get_probe_func_t
+    dtrace_nfsclient_accesscache_get_hit_probe,
+    dtrace_nfsclient_accesscache_get_miss_probe;
+
+typedef void (*dtrace_nfsclient_accesscache_load_probe_func_t)(uint32_t,
+    struct vnode *, uid_t, uint32_t, int);
+extern dtrace_nfsclient_accesscache_load_probe_func_t
+    dtrace_nfsclient_accesscache_load_done_probe;
+
+/* dtnfsclient NFSv[23] attribute cache provider hooks. */
+typedef void (*dtrace_nfsclient_attrcache_flush_probe_func_t)(uint32_t,
+    struct vnode *);
+extern dtrace_nfsclient_attrcache_flush_probe_func_t
+    dtrace_nfsclient_attrcache_flush_done_probe;
+
+typedef void (*dtrace_nfsclient_attrcache_get_hit_probe_func_t)(uint32_t,
+    struct vnode *, struct vattr *);
+extern dtrace_nfsclient_attrcache_get_hit_probe_func_t
+    dtrace_nfsclient_attrcache_get_hit_probe;
+
+typedef void (*dtrace_nfsclient_attrcache_get_miss_probe_func_t)(uint32_t,
+    struct vnode *);
+extern dtrace_nfsclient_attrcache_get_miss_probe_func_t
+    dtrace_nfsclient_attrcache_get_miss_probe;
+
+typedef void (*dtrace_nfsclient_attrcache_load_probe_func_t)(uint32_t,
+    struct vnode *, struct vattr *, int);
+extern dtrace_nfsclient_attrcache_load_probe_func_t
+    dtrace_nfsclient_attrcache_load_done_probe;
+
+/* dtnfsclient NFSv[23] RPC provider hooks. */
+typedef void (*dtrace_nfsclient_nfs23_start_probe_func_t)(uint32_t,
+    struct vnode *, struct mbuf *, struct ucred *, int);
+extern dtrace_nfsclient_nfs23_start_probe_func_t
+    dtrace_nfsclient_nfs23_start_probe;
+
+typedef void (*dtrace_nfsclient_nfs23_done_probe_func_t)(uint32_t,
+    struct vnode *, struct mbuf *, struct ucred *, int, int);
+extern dtrace_nfsclient_nfs23_done_probe_func_t
+    dtrace_nfsclient_nfs23_done_probe;
+
+/*
+ * Functions which allow the dtrace module to check that the kernel 
+ * hooks have been compiled with sufficient space for it's private
+ * structures.
+ */
+size_t	kdtrace_proc_size(void);
+void kdtrace_proc_ctor(void *, struct proc *);
+void kdtrace_proc_dtor(void *, struct proc *);
+size_t	kdtrace_thread_size(void);
+void kdtrace_thread_ctor(void *, struct lwp *);
+void kdtrace_thread_dtor(void *, struct lwp *);
+
+/*
+ * OpenSolaris compatible time functions returning nanoseconds.
+ * On OpenSolaris these return hrtime_t which we define as uint64_t.
+ */
+uint64_t	dtrace_gethrtime(void);
+uint64_t	dtrace_gethrestime(void);
+
+#endif /* _SYS_DTRACE_BSD_H */

sys/sys/lwp.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: lwp.h,v 1.127 2009/11/29 16:23:49 rmind Exp $	*/
+/*	$NetBSD: lwp.h,v 1.128 2010/02/21 02:11:39 darran Exp $	*/
 
 /*-
  * Copyright (c) 2001, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -179,6 +179,8 @@ struct lwp {
 	/* These are only used by 'options SYSCALL_TIMES' */
 	uint32_t        l_syscall_time; /* !: time epoch for current syscall */
 	uint64_t        *l_syscall_counter; /* !: counter for current process */
+
+	struct kdtrace_thread *l_dtrace; /* ?: DTrace-specific data. */
 };
 
 /*

sys/sys/proc.h

@@ -1,4 +1,4 @@
-/*	$NetBSD: proc.h,v 1.294 2009/12/10 14:13:54 matt Exp $	*/
+/*	$NetBSD: proc.h,v 1.295 2010/02/21 02:11:39 darran Exp $	*/
 
 /*-
  * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -326,6 +326,7 @@ struct proc {
 	u_short		p_acflag;	/* p: Acc. flags; see struct lwp also */
 	struct mdproc	p_md;		/* p: Any machine-dependent fields */
 	vaddr_t		p_stackbase;	/* :: ASLR randomized stack base */
+	struct kdtrace_proc *p_dtrace;	/* :: DTrace-specific data. */
 };
 
 #define	p_rlimit	p_limit->pl_rlimit