Correctly handle cases when a thread single-steps into the kernel as it can

happen on syscalls or "int" instructions. The debug exception handler sets
the thread debug flags B_THREAD_DEBUG_STOP and
B_THREAD_DEBUG_NOTIFY_SINGLE_STEP (new) and lets the thread continue. Before
leaving the kernel the thread is stopped and a single-step notification is
sent. Fixes #3487.


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@35620 a95241bf-73f2-0310-859d-f6bbb57e9c96

headers/private/kernel/user_debugger.h

@@ -159,8 +159,9 @@ enum {
 	B_THREAD_DEBUG_STOP					= 0x0004,
 	B_THREAD_DEBUG_STOPPED				= 0x0008,
 	B_THREAD_DEBUG_SINGLE_STEP			= 0x0010,
+	B_THREAD_DEBUG_NOTIFY_SINGLE_STEP	= 0x0020,
 
-	B_THREAD_DEBUG_NUB_THREAD		= 0x0020,	// marks the nub thread
+	B_THREAD_DEBUG_NUB_THREAD			= 0x0040,	// marks the nub thread
 
 	B_THREAD_DEBUG_KERNEL_FLAG_MASK		= 0xffff,
 

src/system/kernel/arch/x86/arch_user_debugger.cpp

@@ -886,7 +886,42 @@ x86_handle_debug_exception(struct iframe *frame)
 			// it, but we don't want it when continuing otherwise.
 			frame->flags &= ~(1 << X86_EFLAGS_TF);
 
+			// Determine whether the exception occurred at a syscall/trap
+			// kernel entry or whether this is genuine kernel single-stepping.
+			bool inKernel = true;
+			struct thread* thread = thread_get_current_thread();
+			if (thread->team != team_get_kernel_team()
+				&& i386_get_user_iframe() == NULL) {
+				// TODO: This is not yet fully correct, since a newly created
+				// thread that doesn't have entered userland yet also has this
+				// property.
+				inKernel = false;
+			}
+
+			if (inKernel) {
 				panic("kernel single step");
+			} else {
+				// The thread is a userland thread and it just entered the
+				// kernel when the single-step exception occurred. This happens
+				// e.g. when sysenter is called with single-stepping enabled.
+				// We need to ignore the exception now and send a single-step
+				// notification later, when the thread wants to return from the
+				// kernel.
+				InterruptsSpinLocker threadLocker(gThreadSpinlock);
+
+				// Check whether the team is still being debugged and set
+				// the B_THREAD_DEBUG_NOTIFY_SINGLE_STEP and
+				// B_THREAD_DEBUG_STOP flags, so that the thread will be
+				// stopped when it is going to leave the kernel and notify the
+				// debugger about the single-step event.
+				int32 teamDebugFlags
+					= atomic_get(&thread->team->debug_info.flags);
+				if (teamDebugFlags & B_TEAM_DEBUG_DEBUGGER_INSTALLED) {
+					atomic_or(&thread->debug_info.flags,
+						B_THREAD_DEBUG_NOTIFY_SINGLE_STEP
+							| B_THREAD_DEBUG_STOP);
+				}
+			}
 		}
 	} else if (dr6 & (1 << X86_DR6_BT)) {
 		// task switch

src/system/kernel/debug/user_debugger.cpp

@@ -954,11 +954,25 @@ user_debug_handle_signal(int signal, struct sigaction *handler, bool deadly)
 void
 user_debug_stop_thread()
 {
-	// prepare the message
+	// check whether this is actually an emulated single-step notification
-	debug_thread_debugged message;
+	InterruptsSpinLocker threadsLocker(gThreadSpinlock);
+	struct thread* thread = thread_get_current_thread();
+	bool singleStepped = false;
+	if ((atomic_and(&thread->debug_info.flags,
+				~B_THREAD_DEBUG_NOTIFY_SINGLE_STEP)
+			& B_THREAD_DEBUG_NOTIFY_SINGLE_STEP) != 0) {
+		singleStepped = true;
+	}
 
-	thread_hit_serious_debug_event(B_DEBUGGER_MESSAGE_THREAD_DEBUGGED, &message,
+	threadsLocker.Unlock();
-		sizeof(message));
+
+	if (singleStepped) {
+		user_debug_single_stepped();
+	} else {
+		debug_thread_debugged message;
+		thread_hit_serious_debug_event(B_DEBUGGER_MESSAGE_THREAD_DEBUGGED,
+			&message, sizeof(message));
+	}
 }