Added an invalid instruction hook (#1132)

* first draft for an invalid instruction hook

* Fixed documentation on return value of invalid insn hook

bindings/python/unicorn/unicorn.py

@@ -146,6 +146,7 @@ _uc.uc_hook_add = _uc.uc_hook_add
 _uc.uc_hook_add.restype = ucerr
 
 UC_HOOK_CODE_CB = ctypes.CFUNCTYPE(None, uc_engine, ctypes.c_uint64, ctypes.c_size_t, ctypes.c_void_p)
+UC_HOOK_INSN_INVALID_CB = ctypes.CFUNCTYPE(ctypes.c_bool, uc_engine, ctypes.c_void_p)
 UC_HOOK_MEM_INVALID_CB = ctypes.CFUNCTYPE(
     ctypes.c_bool, uc_engine, ctypes.c_int,
     ctypes.c_uint64, ctypes.c_int, ctypes.c_int64, ctypes.c_void_p
@@ -492,6 +493,11 @@ class Uc(object):
         (cb, data) = self._callbacks[user_data]
         cb(self, intno, data)
 
+    def _hook_insn_invalid_cb(self, handle, user_data):
+        # call user's callback with self object
+        (cb, data) = self._callbacks[user_data]
+        return cb(self, data)
+
     def _hook_insn_in_cb(self, handle, port, size, user_data):
         # call user's callback with self object
         (cb, data) = self._callbacks[user_data]
@@ -536,6 +542,13 @@ class Uc(object):
                 ctypes.cast(self._callback_count, ctypes.c_void_p),
                 ctypes.c_uint64(begin), ctypes.c_uint64(end)
             )
+        elif htype == uc.UC_HOOK_INSN_INVALID:
+            cb = ctypes.cast(UC_HOOK_INSN_INVALID_CB(self._hook_insn_invalid_cb), UC_HOOK_INSN_INVALID_CB)
+            status = _uc.uc_hook_add(
+                self._uch, ctypes.byref(_h2), htype, cb,
+                ctypes.cast(self._callback_count, ctypes.c_void_p),
+                ctypes.c_uint64(begin), ctypes.c_uint64(end)
+            )
         else:
             if htype in (uc.UC_HOOK_BLOCK, uc.UC_HOOK_CODE):
                 # set callback with wrapper, so it can be called

bindings/python/unicorn/unicorn_const.py

@@ -85,6 +85,7 @@ UC_HOOK_MEM_READ = 1024
 UC_HOOK_MEM_WRITE = 2048
 UC_HOOK_MEM_FETCH = 4096
 UC_HOOK_MEM_READ_AFTER = 8192
+UC_HOOK_INSN_INVALID = 16384
 UC_HOOK_MEM_UNMAPPED = 112
 UC_HOOK_MEM_PROT = 896
 UC_HOOK_MEM_READ_INVALID = 144

include/uc_priv.h

@@ -107,6 +107,7 @@ enum uc_hook_idx {
     UC_HOOK_MEM_WRITE_IDX,
     UC_HOOK_MEM_FETCH_IDX,
     UC_HOOK_MEM_READ_AFTER_IDX,
+    UC_HOOK_INSN_INVALID_IDX,
 
     UC_HOOK_MAX,
 };

include/unicorn/unicorn.h

@@ -173,6 +173,15 @@ typedef void (*uc_cb_hookcode_t)(uc_engine *uc, uint64_t address, uint32_t size,
 */
 typedef void (*uc_cb_hookintr_t)(uc_engine *uc, uint32_t intno, void *user_data);
 
+/*
+  Callback function for tracing invalid instructions
+
+  @user_data: user data passed to tracing APIs.
+
+  @return: return true to continue, or false to stop program (due to invalid instruction).
+*/
+typedef bool (*uc_cb_hookinsn_invalid_t)(uc_engine *uc, void *user_data);
+
 /*
   Callback function for tracing IN instruction of X86
 
@@ -236,6 +245,8 @@ typedef enum uc_hook_type {
     // Hook memory read events, but only successful access.
     // The callback will be triggered after successful read.
     UC_HOOK_MEM_READ_AFTER = 1 << 13,
+    // Hook invalid instructions exceptions.
+    UC_HOOK_INSN_INVALID = 1 << 14,
 } uc_hook_type;
 
 // Hook type for all events of unmapped memory access

qemu/cpu-exec.c

@@ -103,13 +103,6 @@ int cpu_exec(struct uc_struct *uc, CPUArchState *env)   // qq
             /* if an exception is pending, we execute it here */
             if (cpu->exception_index >= 0) {
                 //printf(">>> GOT INTERRUPT. exception idx = %x\n", cpu->exception_index);	// qq
-                if (uc->stop_interrupt && uc->stop_interrupt(cpu->exception_index)) {
-                    cpu->halted = 1;
-                    uc->invalid_error = UC_ERR_INSN_INVALID;
-                    ret = EXCP_HLT;
-                    break;
-                }
-
                 if (cpu->exception_index >= EXCP_INTERRUPT) {
                     /* exit request from the cpu execution loop */
                     ret = cpu->exception_index;
@@ -129,17 +122,30 @@ int cpu_exec(struct uc_struct *uc, CPUArchState *env)   // qq
                     ret = cpu->exception_index;
                     break;
 #else
-                    // Unicorn: call registered interrupt callbacks
+                    if (uc->stop_interrupt && uc->stop_interrupt(cpu->exception_index)) {
-                    HOOK_FOREACH_VAR_DECLARE;
+                        // Unicorn: call registered invalid instruction callbacks
-                    HOOK_FOREACH(uc, hook, UC_HOOK_INTR) {
+                        HOOK_FOREACH_VAR_DECLARE;
-                        ((uc_cb_hookintr_t)hook->callback)(uc, cpu->exception_index, hook->user_data);
+                        HOOK_FOREACH(uc, hook, UC_HOOK_INSN_INVALID) {
-                        catched = true;
+                            catched = ((uc_cb_hookinsn_invalid_t)hook->callback)(uc, hook->user_data);
+                            if (catched)
+                                break;
+                        }
+                        if (!catched)
+                            uc->invalid_error = UC_ERR_INSN_INVALID;
+                    } else {
+                        // Unicorn: call registered interrupt callbacks
+                        HOOK_FOREACH_VAR_DECLARE;
+                        HOOK_FOREACH(uc, hook, UC_HOOK_INTR) {
+                            ((uc_cb_hookintr_t)hook->callback)(uc, cpu->exception_index, hook->user_data);
+                            catched = true;
+                        }
+                        if (!catched)
+                            uc->invalid_error = UC_ERR_EXCEPTION;
                     }
 
                     // Unicorn: If un-catched interrupt, stop executions.
                     if (!catched) {
                         cpu->halted = 1;
-                        uc->invalid_error = UC_ERR_EXCEPTION;
                         ret = EXCP_HLT;
                         break;
                     }