Nyx api support (#97)

* add nyx support

* target independent helper call.

accel/tcg/tcg-runtime.c

@@ -45,10 +45,10 @@ void HELPER(libafl_qemu_handle_breakpoint)(CPUArchState *env, uint64_t pc)
     libafl_exit_request_breakpoint(cpu, (target_ulong) pc);
 }
 
-void HELPER(libafl_qemu_handle_sync_backdoor)(CPUArchState *env, uint64_t pc)
+void HELPER(libafl_qemu_handle_custom_insn)(CPUArchState *env, uint64_t pc, uint32_t kind)
 {
     CPUState* cpu = env_cpu(env);
-    libafl_exit_request_sync_backdoor(cpu, (target_ulong) pc);
+    libafl_exit_request_custom_insn(cpu, (target_ulong) pc, (enum libafl_custom_insn_kind) kind);
 }
 
 //// --- End LibAFL code ---

accel/tcg/tcg-runtime.h

@@ -329,7 +329,7 @@ DEF_HELPER_FLAGS_5(gvec_bitsel, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_2(libafl_qemu_handle_breakpoint, TCG_CALL_NO_RWG,
                     void, env, i64)
 
-DEF_HELPER_FLAGS_2(libafl_qemu_handle_sync_backdoor, TCG_CALL_NO_RWG,
-                    void, env, i64)
+DEF_HELPER_FLAGS_3(libafl_qemu_handle_custom_insn, TCG_CALL_NO_RWG,
+                    void, env, i64, i32)
 
 //// --- End LibAFL code ---

accel/tcg/translator.c

@@ -200,7 +200,7 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns,
                         db->pc_next += 4;
 
                         TCGv_i64 tmp0 = tcg_constant_i64((uint64_t)db->pc_next);
-                        gen_helper_libafl_qemu_handle_sync_backdoor(tcg_env, tmp0);
+                        gen_helper_libafl_qemu_handle_custom_insn(tcg_env, tmp0, tcg_constant_i32(LIBAFL_CUSTOM_INSN_LIBAFL));
                         tcg_temp_free_i64(tmp0);
                     }
                 }

include/libafl/exit.h

@@ -10,18 +10,19 @@ struct libafl_breakpoint {
     struct libafl_breakpoint* next;
 };
 
-int libafl_qemu_set_breakpoint(target_ulong pc);
-int libafl_qemu_remove_breakpoint(target_ulong pc);
-void libafl_qemu_trigger_breakpoint(CPUState* cpu);
-void libafl_qemu_breakpoint_run(vaddr pc_next);
-
 enum libafl_exit_reason_kind {
     INTERNAL = 0,
     BREAKPOINT = 1,
-    SYNC_EXIT = 2,
+    CUSTOM_INSN = 2,
     TIMEOUT = 3,
 };
 
+enum libafl_custom_insn_kind {
+    LIBAFL_CUSTOM_INSN_UNDEFINED = 0,
+    LIBAFL_CUSTOM_INSN_LIBAFL = 1,
+    LIBAFL_CUSTOM_INSN_NYX = 2,
+};
+
 // QEMU exited on its own for some reason.
 struct libafl_exit_reason_internal {
     ShutdownCause cause;
@@ -34,23 +35,32 @@ struct libafl_exit_reason_breakpoint {
 };
 
 // A synchronous exit has been triggered.
-struct libafl_exit_reason_sync_exit {};
+struct libafl_exit_reason_custom_insn {
+    enum libafl_custom_insn_kind kind;
+};
 
 // A timeout occured and we were asked to exit on timeout
-struct libafl_exit_reason_timeout {};
+struct libafl_exit_reason_timeout {
+};
 
 struct libafl_exit_reason {
     enum libafl_exit_reason_kind kind;
     CPUState* cpu; // CPU that triggered an exit.
     vaddr next_pc; // The PC that should be stored in the CPU when re-entering.
     union {
-        struct libafl_exit_reason_internal internal;        // kind == INTERNAL
-        struct libafl_exit_reason_breakpoint breakpoint;    // kind == BREAKPOINT
-        struct libafl_exit_reason_sync_exit sync_exit;      // kind == SYNC_EXIT
-        struct libafl_exit_reason_timeout timeout;          // kind == TIMEOUT
+        struct libafl_exit_reason_internal internal;     // kind == INTERNAL
+        struct libafl_exit_reason_breakpoint breakpoint; // kind == BREAKPOINT
+        struct libafl_exit_reason_custom_insn
+            custom_insn;                           // kind == CUSTOM_INSN
+        struct libafl_exit_reason_timeout timeout; // kind == TIMEOUT
     } data;
 };
 
+int libafl_qemu_set_breakpoint(target_ulong pc);
+int libafl_qemu_remove_breakpoint(target_ulong pc);
+void libafl_qemu_trigger_breakpoint(CPUState* cpu);
+void libafl_qemu_breakpoint_run(vaddr pc_next);
+
 // Only makes sense to call if an exit was expected
 // Will return NULL if there was no exit expected.
 CPUState* libafl_last_exit_cpu(void);
@@ -62,7 +72,8 @@ void libafl_sync_exit_cpu(void);
 void libafl_exit_request_internal(CPUState* cpu, uint64_t pc,
                                   ShutdownCause cause, int signal);
 void libafl_exit_request_breakpoint(CPUState* cpu, target_ulong pc);
-void libafl_exit_request_sync_backdoor(CPUState* cpu, target_ulong pc);
+void libafl_exit_request_custom_insn(CPUState* cpu, target_ulong pc,
+                                     enum libafl_custom_insn_kind kind);
 
 #ifndef CONFIG_USER_ONLY
 void libafl_exit_request_timeout(void);

include/libafl/tcg.h

@@ -6,5 +6,5 @@
 #include "tcg/tcg.h"
 #include "tcg/helper-info.h"
 
-void tcg_gen_callN(void *func, TCGHelperInfo *info,
-                          TCGTemp *ret, TCGTemp **args);
+void tcg_gen_callN(void* func, TCGHelperInfo* info, TCGTemp* ret,
+                   TCGTemp** args);

libafl/exit.c

@@ -110,9 +110,10 @@ void libafl_exit_request_internal(CPUState* cpu, uint64_t pc,
     expected_exit = true;
 }
 
-void libafl_exit_request_sync_backdoor(CPUState* cpu, target_ulong pc)
+void libafl_exit_request_custom_insn(CPUState* cpu, target_ulong pc,
+                                     enum libafl_custom_insn_kind kind)
 {
-    last_exit_reason.kind = SYNC_EXIT;
+    last_exit_reason.kind = CUSTOM_INSN;
 
     prepare_qemu_exit(cpu, pc);
 }

libafl/hooks/tcg/block.c

@@ -81,7 +81,8 @@ void libafl_qemu_hook_block_run(target_ulong pc)
             TCGv_i64 tmp0 = tcg_constant_i64(hook->data);
             TCGv_i64 tmp1 = tcg_constant_i64(cur_id);
             TCGTemp* tmp2[2] = {tcgv_i64_temp(tmp0), tcgv_i64_temp(tmp1)};
-            tcg_gen_callN(hook->helper_info.func, &hook->helper_info, NULL, tmp2);
+            tcg_gen_callN(hook->helper_info.func, &hook->helper_info, NULL,
+                          tmp2);
             tcg_temp_free_i64(tmp0);
             tcg_temp_free_i64(tmp1);
         }

libafl/hooks/tcg/edge.c

@@ -9,8 +9,7 @@ static TCGHelperInfo libafl_exec_edge_hook_info = {
     .name = "libafl_exec_edge_hook",
     .flags = dh_callflag(void),
     .typemask =
-        dh_typemask(void, 0) | dh_typemask(i64, 1) | dh_typemask(i64, 2)
-};
+        dh_typemask(void, 0) | dh_typemask(i64, 1) | dh_typemask(i64, 2)};
 
 GEN_REMOVE_HOOK(edge)
 
@@ -86,7 +85,8 @@ void libafl_qemu_hook_edge_run(void)
             TCGv_i64 tmp0 = tcg_constant_i64(hook->data);
             TCGv_i64 tmp1 = tcg_constant_i64(hook->cur_id);
             TCGTemp* tmp2[2] = {tcgv_i64_temp(tmp0), tcgv_i64_temp(tmp1)};
-            tcg_gen_callN(hook->helper_info.func, &hook->helper_info, NULL, tmp2);
+            tcg_gen_callN(hook->helper_info.func, &hook->helper_info, NULL,
+                          tmp2);
             tcg_temp_free_i64(tmp0);
             tcg_temp_free_i64(tmp1);
         }

libafl/hooks/tcg/read_write.c

@@ -216,7 +216,8 @@ static void libafl_gen_rw(TCGTemp* addr, MemOpIdx oi,
 #else
                                     tcgv_i64_temp(tmp2)};
 #endif
-                tcg_gen_callN(hook->helper_infoN.func, &hook->helper_infoN, NULL, tmp3);
+                tcg_gen_callN(hook->helper_infoN.func, &hook->helper_infoN,
+                              NULL, tmp3);
                 tcg_temp_free_i64(tmp0);
                 tcg_temp_free_i64(tmp1);
 #if TARGET_LONG_BITS == 32

target/i386/tcg/translate.c

@@ -3287,6 +3287,23 @@ static void disas_insn_old(DisasContext *s, CPUState *cpu, int b)
             gen_op_st_v(s, CODE64(s) + MO_32, s->T0, s->A0);
             break;
 
+//// --- Begin LibAFL code ---
+        case 0xc1: /* vmcall */
+            // move pc to T0
+            tcg_gen_movi_tl(s->T0, s->pc - s->cs_base);
+
+            // gen jump to next insn
+            gen_op_jmp_v(s, s->T0);
+            gen_bnd_jmp(s);
+            s->base.is_jmp = DISAS_JUMP;
+
+            // gen helper to signal to get out
+            TCGv_i64 new_pc = tcg_temp_new_i64();
+            tcg_gen_extu_tl_i64(new_pc, s->T0);
+            gen_helper_libafl_qemu_handle_custom_insn(tcg_env, new_pc, tcg_constant_i32(LIBAFL_CUSTOM_INSN_NYX));
+            break;
+//// --- End LibAFL code ---
+
         case 0xc8: /* monitor */
             if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) || CPL(s) != 0) {
                 goto illegal_op;