Implement RIP traces for winhv/kvm backends (#206)

0vercl0k · May 26, 2024 · 0737f9b · 0737f9b
1 parent f1c3a91
commit 0737f9b
Show file tree

Hide file tree

Showing 18 changed files with 781 additions and 252 deletions.
diff --git a/README.md b/README.md
@@ -99,7 +99,7 @@ wtf.exe master --name hevd --max_len=1028 --runs=0 --inputs=outputs --outputs=mi
 
 ### Generating execution traces
 
-The main mechanism available to instrospect in an execution backend is to generate an execution trace. *bochscpu* is the only backend that has the ability to generate a complete execution traces so it is best to use for debugging purposes. The other backends only generate execution traces used to measure code-coverage (address of the first instruction in a basic block).
+The main mechanism available to instrospect in an execution backend is to generate an execution trace. *bochscpu* is the fastest backend to do that, because exiting VMX mode is very expensive on the other backends.
 
 This is how you would generate an execution trace for the `crash-0xfffff764b91c0000-0x0-0xffffbf84fb10e780-0x2-0x0` test-case:
 
@@ -280,15 +280,15 @@ In this section I briefly mention various differences between the execution back
 - ✔ Code-coverage via software breakpoints,
 - ❌ Demand-paging so start-up is slow (as it needs to load the full crash-dump in memory),
 - ✔ Timeout is implemented with a timer,
-- ✔ Only code-coverage traces are supported,
+- ✅ Full execution traces are supported but are slow (exiting VMX is costly),
 - ✔ Deterministic if handling source of non determinism manually (for example, patching `nt!ExGenRamdom` that uses `rdrand`),
 - ✔ Speed seems to be ok for long executions (lots of bottleneck in whv though; ~10x slower than WHV when I was fuzzing IDA).
 
 ### KVM
 - ✔ Code-coverage via software breakpoints,
 - ✅ Demand-paging is supported via UFDD,
 - ✔ Timeout is implemented with a timer. ✅ If the hardware supports PMU virtualization, it is used to generate a [PMI](https://forum.osdev.org/viewtopic.php?f=1&t=27040) after X retired instructions (`MSR_IA32_FIXED_CTR0`),
-- ✔ Only code-coverage traces are supported,
+- ✅ Full execution traces are supported but are slow (exiting VMX is costly),
 - ✔ Deterministic if handling source of non determinism manually (for example, patching `nt!ExGenRamdom` that uses `rdrand`),
 - ✅ Fastest for long executions (~500m - 1.5 billion instructions; ~100x faster than *bochscpu*, ~10x faster than *whv* when I was fuzzing IDA).
 

diff --git a/src/wtf/backend.cc b/src/wtf/backend.cc
@@ -13,6 +13,8 @@ bool Backend_t::SetTraceFile(const fs::path &, const TraceType_t) {
   return true;
 }
 
+[[nodiscard]] bool Backend_t::EnableSingleStep(CpuState_t &) { return true; }
+
 bool Backend_t::PhysWrite(const Gpa_t Gpa, const uint8_t *Buffer,
                           const uint64_t BufferSize, const bool Dirty) {
   uint8_t *Dst = PhysTranslate(Gpa);
@@ -262,75 +264,92 @@ bool Backend_t::SetCrashBreakpoint(const char *Symbol) {
   return SetBreakpoint(Symbol, CrashBreakpointHandler);
 }
 
-uint64_t Backend_t::Rsp() { return GetReg(Registers_t::Rsp); }
+void Backend_t::TrapFlag(const bool Arm) {
+  uint64_t NewValue = Rflags();
+  if (Arm) {
+    NewValue |= RFLAGS_TRAP_FLAG_FLAG;
+  } else {
+    NewValue &= ~RFLAGS_TRAP_FLAG_FLAG;
+  }
+
+  Rflags(NewValue);
+}
+
+uint64_t Backend_t::Rflags() const { return GetReg(Registers_t::Rflags); }
+void Backend_t::Rflags(const uint64_t Value) {
+  SetReg(Registers_t::Rflags, Value);
+}
+void Backend_t::Rflags(const Gva_t Value) { Rflags(Value.U64()); }
+
+uint64_t Backend_t::Rsp() const { return GetReg(Registers_t::Rsp); }
 void Backend_t::Rsp(const uint64_t Value) { SetReg(Registers_t::Rsp, Value); }
 void Backend_t::Rsp(const Gva_t Value) { Rsp(Value.U64()); }
 
-uint64_t Backend_t::Rbp() { return GetReg(Registers_t::Rbp); }
+uint64_t Backend_t::Rbp() const { return GetReg(Registers_t::Rbp); }
 void Backend_t::Rbp(const uint64_t Value) { SetReg(Registers_t::Rbp, Value); }
 void Backend_t::Rbp(const Gva_t Value) { Rbp(Value.U64()); }
 
-uint64_t Backend_t::Rip() { return GetReg(Registers_t::Rip); }
+uint64_t Backend_t::Rip() const { return GetReg(Registers_t::Rip); }
 void Backend_t::Rip(const uint64_t Value) { SetReg(Registers_t::Rip, Value); }
 void Backend_t::Rip(const Gva_t Value) { Rip(Value.U64()); }
 
-uint64_t Backend_t::Rax() { return GetReg(Registers_t::Rax); }
+uint64_t Backend_t::Rax() const { return GetReg(Registers_t::Rax); }
 void Backend_t::Rax(const uint64_t Value) { SetReg(Registers_t::Rax, Value); }
 void Backend_t::Rax(const Gva_t Value) { Rax(Value.U64()); }
 
-uint64_t Backend_t::Rbx() { return GetReg(Registers_t::Rbx); }
+uint64_t Backend_t::Rbx() const { return GetReg(Registers_t::Rbx); }
 void Backend_t::Rbx(const uint64_t Value) { SetReg(Registers_t::Rbx, Value); }
 void Backend_t::Rbx(const Gva_t Value) { Rbx(Value.U64()); }
 
-uint64_t Backend_t::Rcx() { return GetReg(Registers_t::Rcx); }
+uint64_t Backend_t::Rcx() const { return GetReg(Registers_t::Rcx); }
 void Backend_t::Rcx(const uint64_t Value) { SetReg(Registers_t::Rcx, Value); }
 void Backend_t::Rcx(const Gva_t Value) { Rcx(Value.U64()); }
 
-uint64_t Backend_t::Rdx() { return GetReg(Registers_t::Rdx); }
+uint64_t Backend_t::Rdx() const { return GetReg(Registers_t::Rdx); }
 void Backend_t::Rdx(const uint64_t Value) { SetReg(Registers_t::Rdx, Value); }
 void Backend_t::Rdx(const Gva_t Value) { Rdx(Value.U64()); }
 
-uint64_t Backend_t::Rsi() { return GetReg(Registers_t::Rsi); }
+uint64_t Backend_t::Rsi() const { return GetReg(Registers_t::Rsi); }
 void Backend_t::Rsi(const uint64_t Value) { SetReg(Registers_t::Rsi, Value); }
 void Backend_t::Rsi(const Gva_t Value) { Rsi(Value.U64()); }
 
-uint64_t Backend_t::Rdi() { return GetReg(Registers_t::Rdi); }
+uint64_t Backend_t::Rdi() const { return GetReg(Registers_t::Rdi); }
 void Backend_t::Rdi(const uint64_t Value) { SetReg(Registers_t::Rdi, Value); }
 void Backend_t::Rdi(const Gva_t Value) { Rdi(Value.U64()); }
 
-uint64_t Backend_t::R8() { return GetReg(Registers_t::R8); }
+uint64_t Backend_t::R8() const { return GetReg(Registers_t::R8); }
 void Backend_t::R8(const uint64_t Value) { SetReg(Registers_t::R8, Value); }
 void Backend_t::R8(const Gva_t Value) { R8(Value.U64()); }
 
-uint64_t Backend_t::R9() { return GetReg(Registers_t::R9); }
+uint64_t Backend_t::R9() const { return GetReg(Registers_t::R9); }
 void Backend_t::R9(const uint64_t Value) { SetReg(Registers_t::R9, Value); }
 void Backend_t::R9(const Gva_t Value) { R9(Value.U64()); }
 
-uint64_t Backend_t::R10() { return GetReg(Registers_t::R10); }
+uint64_t Backend_t::R10() const { return GetReg(Registers_t::R10); }
 void Backend_t::R10(const uint64_t Value) { SetReg(Registers_t::R10, Value); }
 void Backend_t::R10(const Gva_t Value) { R10(Value.U64()); }
 
-uint64_t Backend_t::R11() { return GetReg(Registers_t::R11); }
+uint64_t Backend_t::R11() const { return GetReg(Registers_t::R11); }
 void Backend_t::R11(const uint64_t Value) { SetReg(Registers_t::R11, Value); }
 void Backend_t::R11(const Gva_t Value) { R11(Value.U64()); }
 
-uint64_t Backend_t::R12() { return GetReg(Registers_t::R12); }
+uint64_t Backend_t::R12() const { return GetReg(Registers_t::R12); }
 void Backend_t::R12(const uint64_t Value) { SetReg(Registers_t::R12, Value); }
 void Backend_t::R12(const Gva_t Value) { R12(Value.U64()); }
 
-uint64_t Backend_t::R13() { return GetReg(Registers_t::R13); }
+uint64_t Backend_t::R13() const { return GetReg(Registers_t::R13); }
 void Backend_t::R13(const uint64_t Value) { SetReg(Registers_t::R13, Value); }
 void Backend_t::R13(const Gva_t Value) { R13(Value.U64()); }
 
-uint64_t Backend_t::R14() { return GetReg(Registers_t::R14); }
+uint64_t Backend_t::R14() const { return GetReg(Registers_t::R14); }
 void Backend_t::R14(const uint64_t Value) { SetReg(Registers_t::R14, Value); }
 void Backend_t::R14(const Gva_t Value) { R14(Value.U64()); }
 
-uint64_t Backend_t::R15() { return GetReg(Registers_t::R15); }
+uint64_t Backend_t::R15() const { return GetReg(Registers_t::R15); }
 void Backend_t::R15(const uint64_t Value) { SetReg(Registers_t::R15, Value); }
 void Backend_t::R15(const Gva_t Value) { R15(Value.U64()); }
 
-uint64_t Backend_t::Cr2() { return GetReg(Registers_t::Cr2); }
+uint64_t Backend_t::Cr2() const { return GetReg(Registers_t::Cr2); }
 void Backend_t::Cr2(const uint64_t Value) { SetReg(Registers_t::Cr2, Value); }
 void Backend_t::Cr2(const Gva_t Value) { Cr2(Value.U64()); }
 

diff --git a/src/wtf/backend.h b/src/wtf/backend.h
@@ -202,7 +202,7 @@ class Backend_t {
   // Registers.
   //
 
-  virtual uint64_t GetReg(const Registers_t Reg) = 0;
+  virtual uint64_t GetReg(const Registers_t Reg) const = 0;
   virtual uint64_t SetReg(const Registers_t Reg, const uint64_t Value) = 0;
 
   //
@@ -224,6 +224,12 @@ class Backend_t {
   virtual bool SetTraceFile(const fs::path &TestcaseTracePath,
                             const TraceType_t TraceType);
 
+  //
+  // Enable single-stepping.
+  //
+
+  virtual bool EnableSingleStep(CpuState_t &CpuState);
+
   //
   // Breakpoints.
   //
@@ -513,75 +519,81 @@ class Backend_t {
   // Shortcuts to grab / set some registers.
   //
 
-  [[nodiscard]] uint64_t Rsp();
+  virtual void TrapFlag(const bool Arm);
+
+  [[nodiscard]] uint64_t Rflags() const;
+  void Rflags(const uint64_t Value);
+  void Rflags(const Gva_t Value);
+
+  [[nodiscard]] uint64_t Rsp() const;
   void Rsp(const uint64_t Value);
   void Rsp(const Gva_t Value);
 
-  [[nodiscard]] uint64_t Rbp();
+  [[nodiscard]] uint64_t Rbp() const;
   void Rbp(const uint64_t Value);
   void Rbp(const Gva_t Value);
 
-  [[nodiscard]] uint64_t Rip();
+  [[nodiscard]] uint64_t Rip() const;
   void Rip(const uint64_t Value);
   void Rip(const Gva_t Value);
 
-  [[nodiscard]] uint64_t Rax();
+  [[nodiscard]] uint64_t Rax() const;
   void Rax(const uint64_t Value);
   void Rax(const Gva_t Value);
 
-  [[nodiscard]] uint64_t Rbx();
+  [[nodiscard]] uint64_t Rbx() const;
   void Rbx(const uint64_t Value);
   void Rbx(const Gva_t Value);
 
-  [[nodiscard]] uint64_t Rcx();
+  [[nodiscard]] uint64_t Rcx() const;
   void Rcx(const uint64_t Value);
   void Rcx(const Gva_t Value);
 
-  [[nodiscard]] uint64_t Rdx();
+  [[nodiscard]] uint64_t Rdx() const;
   void Rdx(const uint64_t Value);
   void Rdx(const Gva_t Value);
 
-  [[nodiscard]] uint64_t Rsi();
+  [[nodiscard]] uint64_t Rsi() const;
   void Rsi(const uint64_t Value);
   void Rsi(const Gva_t Value);
 
-  [[nodiscard]] uint64_t Rdi();
+  [[nodiscard]] uint64_t Rdi() const;
   void Rdi(const uint64_t Value);
   void Rdi(const Gva_t Value);
 
-  [[nodiscard]] uint64_t R8();
+  [[nodiscard]] uint64_t R8() const;
   void R8(const uint64_t Value);
   void R8(const Gva_t Value);
 
-  [[nodiscard]] uint64_t R9();
+  [[nodiscard]] uint64_t R9() const;
   void R9(const uint64_t Value);
   void R9(const Gva_t Value);
 
-  [[nodiscard]] uint64_t R10();
+  [[nodiscard]] uint64_t R10() const;
   void R10(const uint64_t Value);
   void R10(const Gva_t Value);
 
-  [[nodiscard]] uint64_t R11();
+  [[nodiscard]] uint64_t R11() const;
   void R11(const uint64_t Value);
   void R11(const Gva_t Value);
 
-  [[nodiscard]] uint64_t R12();
+  [[nodiscard]] uint64_t R12() const;
   void R12(const uint64_t Value);
   void R12(const Gva_t Value);
 
-  [[nodiscard]] uint64_t R13();
+  [[nodiscard]] uint64_t R13() const;
   void R13(const uint64_t Value);
   void R13(const Gva_t Value);
 
-  [[nodiscard]] uint64_t R14();
+  [[nodiscard]] uint64_t R14() const;
   void R14(const uint64_t Value);
   void R14(const Gva_t Value);
 
-  [[nodiscard]] uint64_t R15();
+  [[nodiscard]] uint64_t R15() const;
   void R15(const uint64_t Value);
   void R15(const Gva_t Value);
 
-  [[nodiscard]] uint64_t Cr2();
+  [[nodiscard]] uint64_t Cr2() const;
   void Cr2(const uint64_t Value);
   void Cr2(const Gva_t Value);
 

diff --git a/src/wtf/bochscpu_backend.cc b/src/wtf/bochscpu_backend.cc
@@ -1121,7 +1121,7 @@ void BochscpuBackend_t::LoadState(const CpuState_t &State) {
   bochscpu_cpu_set_state(Cpu_, &Bochs);
 }
 
-uint64_t BochscpuBackend_t::GetReg(const Registers_t Reg) {
+uint64_t BochscpuBackend_t::GetReg(const Registers_t Reg) const {
   using BochscpuGetReg_t = uint64_t (*)(bochscpu_cpu_t);
   static const std::unordered_map<Registers_t, BochscpuGetReg_t>
       RegisterMappingGetters = {{Registers_t::Rax, bochscpu_cpu_rax},
@@ -1320,4 +1320,4 @@ void BochscpuBackend_t::DumpTenetDelta(const bool Force) {
   if (NeedNewLine) {
     fmt::print(TraceFile_, "\n");
   }
-}
+}
diff --git a/src/wtf/bochscpu_backend.h b/src/wtf/bochscpu_backend.h
@@ -220,7 +220,7 @@ class BochscpuBackend_t : public Backend_t {
   // Registers.
   //
 
-  uint64_t GetReg(const Registers_t Reg) override;
+  uint64_t GetReg(const Registers_t Reg) const override;
   uint64_t SetReg(const Registers_t Reg, const uint64_t Value) override;
 
   //

diff --git a/src/wtf/crash_detection_umode.cc b/src/wtf/crash_detection_umode.cc
@@ -19,20 +19,6 @@ void CrashDetectionPrint(const char *Format, const Args_t &...args) {
 
 bool SetupUsermodeCrashDetectionHooks() {
 
-  //
-  // This is to catch the PMI interrupt if performance counters are used to
-  // bound execution.
-  //
-
-  if (!g_Backend->SetBreakpoint("hal!HalpPerfInterrupt",
-                                [](Backend_t *Backend) {
-                                  CrashDetectionPrint("Perf interrupt\n");
-                                  Backend->Stop(Timedout_t());
-                                })) {
-    fmt::print(
-        "Failed to set breakpoint on HalpPerfInterrupt, but ignoring..\n");
-  }
-
   //
   // Avoid the fuzzer to spin out of control if we mess-up real bad.
   //
@@ -130,16 +116,22 @@ bool SetupUsermodeCrashDetectionHooks() {
 
   //
   // As we can't set-up the exception bitmap so that we receive a vmexit on
-  // failfast exceptions, we instead set a breakpoint to the function handling
-  // the interruption.
+  // failfast exceptions, we instead set a breakpoint to one of the handling
+  // function.
+  //
+  // We do not set a breakpoint directly on the IDT handler because the
+  // single-step feature itself will set a breakpoint on that location.
   //
   // kd> !idt 0x29
   // Dumping IDT: fffff8053f15b000
   // 29:	fffff8053b9ccb80 nt!KiRaiseSecurityCheckFailure
   //
+  // To not interfere with that we simply pick another location. the
+  // interruption.
+  //
 
   if (!g_Backend->SetBreakpoint(
-          "nt!KiRaiseSecurityCheckFailure", [](Backend_t *Backend) {
+          "nt!KiFastFailDispatch", [](Backend_t *Backend) {
             const Gva_t Rsp = Gva_t(Backend->Rsp());
             const Gva_t ExceptionAddress = Backend->VirtReadGva(Rsp);
             CrashDetectionPrint(

diff --git a/src/wtf/fuzzer_hevd.cc b/src/wtf/fuzzer_hevd.cc
@@ -59,6 +59,7 @@ bool InsertTestcase(const uint8_t *Buffer, const size_t BufferSize) {
 }
 
 bool Init(const Options_t &Opts, const CpuState_t &) {
+
   //
   // Stop the test-case once we return back from the call [DeviceIoControl]
   //
@@ -93,7 +94,10 @@ bool Init(const Options_t &Opts, const CpuState_t &) {
   // kd> ub fffff805`3b8287c4 l1
   // nt!ExGenRandom+0xe0:
   // fffff805`3b8287c0 480fc7f2        rdrand  rdx
-  const Gva_t ExGenRandom = Gva_t(g_Dbg->GetSymbol("nt!ExGenRandom") + 0xe0 + 4);
+  //
+
+  const Gva_t ExGenRandom =
+      Gva_t(g_Dbg->GetSymbol("nt!ExGenRandom") + 0xe0 + 4);
   if (g_Backend->VirtRead4(ExGenRandom - Gva_t(4)) != 0xf2c70f48) {
     fmt::print("It seems that nt!ExGenRandom's code has changed, update the "
                "offset!\n");

diff --git a/src/wtf/fuzzer_ioctl.cc b/src/wtf/fuzzer_ioctl.cc
@@ -192,6 +192,8 @@ bool Init(const Options_t &Opts, const CpuState_t &) {
   // kd> ub fffff805`3b8287c4 l1
   // nt!ExGenRandom+0xe0:
   // fffff805`3b8287c0 480fc7f2        rdrand  rdx
+  //
+
   const Gva_t ExGenRandom = Gva_t(g_Dbg->GetSymbol("nt!ExGenRandom") + 0xe0 + 4);
   if (g_Backend->VirtRead4(ExGenRandom - Gva_t(4)) != 0xf2c70f48) {
     fmt::print("It seems that nt!ExGenRandom's code has changed, update the "