From 7562d10f80ed23b6267b8f4cbe22bfb8c714d241 Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Sun, 18 Nov 2018 13:51:10 +0100 Subject: [PATCH 01/13] Moved VMX-specific definitions to vmx.h - Moved vcpu_vmx_data and interruptibility_state_t from {vcpu,vcpu_state}.h to vmx.h. - Removed vcpu_state_t::interruptibility_state_t since it's not used. Signed-off-by: Alexandro Sanchez Bach --- core/include/vcpu.h | 34 ------------------------------ core/include/vmx.h | 50 ++++++++++++++++++++++++++++++++++++++++++++ include/vcpu_state.h | 15 ------------- 3 files changed, 50 insertions(+), 49 deletions(-) diff --git a/core/include/vcpu.h b/core/include/vcpu.h index b3ff6bd1..a0049339 100644 --- a/core/include/vcpu.h +++ b/core/include/vcpu.h @@ -68,40 +68,6 @@ struct cvtlb { struct hax_mmu; struct per_cpu_data; -struct vcpu_vmx_data { - uint32_t pin_ctls_base; - uint32_t pcpu_ctls_base; - uint32_t scpu_ctls_base; - uint32_t entry_ctls_base; - uint32_t exc_bitmap_base; - uint32_t exit_ctls_base; - - uint32_t pin_ctls; - uint32_t pcpu_ctls; - uint32_t scpu_ctls; - uint32_t entry_ctls; - uint32_t exc_bitmap; - uint32_t exit_ctls; - - uint64_t cr0_mask, cr0_shadow; - uint64_t cr4_mask, cr4_shadow; - uint32_t entry_exception_vector; - uint32_t entry_exception_error_code; - - uint32_t exit_exception_error_code; - interruption_info_t exit_intr_info; - interruption_info_t entry_intr_info; - uint32_t exit_idt_vectoring; - uint32_t exit_instr_length; - uint32_t entry_instr_length; - - exit_reason_t exit_reason; - exit_qualification_t exit_qualification; - interruptibility_state_t interruptibility_state; - - uint64_t exit_gpa; -}; - /* Information saved by instruction decoder and used by post-MMIO handler */ struct vcpu_post_mmio { enum { diff --git a/core/include/vmx.h b/core/include/vmx.h index 016287b7..52fe77ad 100644 --- a/core/include/vmx.h +++ b/core/include/vmx.h @@ -102,6 +102,7 @@ enum { VMX_EXIT_XRSTORS = 64 }; +// Intel SDM Vol. 3D: Appendix B: Field Encoding in VMCS enum component_index_t { VMX_PIN_CONTROLS = 0x00004000, VMX_PRIMARY_PROCESSOR_CONTROLS = 0x00004002, @@ -467,6 +468,21 @@ union instruction_info_t { typedef union instruction_info_t instruction_info_t; +// Intel SDM Vol. 3C: Table 24-3. Format of Interruptibility State +union interruptibility_state_t { + uint32_t raw; + struct { + uint32_t sti_blocking : 1; + uint32_t movss_blocking : 1; + uint32_t smi_blocking : 1; + uint32_t nmi_blocking : 1; + uint32_t reserved : 28; + }; + uint64_t pad; +} PACKED; + +typedef union interruptibility_state_t interruptibility_state_t; + // 64-bit OK union interruption_info_t { uint32_t raw; @@ -636,6 +652,40 @@ struct invept_desc { struct vcpu_state_t; struct vcpu_t; +struct vcpu_vmx_data { + uint32_t pin_ctls_base; + uint32_t pcpu_ctls_base; + uint32_t scpu_ctls_base; + uint32_t entry_ctls_base; + uint32_t exc_bitmap_base; + uint32_t exit_ctls_base; + + uint32_t pin_ctls; + uint32_t pcpu_ctls; + uint32_t scpu_ctls; + uint32_t entry_ctls; + uint32_t exc_bitmap; + uint32_t exit_ctls; + + uint64_t cr0_mask, cr0_shadow; + uint64_t cr4_mask, cr4_shadow; + uint32_t entry_exception_vector; + uint32_t entry_exception_error_code; + + uint32_t exit_exception_error_code; + interruption_info_t exit_intr_info; + interruption_info_t entry_intr_info; + uint32_t exit_idt_vectoring; + uint32_t exit_instr_length; + uint32_t entry_instr_length; + + exit_reason_t exit_reason; + exit_qualification_t exit_qualification; + interruptibility_state_t interruptibility_state; + + uint64_t exit_gpa; +}; + vmx_result_t ASMCALL asm_invept(uint type, struct invept_desc *desc); vmx_result_t ASMCALL asm_vmclear(const hax_paddr_t *addr_in); vmx_result_t ASMCALL asm_vmptrld(const hax_paddr_t *addr_in); diff --git a/include/vcpu_state.h b/include/vcpu_state.h index e4507066..d42c862e 100644 --- a/include/vcpu_state.h +++ b/include/vcpu_state.h @@ -31,20 +31,6 @@ #ifndef HAX_VCPU_STATE_H_ #define HAX_VCPU_STATE_H_ -union interruptibility_state_t { - uint32_t raw; - struct { - uint32_t sti_blocking : 1; - uint32_t movss_blocking : 1; - uint32_t smi_blocking : 1; - uint32_t nmi_blocking : 1; - uint32_t reserved : 28; - }; - uint64_t pad; -} PACKED; - -typedef union interruptibility_state_t interruptibility_state_t; - // Segment descriptor struct segment_desc_t { uint16_t selector; @@ -187,7 +173,6 @@ struct vcpu_state_t { uint32_t _activity_state; uint32_t pad; - interruptibility_state_t _interruptibility_state; } PACKED; void dump(void); From 4247f6cde25c5b37ecedbcc872a6a4f80025e12c Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Sun, 18 Nov 2018 13:56:09 +0100 Subject: [PATCH 02/13] Added VMCS-cache macros Signed-off-by: Alexandro Sanchez Bach --- core/include/vmx.h | 266 +++++++++++++++++++++++++++++++++++++++++++++ core/vcpu.c | 1 + core/vmx.c | 17 +++ 3 files changed, 284 insertions(+) diff --git a/core/include/vmx.h b/core/include/vmx.h index 52fe77ad..0edc33d1 100644 --- a/core/include/vmx.h +++ b/core/include/vmx.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2009 Intel Corporation + * Copyright (c) 2018 Kryptos Logic * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -257,6 +258,208 @@ enum component_index_t { typedef enum component_index_t component_index_t; +// VMCS component types +#define COMP_TYPE_W_16 uint16_t +#define COMP_TYPE_W_32 uint32_t +#define COMP_TYPE_W_64 uint64_t +#ifdef HAX_ARCH_X86_64 +#define COMP_TYPE_W_UL uint64_t +#else +#define COMP_TYPE_W_UL uint32_t +#endif + +/** + * VMCS Cache + * ========== + * Stores all VMCS components declared through the macro + * COMP(cache_r, cache_w, width, value), with the arguments: + * - cache_r: Boolean value to toggle read-cache: + * - 0 Reading cache disabled + * - 1 Reading cache enabled + * - cache_w: Boolean value to toggle write-cache: + * - 0 Writing cache disabled + * - 1 Writing cache enabled + * - width: Component width. + * - W_16 (0): 16-bit unsigned integer. + * - W_64 (1): 64-bit unsigned integer. + * - W_32 (2): 32-bit unsigned integer. + * - W_UL (3): Natural-width unsigned integer. + * - name: Component name, defined as encoding value. + * + * For optimal memory packing, components should be declared + * in the following order: W_64, W_UL, W_32, W_16. + */ +#define VMCS_COMPS \ + /* 64-bit components */ \ + COMP(1, 1, W_64, VMX_IO_BITMAP_A) \ + COMP(0, 0, W_64, VMX_IO_BITMAP_B) \ + COMP(0, 0, W_64, VMX_MSR_BITMAP) \ + COMP(0, 0, W_64, VMX_EXIT_MSR_STORE_ADDRESS) \ + COMP(0, 0, W_64, VMX_EXIT_MSR_LOAD_ADDRESS) \ + COMP(0, 0, W_64, VMX_ENTRY_MSR_LOAD_ADDRESS) \ + COMP(0, 0, W_64, VMX_TSC_OFFSET) \ + COMP(0, 0, W_64, VMX_VAPIC_PAGE) \ + COMP(0, 0, W_64, VMX_APIC_ACCESS_PAGE) \ + COMP(0, 0, W_64, VMX_EPTP) \ + COMP(0, 0, W_64, VM_EXIT_INFO_GUEST_PHYSICAL_ADDRESS) \ + COMP(0, 0, W_64, HOST_PAT) \ + COMP(0, 0, W_64, HOST_EFER) \ + COMP(0, 0, W_64, HOST_PERF_GLOBAL_CTRL) \ + COMP(0, 0, W_64, GUEST_VMCS_LINK_PTR) \ + COMP(0, 0, W_64, GUEST_DEBUGCTL) \ + COMP(0, 0, W_64, GUEST_PAT) \ + COMP(0, 0, W_64, GUEST_EFER) \ + COMP(0, 0, W_64, GUEST_PERF_GLOBAL_CTRL) \ + COMP(0, 0, W_64, GUEST_PDPTE0) \ + COMP(0, 0, W_64, GUEST_PDPTE1) \ + COMP(0, 0, W_64, GUEST_PDPTE2) \ + COMP(0, 0, W_64, GUEST_PDPTE3) \ + /* Natural-width components */ \ + COMP(0, 0, W_UL, VMX_CR0_MASK) \ + COMP(0, 0, W_UL, VMX_CR4_MASK) \ + COMP(0, 0, W_UL, VMX_CR0_READ_SHADOW) \ + COMP(0, 0, W_UL, VMX_CR4_READ_SHADOW) \ + COMP(0, 0, W_UL, VMX_CR3_TARGET_VAL_BASE) \ + COMP(0, 0, W_UL, VM_EXIT_INFO_QUALIFICATION) \ + COMP(0, 0, W_UL, VM_EXIT_INFO_IO_ECX) \ + COMP(0, 0, W_UL, VM_EXIT_INFO_IO_ESI) \ + COMP(0, 0, W_UL, VM_EXIT_INFO_IO_EDI) \ + COMP(0, 0, W_UL, VM_EXIT_INFO_IO_EIP) \ + COMP(0, 0, W_UL, VM_EXIT_INFO_GUEST_LINEAR_ADDRESS) \ + COMP(0, 0, W_UL, HOST_RIP) \ + COMP(0, 0, W_UL, HOST_RSP) \ + COMP(0, 0, W_UL, HOST_CR0) \ + COMP(0, 0, W_UL, HOST_CR3) \ + COMP(0, 0, W_UL, HOST_CR4) \ + COMP(0, 0, W_UL, HOST_FS_BASE) \ + COMP(0, 0, W_UL, HOST_GS_BASE) \ + COMP(0, 0, W_UL, HOST_TR_BASE) \ + COMP(0, 0, W_UL, HOST_GDTR_BASE) \ + COMP(0, 0, W_UL, HOST_IDTR_BASE) \ + COMP(0, 0, W_UL, HOST_SYSENTER_ESP) \ + COMP(0, 0, W_UL, HOST_SYSENTER_EIP) \ + COMP(0, 0, W_UL, GUEST_RIP) \ + COMP(0, 0, W_UL, GUEST_RFLAGS) \ + COMP(0, 0, W_UL, GUEST_RSP) \ + COMP(0, 0, W_UL, GUEST_CR0) \ + COMP(0, 0, W_UL, GUEST_CR3) \ + COMP(0, 0, W_UL, GUEST_CR4) \ + COMP(0, 0, W_UL, GUEST_ES_BASE) \ + COMP(0, 0, W_UL, GUEST_CS_BASE) \ + COMP(0, 0, W_UL, GUEST_SS_BASE) \ + COMP(0, 0, W_UL, GUEST_DS_BASE) \ + COMP(0, 0, W_UL, GUEST_FS_BASE) \ + COMP(0, 0, W_UL, GUEST_GS_BASE) \ + COMP(0, 0, W_UL, GUEST_LDTR_BASE) \ + COMP(0, 0, W_UL, GUEST_TR_BASE) \ + COMP(0, 0, W_UL, GUEST_GDTR_BASE) \ + COMP(0, 0, W_UL, GUEST_IDTR_BASE) \ + COMP(0, 0, W_UL, GUEST_DR7) \ + COMP(0, 0, W_UL, GUEST_PENDING_DBE) \ + COMP(0, 0, W_UL, GUEST_SYSENTER_ESP) \ + COMP(0, 0, W_UL, GUEST_SYSENTER_EIP) \ + /* 32-bit components */ \ + COMP(0, 0, W_32, VMX_PIN_CONTROLS) \ + COMP(0, 0, W_32, VMX_PRIMARY_PROCESSOR_CONTROLS) \ + COMP(0, 0, W_32, VMX_SECONDARY_PROCESSOR_CONTROLS) \ + COMP(0, 0, W_32, VMX_EXCEPTION_BITMAP) \ + COMP(0, 0, W_32, VMX_PAGE_FAULT_ERROR_CODE_MASK) \ + COMP(0, 0, W_32, VMX_PAGE_FAULT_ERROR_CODE_MATCH) \ + COMP(0, 0, W_32, VMX_EXIT_CONTROLS) \ + COMP(0, 0, W_32, VMX_EXIT_MSR_STORE_COUNT) \ + COMP(0, 0, W_32, VMX_EXIT_MSR_LOAD_COUNT) \ + COMP(0, 0, W_32, VMX_ENTRY_CONTROLS) \ + COMP(0, 0, W_32, VMX_ENTRY_MSR_LOAD_COUNT) \ + COMP(0, 0, W_32, VMX_ENTRY_INTERRUPT_INFO) \ + COMP(0, 0, W_32, VMX_ENTRY_EXCEPTION_ERROR_CODE) \ + COMP(0, 0, W_32, VMX_ENTRY_INSTRUCTION_LENGTH) \ + COMP(0, 0, W_32, VMX_TPR_THRESHOLD) \ + COMP(0, 0, W_32, VMX_CR3_TARGET_COUNT) \ + COMP(0, 0, W_32, VMX_PREEMPTION_TIMER) \ + COMP(0, 0, W_32, VMX_INSTRUCTION_ERROR_CODE) \ + COMP(0, 0, W_32, VM_EXIT_INFO_REASON) \ + COMP(0, 0, W_32, VM_EXIT_INFO_INTERRUPT_INFO) \ + COMP(0, 0, W_32, VM_EXIT_INFO_EXCEPTION_ERROR_CODE) \ + COMP(0, 0, W_32, VM_EXIT_INFO_IDT_VECTORING) \ + COMP(0, 0, W_32, VM_EXIT_INFO_IDT_VECTORING_ERROR_CODE) \ + COMP(0, 0, W_32, VM_EXIT_INFO_INSTRUCTION_LENGTH) \ + COMP(0, 0, W_32, VM_EXIT_INFO_INSTRUCTION_INFO) \ + COMP(0, 0, W_32, HOST_SYSENTER_CS) \ + COMP(0, 0, W_32, GUEST_ES_AR) \ + COMP(0, 0, W_32, GUEST_CS_AR) \ + COMP(0, 0, W_32, GUEST_SS_AR) \ + COMP(0, 0, W_32, GUEST_DS_AR) \ + COMP(0, 0, W_32, GUEST_FS_AR) \ + COMP(0, 0, W_32, GUEST_GS_AR) \ + COMP(0, 0, W_32, GUEST_LDTR_AR) \ + COMP(0, 0, W_32, GUEST_TR_AR) \ + COMP(0, 0, W_32, GUEST_ES_LIMIT) \ + COMP(0, 0, W_32, GUEST_CS_LIMIT) \ + COMP(0, 0, W_32, GUEST_SS_LIMIT) \ + COMP(0, 0, W_32, GUEST_DS_LIMIT) \ + COMP(0, 0, W_32, GUEST_FS_LIMIT) \ + COMP(0, 0, W_32, GUEST_GS_LIMIT) \ + COMP(0, 0, W_32, GUEST_LDTR_LIMIT) \ + COMP(0, 0, W_32, GUEST_TR_LIMIT) \ + COMP(0, 0, W_32, GUEST_GDTR_LIMIT) \ + COMP(0, 0, W_32, GUEST_IDTR_LIMIT) \ + COMP(0, 0, W_32, GUEST_SYSENTER_CS) \ + COMP(0, 0, W_32, GUEST_SMBASE) \ + COMP(0, 0, W_32, GUEST_INTERRUPTIBILITY) \ + COMP(0, 0, W_32, GUEST_ACTIVITY_STATE) \ + /* 16-bit components */ \ + COMP(0, 0, W_16, VMX_VPID) \ + COMP(0, 0, W_16, HOST_CS_SELECTOR) \ + COMP(0, 0, W_16, HOST_DS_SELECTOR) \ + COMP(0, 0, W_16, HOST_ES_SELECTOR) \ + COMP(0, 0, W_16, HOST_FS_SELECTOR) \ + COMP(0, 0, W_16, HOST_GS_SELECTOR) \ + COMP(0, 0, W_16, HOST_SS_SELECTOR) \ + COMP(0, 0, W_16, HOST_TR_SELECTOR) \ + COMP(0, 0, W_16, GUEST_ES_SELECTOR) \ + COMP(0, 0, W_16, GUEST_CS_SELECTOR) \ + COMP(0, 0, W_16, GUEST_SS_SELECTOR) \ + COMP(0, 0, W_16, GUEST_DS_SELECTOR) \ + COMP(0, 0, W_16, GUEST_FS_SELECTOR) \ + COMP(0, 0, W_16, GUEST_GS_SELECTOR) \ + COMP(0, 0, W_16, GUEST_LDTR_SELECTOR) \ + COMP(0, 0, W_16, GUEST_TR_SELECTOR) + + // Macros for declaring cache and R/W-flags declarations +#define COMP_CACHE_R_0(name) +#define COMP_CACHE_R_1(name) \ + bool name##_cache : 1; +#define COMP_CACHE_W_0(name) +#define COMP_CACHE_W_1(name) \ + bool name##_dirty : 1; + +#define COMP_CACHE_R(cache_r, cache_w, width, name) \ + COMP_CACHE_R_##cache_r(name) +#define COMP_CACHE_W(cache_r, cache_w, width, name) \ + COMP_CACHE_W_##cache_w(name) +#define COMP_DECLARE(cache_r, cache_w, width, name) \ + COMP_TYPE_##width name##_value; + +// Structures +struct vmx_vmcs_t { +#define COMP COMP_DECLARE + VMCS_COMPS +#undef COMP +}; + +struct vmx_vmcs_cache_r_t { +#define COMP COMP_CACHE_R + VMCS_COMPS +#undef COMP +}; + +struct vmx_vmcs_cache_w_t { + bool dirty : 1; +#define COMP COMP_CACHE_W + VMCS_COMPS +#undef COMP +}; + // PIN-BASED CONTROLS #define EXT_INTERRUPT_EXITING 0x00000001 #define NMI_EXITING 0x00000008 @@ -653,6 +856,11 @@ struct vcpu_state_t; struct vcpu_t; struct vcpu_vmx_data { + struct vcpu_t *parent; + struct vmx_vmcs_t vmcs; + struct vmx_vmcs_cache_r_t vmcs_cache_r; + struct vmx_vmcs_cache_w_t vmcs_cache_w; + uint32_t pin_ctls_base; uint32_t pcpu_ctls_base; uint32_t scpu_ctls_base; @@ -706,6 +914,64 @@ void vmx_vmwrite(struct vcpu_t *vcpu, const char *name, #define vmwrite(vcpu, x, y) vmx_vmwrite(vcpu, #x, x, y) +/** + * vmcs_read_ + * Reads the VMCS-component from the specified VCPU (cached if enabled). + */ +#define COMP_R_0(width, name) \ + static inline COMP_TYPE_##width \ + vmcs_read_##name(struct vcpu_vmx_data* vcpu_vmx) { \ + return vmread(vcpu_vmx->parent, name); \ + } +#define COMP_R_1(width, name) \ + static inline COMP_TYPE_##width \ + vmcs_read_##name(struct vcpu_vmx_data* vcpu_vmx) { \ + if (vcpu_vmx->vmcs_cache_r.name##_cache) \ + return vcpu_vmx->vmcs.name##_value; \ + vcpu_vmx->vmcs.name##_value = vmread(vcpu_vmx->parent, name); \ + vcpu_vmx->vmcs_cache_r.name##_cache = 1; \ + return vcpu_vmx->vmcs.name##_value; \ + } + + /** + * vmcs_write_ + * Writes the VMCS-component to the specified VCPU (cached if enabled). + */ +#define COMP_W_0(width, name) \ + static inline void vmcs_write_##name(struct vcpu_vmx_data* vcpu_vmx, \ + COMP_TYPE_##width value) { \ + vmwrite(vcpu_vmx->parent, name, value); \ + } +#define COMP_W_1(width, name) \ + static inline void vmcs_write_##name(struct vcpu_vmx_data* vcpu_vmx, \ + COMP_TYPE_##width value) { \ + vcpu_vmx->vmcs.name##_value = value; \ + vcpu_vmx->vmcs_cache_r.name##_cache = 1; \ + vcpu_vmx->vmcs_cache_w.name##_dirty = 1; \ + } + + // Declarations +#define COMP_R(cache_r, cache_w, width, name) \ + COMP_R_##cache_r(width, name); +#define COMP_W(cache_r, cache_w, width, name) \ + COMP_W_##cache_w(width, name); + +// Helper functions +#define COMP COMP_R +VMCS_COMPS +#undef COMP + +#define COMP COMP_W +VMCS_COMPS +#undef COMP + +#define vmcs_read(vcpu, name) \ + vmcs_read_##name(&vcpu->vmx) +#define vmcs_write(vcpu, name, value) \ + vmcs_write_##name(&vcpu->vmx, value) + +void vmcs_write_pending(struct vcpu_t *vcpu); + #define VMREAD_SEG(vcpu, seg, val) \ ((val).selector = vmread(vcpu, GUEST_##seg##_SELECTOR), \ (val).base = vmread(vcpu, GUEST_##seg##_BASE), \ diff --git a/core/vcpu.c b/core/vcpu.c index 2b205587..b9712754 100644 --- a/core/vcpu.c +++ b/core/vcpu.c @@ -562,6 +562,7 @@ static void vcpu_init(struct vcpu_t *vcpu) vmx(vcpu, cr0_shadow) = 0; vmx(vcpu, cr4_mask) = 0; vmx(vcpu, cr4_shadow) = 0; + vmx(vcpu, parent) = vcpu; vcpu->ref_count = 1; diff --git a/core/vmx.c b/core/vmx.c index ef960983..2b824fae 100644 --- a/core/vmx.c +++ b/core/vmx.c @@ -296,3 +296,20 @@ void get_interruption_info_t(interruption_info_t *info, uint8_t v, uint8_t t) info->reserved = 0; info->valid = 1; } + +#define COMP_PENDING_0(name) +#define COMP_PENDING_1(name) \ + if (vcpu->vmx.vmcs_cache_w.name##_dirty) \ + vmwrite(vcpu, name, vcpu->vmx.vmcs.name##_value); +#define COMP_PENDING(cache_r, cache_w, width, name) \ + COMP_PENDING_##cache_w(name) + +void vmcs_write_pending(struct vcpu_t* vcpu) +{ + if (!vcpu || !vcpu->vmx.vmcs_cache_w.dirty) + return; + +#define COMP COMP_PENDING + VMCS_COMPS +#undef COMP +} From 5ee10eb97b498f2408554d6a1677d25f753fb790 Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Sun, 18 Nov 2018 15:34:55 +0100 Subject: [PATCH 03/13] Optimize code by using new VMCS-cache macros - Replaced vmcs_pending* fields, with the new automatically-generated function that updates VMCS based on vmcs_cache_w flags. Only `vmcs_pending_guest_cr3` has been preserved since it requires extra logic. - Removed unnecessary vmread's at the end of `cpu_vmx_execute` that are not always required. Corresponding members in `vcpu_vmx_data` have been removed and function depending on them now call `vmcs_read`. Signed-off-by: Alexandro Sanchez Bach --- core/cpu.c | 40 ---------------------------------------- core/include/vcpu.h | 4 ---- core/include/vmx.h | 31 +++++++++++++------------------ core/intr_exc.c | 20 ++++++++------------ core/vcpu.c | 28 +++++++++++++++------------- core/vmx.c | 8 +++++++- core/vtlb.c | 2 +- 7 files changed, 44 insertions(+), 89 deletions(-) diff --git a/core/cpu.c b/core/cpu.c index f6c07bd2..6e1b9040 100644 --- a/core/cpu.c +++ b/core/cpu.c @@ -312,36 +312,6 @@ vmx_result_t cpu_vmx_run(struct vcpu_t *vcpu, struct hax_tunnel *htun) return result; } -void vcpu_handle_vmcs_pending(struct vcpu_t *vcpu) -{ - if (!vcpu || !vcpu->vmcs_pending) - return; - if (vcpu->vmcs_pending_entry_error_code) { - vmwrite(vcpu, VMX_ENTRY_EXCEPTION_ERROR_CODE, - vmx(vcpu, entry_exception_error_code)); - vcpu->vmcs_pending_entry_error_code = 0; - } - - if (vcpu->vmcs_pending_entry_instr_length) { - vmwrite(vcpu, VMX_ENTRY_INSTRUCTION_LENGTH, - vmx(vcpu, entry_instr_length)); - vcpu->vmcs_pending_entry_instr_length = 0; - } - - if (vcpu->vmcs_pending_entry_intr_info) { - vmwrite(vcpu, VMX_ENTRY_INTERRUPT_INFO, - vmx(vcpu, entry_intr_info).raw); - vcpu->vmcs_pending_entry_intr_info = 0; - } - - if (vcpu->vmcs_pending_guest_cr3) { - vmwrite(vcpu, GUEST_CR3, vtlb_get_cr3(vcpu)); - vcpu->vmcs_pending_guest_cr3 = 0; - } - vcpu->vmcs_pending = 0; - return; -} - /* Return the value same as ioctl value */ int cpu_vmx_execute(struct vcpu_t *vcpu, struct hax_tunnel *htun) { @@ -416,16 +386,6 @@ int cpu_vmx_execute(struct vcpu_t *vcpu, struct hax_tunnel *htun) vmx(vcpu, exit_qualification).raw = vmread( vcpu, VM_EXIT_INFO_QUALIFICATION); - vmx(vcpu, exit_intr_info).raw = vmread( - vcpu, VM_EXIT_INFO_INTERRUPT_INFO); - vmx(vcpu, exit_exception_error_code) = vmread( - vcpu, VM_EXIT_INFO_EXCEPTION_ERROR_CODE); - vmx(vcpu, exit_idt_vectoring) = vmread( - vcpu, VM_EXIT_INFO_IDT_VECTORING); - vmx(vcpu, exit_instr_length) = vmread( - vcpu, VM_EXIT_INFO_INSTRUCTION_LENGTH); - vmx(vcpu, exit_gpa) = vmread( - vcpu, VM_EXIT_INFO_GUEST_PHYSICAL_ADDRESS); vmx(vcpu, interruptibility_state).raw = vmread( vcpu, GUEST_INTERRUPTIBILITY); diff --git a/core/include/vcpu.h b/core/include/vcpu.h index a0049339..419e366f 100644 --- a/core/include/vcpu.h +++ b/core/include/vcpu.h @@ -154,10 +154,6 @@ struct vcpu_t { #define GS_STALE 0 #define GS_VALID 1 uint64_t cur_state : 1; - uint64_t vmcs_pending : 1; - uint64_t vmcs_pending_entry_error_code : 1; - uint64_t vmcs_pending_entry_instr_length : 1; - uint64_t vmcs_pending_entry_intr_info : 1; uint64_t vmcs_pending_guest_cr3 : 1; uint64_t debug_control_dirty : 1; uint64_t dr_dirty : 1; diff --git a/core/include/vmx.h b/core/include/vmx.h index 0edc33d1..dc9435c7 100644 --- a/core/include/vmx.h +++ b/core/include/vmx.h @@ -291,7 +291,7 @@ typedef enum component_index_t component_index_t; */ #define VMCS_COMPS \ /* 64-bit components */ \ - COMP(1, 1, W_64, VMX_IO_BITMAP_A) \ + COMP(0, 0, W_64, VMX_IO_BITMAP_A) \ COMP(0, 0, W_64, VMX_IO_BITMAP_B) \ COMP(0, 0, W_64, VMX_MSR_BITMAP) \ COMP(0, 0, W_64, VMX_EXIT_MSR_STORE_ADDRESS) \ @@ -301,7 +301,7 @@ typedef enum component_index_t component_index_t; COMP(0, 0, W_64, VMX_VAPIC_PAGE) \ COMP(0, 0, W_64, VMX_APIC_ACCESS_PAGE) \ COMP(0, 0, W_64, VMX_EPTP) \ - COMP(0, 0, W_64, VM_EXIT_INFO_GUEST_PHYSICAL_ADDRESS) \ + COMP(1, 0, W_64, VM_EXIT_INFO_GUEST_PHYSICAL_ADDRESS) \ COMP(0, 0, W_64, HOST_PAT) \ COMP(0, 0, W_64, HOST_EFER) \ COMP(0, 0, W_64, HOST_PERF_GLOBAL_CTRL) \ @@ -370,19 +370,19 @@ typedef enum component_index_t component_index_t; COMP(0, 0, W_32, VMX_EXIT_MSR_LOAD_COUNT) \ COMP(0, 0, W_32, VMX_ENTRY_CONTROLS) \ COMP(0, 0, W_32, VMX_ENTRY_MSR_LOAD_COUNT) \ - COMP(0, 0, W_32, VMX_ENTRY_INTERRUPT_INFO) \ - COMP(0, 0, W_32, VMX_ENTRY_EXCEPTION_ERROR_CODE) \ - COMP(0, 0, W_32, VMX_ENTRY_INSTRUCTION_LENGTH) \ + COMP(0, 1, W_32, VMX_ENTRY_INTERRUPT_INFO) \ + COMP(0, 1, W_32, VMX_ENTRY_EXCEPTION_ERROR_CODE) \ + COMP(0, 1, W_32, VMX_ENTRY_INSTRUCTION_LENGTH) \ COMP(0, 0, W_32, VMX_TPR_THRESHOLD) \ COMP(0, 0, W_32, VMX_CR3_TARGET_COUNT) \ COMP(0, 0, W_32, VMX_PREEMPTION_TIMER) \ COMP(0, 0, W_32, VMX_INSTRUCTION_ERROR_CODE) \ COMP(0, 0, W_32, VM_EXIT_INFO_REASON) \ - COMP(0, 0, W_32, VM_EXIT_INFO_INTERRUPT_INFO) \ - COMP(0, 0, W_32, VM_EXIT_INFO_EXCEPTION_ERROR_CODE) \ - COMP(0, 0, W_32, VM_EXIT_INFO_IDT_VECTORING) \ + COMP(1, 0, W_32, VM_EXIT_INFO_INTERRUPT_INFO) \ + COMP(1, 0, W_32, VM_EXIT_INFO_EXCEPTION_ERROR_CODE) \ + COMP(1, 0, W_32, VM_EXIT_INFO_IDT_VECTORING) \ COMP(0, 0, W_32, VM_EXIT_INFO_IDT_VECTORING_ERROR_CODE) \ - COMP(0, 0, W_32, VM_EXIT_INFO_INSTRUCTION_LENGTH) \ + COMP(1, 0, W_32, VM_EXIT_INFO_INSTRUCTION_LENGTH) \ COMP(0, 0, W_32, VM_EXIT_INFO_INSTRUCTION_INFO) \ COMP(0, 0, W_32, HOST_SYSENTER_CS) \ COMP(0, 0, W_32, GUEST_ES_AR) \ @@ -880,18 +880,12 @@ struct vcpu_vmx_data { uint32_t entry_exception_vector; uint32_t entry_exception_error_code; - uint32_t exit_exception_error_code; - interruption_info_t exit_intr_info; interruption_info_t entry_intr_info; - uint32_t exit_idt_vectoring; - uint32_t exit_instr_length; uint32_t entry_instr_length; exit_reason_t exit_reason; exit_qualification_t exit_qualification; interruptibility_state_t interruptibility_state; - - uint64_t exit_gpa; }; vmx_result_t ASMCALL asm_invept(uint type, struct invept_desc *desc); @@ -946,11 +940,12 @@ void vmx_vmwrite(struct vcpu_t *vcpu, const char *name, static inline void vmcs_write_##name(struct vcpu_vmx_data* vcpu_vmx, \ COMP_TYPE_##width value) { \ vcpu_vmx->vmcs.name##_value = value; \ - vcpu_vmx->vmcs_cache_r.name##_cache = 1; \ + /*vcpu_vmx->vmcs_cache_r.name##_cache = 1;*/ \ vcpu_vmx->vmcs_cache_w.name##_dirty = 1; \ + vcpu_vmx->vmcs_cache_w.dirty = 1; \ } - // Declarations +// Declarations #define COMP_R(cache_r, cache_w, width, name) \ COMP_R_##cache_r(width, name); #define COMP_W(cache_r, cache_w, width, name) \ @@ -970,7 +965,7 @@ VMCS_COMPS #define vmcs_write(vcpu, name, value) \ vmcs_write_##name(&vcpu->vmx, value) -void vmcs_write_pending(struct vcpu_t *vcpu); +void vcpu_handle_vmcs_pending(struct vcpu_t *vcpu); #define VMREAD_SEG(vcpu, seg, val) \ ((val).selector = vmread(vcpu, GUEST_##seg##_SELECTOR), \ diff --git a/core/intr_exc.c b/core/intr_exc.c index a233b8c0..8287c74a 100644 --- a/core/intr_exc.c +++ b/core/intr_exc.c @@ -137,7 +137,7 @@ uint hax_intr_is_blocked(struct vcpu_t *vcpu) void hax_handle_idt_vectoring(struct vcpu_t *vcpu) { uint8_t vector; - uint32_t idt_vec = vmread(vcpu, VM_EXIT_INFO_IDT_VECTORING); + uint32_t idt_vec = vmcs_read(vcpu, VM_EXIT_INFO_IDT_VECTORING); if (idt_vec & 0x80000000) { if (!(idt_vec & 0x700)) { @@ -147,9 +147,9 @@ void hax_handle_idt_vectoring(struct vcpu_t *vcpu) hax_debug("extern interrupt is vectoring....vector:%d\n", vector); } else { hax_debug("VM Exit @ IDT vectoring, type:%d, vector:%d," - " error code:%llx\n", + " error code:%x\n", (idt_vec & 0x700) >> 8, idt_vec & 0xff, - vmread(vcpu, VM_EXIT_INFO_IDT_VECTORING_ERROR_CODE)); + vmcs_read(vcpu, VM_EXIT_INFO_IDT_VECTORING_ERROR_CODE)); } } } @@ -198,8 +198,8 @@ void hax_inject_exception(struct vcpu_t *vcpu, uint8_t vector, uint32_t error_co { uint32_t intr_info = 0; uint8_t first_vec; - uint32_t vect_info = vmx(vcpu, exit_idt_vectoring); - uint32_t exit_instr_length = vmx(vcpu, exit_instr_length); + uint32_t vect_info = vmcs_read(vcpu, VM_EXIT_INFO_IDT_VECTORING); + uint32_t exit_instr_length = vmcs_read(vcpu, VM_EXIT_INFO_INSTRUCTION_LENGTH); if (vcpu->event_injected == 1) hax_debug("Event is injected already!!:\n"); @@ -218,8 +218,7 @@ void hax_inject_exception(struct vcpu_t *vcpu, uint8_t vector, uint32_t error_co if (error_code != NO_ERROR_CODE) { intr_info |= 1 << 11; if (vector == VECTOR_PF) { - vcpu->vmcs_pending_entry_error_code = 1; - vmx(vcpu, entry_exception_error_code) = error_code; + vmcs_write(vcpu, VMX_ENTRY_EXCEPTION_ERROR_CODE, error_code); } else { vmwrite(vcpu, VMX_ENTRY_EXCEPTION_ERROR_CODE, error_code); } @@ -227,11 +226,8 @@ void hax_inject_exception(struct vcpu_t *vcpu, uint8_t vector, uint32_t error_co } if (vector == VECTOR_PF) { - vcpu->vmcs_pending_entry_instr_length = 1; - vmx(vcpu, entry_instr_length) = exit_instr_length; - vcpu->vmcs_pending_entry_intr_info = 1; - vmx(vcpu, entry_intr_info).raw = intr_info; - vcpu->vmcs_pending = 1; + vmcs_write(vcpu, VMX_ENTRY_INSTRUCTION_LENGTH, exit_instr_length); + vmcs_write(vcpu, VMX_ENTRY_INTERRUPT_INFO, intr_info); } else { vmwrite(vcpu, VMX_ENTRY_INSTRUCTION_LENGTH, exit_instr_length); vmwrite(vcpu, VMX_ENTRY_INTERRUPT_INFO, intr_info); diff --git a/core/vcpu.c b/core/vcpu.c index b9712754..dba3e2f7 100644 --- a/core/vcpu.c +++ b/core/vcpu.c @@ -1748,7 +1748,7 @@ static void advance_rip(struct vcpu_t *vcpu) vcpu->interruptibility_dirty = 1; } - state->_rip += vmx(vcpu, exit_instr_length); + state->_rip += vmcs_read(vcpu, VM_EXIT_INFO_INSTRUCTION_LENGTH); vcpu->rip_dirty = 1; } @@ -2092,6 +2092,7 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) em_mode_t mode; em_context_t *em_ctxt = &vcpu->emulate_ctxt; uint8_t instr[INSTR_MAX_LEN] = {0}; + uint32_t exit_instr_length = vmcs_read(vcpu, VM_EXIT_INFO_INSTRUCTION_LENGTH); uint64_t cs_base = vcpu->state->_cs.base; uint64_t rip = vcpu->state->_rip; uint64_t va; @@ -2138,16 +2139,16 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) hax_panic_vcpu(vcpu, "em_decode_insn() failed: vcpu_id=%u," " len=%u, CS:IP=0x%llx:0x%llx, instr[0..5]=" "0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", vcpu->vcpu_id, - vcpu->vmx.exit_instr_length, cs_base, rip, instr[0], - instr[1], instr[2], instr[3], instr[4], instr[5]); + exit_instr_length, cs_base, rip, instr[0], instr[1], + instr[2], instr[3], instr[4], instr[5]); dump_vmcs(vcpu); return HAX_RESUME; } - if (em_ctxt->len != vcpu->vmx.exit_instr_length) { + if (em_ctxt->len != exit_instr_length) { hax_debug("Inferred instruction length %u does not match VM-exit" " instruction length %u (CS:IP=0x%llx:0x%llx, instr[0..5]=" "0x%x 0x%x 0x%x 0x%x 0x%x 0x%x)\n", em_ctxt->len, - vcpu->vmx.exit_instr_length, cs_base, rip, instr[0], instr[1], + exit_instr_length, cs_base, rip, instr[0], instr[1], instr[2], instr[3], instr[4], instr[5]); } rc = em_emulate_insn(em_ctxt); @@ -2155,8 +2156,8 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) hax_panic_vcpu(vcpu, "em_emulate_insn() failed: vcpu_id=%u," " len=%u, CS:IP=0x%llx:0x%llx, instr[0..5]=" "0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", vcpu->vcpu_id, - vcpu->vmx.exit_instr_length, cs_base, rip, instr[0], - instr[1], instr[2], instr[3], instr[4], instr[5]); + exit_instr_length, cs_base, rip, instr[0], instr[1], + instr[2], instr[3], instr[4], instr[5]); dump_vmcs(vcpu); return HAX_RESUME; } @@ -2233,7 +2234,7 @@ static em_status_t vcpu_read_memory(void *obj, uint64_t ea, uint64_t *value, uint64_t pa; if (flags & EM_OPS_NO_TRANSLATION) { - pa = vmx(vcpu, exit_gpa); + pa = vmcs_read(vcpu, VM_EXIT_INFO_GUEST_PHYSICAL_ADDRESS); } else { vcpu_translate(vcpu, ea, 0, &pa, NULL, false); } @@ -2271,7 +2272,7 @@ static em_status_t vcpu_write_memory(void *obj, uint64_t ea, uint64_t *value, uint64_t pa; if (flags & EM_OPS_NO_TRANSLATION) { - pa = vmx(vcpu, exit_gpa); + pa = vmcs_read(vcpu, VM_EXIT_INFO_GUEST_PHYSICAL_ADDRESS); } else { vcpu_translate(vcpu, ea, 0, &pa, NULL, false); } @@ -2321,7 +2322,7 @@ static int exit_exc_nmi(struct vcpu_t *vcpu, struct hax_tunnel *htun) struct vcpu_state_t *state = vcpu->state; interruption_info_t exit_intr_info; - exit_intr_info.raw = vmx(vcpu, exit_intr_info).raw; + exit_intr_info.raw = vmcs_read(vcpu, VM_EXIT_INFO_INTERRUPT_INFO); htun->_exit_reason = vmx(vcpu, exit_reason).basic_reason; hax_debug("exception vmexit vector:%x\n", exit_intr_info.vector); @@ -3680,7 +3681,7 @@ static int exit_ept_misconfiguration(struct vcpu_t *vcpu, htun->_exit_reason = vmx(vcpu, exit_reason).basic_reason; #ifdef CONFIG_HAX_EPT2 - gpa = vmx(vcpu, exit_gpa); + gpa = vmcs_read(vcpu, VM_EXIT_INFO_GUEST_PHYSICAL_ADDRESS); ret = ept_handle_misconfiguration(&vcpu->vm->gpa_space, &vcpu->vm->ept_tree, gpa); if (ret > 0) { @@ -3712,7 +3713,7 @@ static int exit_ept_violation(struct vcpu_t *vcpu, struct hax_tunnel *htun) return HAX_RESUME; } - gpa = vmx(vcpu, exit_gpa); + gpa = vmcs_read(vcpu, VM_EXIT_INFO_GUEST_PHYSICAL_ADDRESS); #ifdef CONFIG_HAX_EPT2 ret = ept_handle_access_violation(&vcpu->vm->gpa_space, &vcpu->vm->ept_tree, @@ -3736,7 +3737,8 @@ static int exit_ept_violation(struct vcpu_t *vcpu, struct hax_tunnel *htun) * TODO: Handle this case properly. */ hax_warning("%s: Unexpected EPT violation cause. Skipping instruction" - " (len=%u)\n", __func__, vcpu->vmx.exit_instr_length); + " (len=%u)\n", __func__, + vmcs_read(vcpu, VM_EXIT_INFO_INSTRUCTION_LENGTH)); advance_rip(vcpu); return HAX_EXIT; } diff --git a/core/vmx.c b/core/vmx.c index 2b824fae..5f7318f6 100644 --- a/core/vmx.c +++ b/core/vmx.c @@ -35,6 +35,7 @@ #include "include/hax_core_interface.h" #include "include/ia32.h" #include "include/ia32_defs.h" +#include "include/vtlb.h" static void _vmx_vmwrite(struct vcpu_t *vcpu, const char *name, component_index_t component, @@ -304,7 +305,7 @@ void get_interruption_info_t(interruption_info_t *info, uint8_t v, uint8_t t) #define COMP_PENDING(cache_r, cache_w, width, name) \ COMP_PENDING_##cache_w(name) -void vmcs_write_pending(struct vcpu_t* vcpu) +void vcpu_handle_vmcs_pending(struct vcpu_t* vcpu) { if (!vcpu || !vcpu->vmx.vmcs_cache_w.dirty) return; @@ -312,4 +313,9 @@ void vmcs_write_pending(struct vcpu_t* vcpu) #define COMP COMP_PENDING VMCS_COMPS #undef COMP + if (vcpu->vmcs_pending_guest_cr3) { + vmwrite(vcpu, GUEST_CR3, vtlb_get_cr3(vcpu)); + vcpu->vmcs_pending_guest_cr3 = 0; + } + vcpu->vmx.vmcs_cache_w.dirty = 0; } diff --git a/core/vtlb.c b/core/vtlb.c index 9943caf6..617a9c21 100644 --- a/core/vtlb.c +++ b/core/vtlb.c @@ -781,7 +781,7 @@ static uint32_t vcpu_mmu_walk(struct vcpu_t *vcpu, hax_vaddr_t va, uint32_t acce bool handle_vtlb(struct vcpu_t *vcpu) { - uint32_t access = vmx(vcpu, exit_exception_error_code); + uint32_t access = vmcs_read(vcpu, VM_EXIT_INFO_EXCEPTION_ERROR_CODE); pagemode_t mode = vcpu_get_pagemode(vcpu); hax_paddr_t pdir = vcpu->state->_cr3 & (mode == PM_PAE ? ~0x1fULL : ~0xfffULL); hax_vaddr_t cr2 = vmx(vcpu, exit_qualification).address; From 5636f629d1ad95a00a4df69e68c6968255a0cbc0 Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Tue, 20 Nov 2018 14:57:41 -0800 Subject: [PATCH 04/13] Fixed padding in vcpu_state_t Signed-off-by: Alexandro Sanchez Bach --- docs/api.md | 16 +--------------- include/vcpu_state.h | 1 + 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/docs/api.md b/docs/api.md index b3f6f283..893831ed 100644 --- a/docs/api.md +++ b/docs/api.md @@ -497,20 +497,6 @@ TODO: Describe * Since: API v1 * Parameter: `struct vcpu_state_t regs`, where ``` - union interruptibility_state_t { - uint32 raw; - struct { - uint32 sti_blocking : 1; - uint32 movss_blocking : 1; - uint32 smi_blocking : 1; - uint32 nmi_blocking : 1; - uint32 reserved : 28; - }; - uint64_t pad; - }; - - typedef union interruptibility_state_t interruptibility_state_t; - struct segment_desc_t { uint16 selector; uint16 _dummy; @@ -652,7 +638,7 @@ TODO: Describe uint32 _activity_state; uint32 pad; - interruptibility_state_t _interruptibility_state; + uint64 pad2; }; ``` * (Input) `regs`: diff --git a/include/vcpu_state.h b/include/vcpu_state.h index d42c862e..a9fa144c 100644 --- a/include/vcpu_state.h +++ b/include/vcpu_state.h @@ -173,6 +173,7 @@ struct vcpu_state_t { uint32_t _activity_state; uint32_t pad; + uint64_t pad2; } PACKED; void dump(void); From 3a3f953964bc5d335fdaee5cf7942b57571c68b1 Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Wed, 21 Nov 2018 11:58:24 +0100 Subject: [PATCH 05/13] Fixed style issues and removed unnecessary fields Signed-off-by: Alexandro Sanchez Bach --- core/include/vcpu.h | 1 - core/include/vmx.h | 13 ++++--------- core/vcpu.c | 1 - 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/core/include/vcpu.h b/core/include/vcpu.h index 419e366f..3e515959 100644 --- a/core/include/vcpu.h +++ b/core/include/vcpu.h @@ -162,7 +162,6 @@ struct vcpu_t { uint64_t fs_base_dirty : 1; uint64_t interruptibility_dirty : 1; uint64_t pcpu_ctls_dirty : 1; - uint64_t padding : 46; }; /* For TSC offseting feature*/ diff --git a/core/include/vmx.h b/core/include/vmx.h index dc9435c7..55ec0ba0 100644 --- a/core/include/vmx.h +++ b/core/include/vmx.h @@ -877,11 +877,6 @@ struct vcpu_vmx_data { uint64_t cr0_mask, cr0_shadow; uint64_t cr4_mask, cr4_shadow; - uint32_t entry_exception_vector; - uint32_t entry_exception_error_code; - - interruption_info_t entry_intr_info; - uint32_t entry_instr_length; exit_reason_t exit_reason; exit_qualification_t exit_qualification; @@ -914,12 +909,12 @@ void vmx_vmwrite(struct vcpu_t *vcpu, const char *name, */ #define COMP_R_0(width, name) \ static inline COMP_TYPE_##width \ - vmcs_read_##name(struct vcpu_vmx_data* vcpu_vmx) { \ + vmcs_read_##name(struct vcpu_vmx_data *vcpu_vmx) { \ return vmread(vcpu_vmx->parent, name); \ } #define COMP_R_1(width, name) \ static inline COMP_TYPE_##width \ - vmcs_read_##name(struct vcpu_vmx_data* vcpu_vmx) { \ + vmcs_read_##name(struct vcpu_vmx_data *vcpu_vmx) { \ if (vcpu_vmx->vmcs_cache_r.name##_cache) \ return vcpu_vmx->vmcs.name##_value; \ vcpu_vmx->vmcs.name##_value = vmread(vcpu_vmx->parent, name); \ @@ -932,12 +927,12 @@ void vmx_vmwrite(struct vcpu_t *vcpu, const char *name, * Writes the VMCS-component to the specified VCPU (cached if enabled). */ #define COMP_W_0(width, name) \ - static inline void vmcs_write_##name(struct vcpu_vmx_data* vcpu_vmx, \ + static inline void vmcs_write_##name(struct vcpu_vmx_data *vcpu_vmx, \ COMP_TYPE_##width value) { \ vmwrite(vcpu_vmx->parent, name, value); \ } #define COMP_W_1(width, name) \ - static inline void vmcs_write_##name(struct vcpu_vmx_data* vcpu_vmx, \ + static inline void vmcs_write_##name(struct vcpu_vmx_data *vcpu_vmx, \ COMP_TYPE_##width value) { \ vcpu_vmx->vmcs.name##_value = value; \ /*vcpu_vmx->vmcs_cache_r.name##_cache = 1;*/ \ diff --git a/core/vcpu.c b/core/vcpu.c index dba3e2f7..d9fa961b 100644 --- a/core/vcpu.c +++ b/core/vcpu.c @@ -557,7 +557,6 @@ static void vcpu_init(struct vcpu_t *vcpu) vcpu->cr_pat = 0x0007040600070406ULL; vcpu->cpuid_features_flag_mask = 0xffffffffffffffffULL; vcpu->cur_state = GS_VALID; - vmx(vcpu, entry_exception_vector) = ~0u; vmx(vcpu, cr0_mask) = 0; vmx(vcpu, cr0_shadow) = 0; vmx(vcpu, cr4_mask) = 0; From 7693efb82cadbd9c4b2c3a78fa5631f25c1d108e Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Wed, 21 Nov 2018 06:38:44 -0800 Subject: [PATCH 06/13] Fixed unnecessary writes to HOST_RIP Signed-off-by: Alexandro Sanchez Bach --- core/cpu.c | 7 ------- core/vcpu.c | 4 ++++ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/cpu.c b/core/cpu.c index 6e1b9040..448c09ca 100644 --- a/core/cpu.c +++ b/core/cpu.c @@ -271,19 +271,12 @@ __attribute__ ((__noinline__)) vmx_result_t cpu_vmx_run(struct vcpu_t *vcpu, struct hax_tunnel *htun) { vmx_result_t result = 0; - mword host_rip; /* prepare the RIP */ hax_debug("vm entry!\n"); vcpu_save_host_state(vcpu); hax_disable_irq(); - /* - * put the vmwrite before is_running, so that the vcpu->cpu_id is set - * when we check vcpu->is_running in vcpu_pause - */ - host_rip = vmx_get_rip(); - vmwrite(vcpu, HOST_RIP, (mword)host_rip); vcpu->is_running = 1; #ifdef DEBUG_HOST_STATE vcpu_get_host_state(vcpu, 1); diff --git a/core/vcpu.c b/core/vcpu.c index d9fa961b..70f5f5f9 100644 --- a/core/vcpu.c +++ b/core/vcpu.c @@ -1413,6 +1413,10 @@ static void fill_common_vmcs(struct vcpu_t *vcpu) vmwrite(vcpu, HOST_GDTR_BASE, get_kernel_gdtr_base()); vmwrite(vcpu, HOST_IDTR_BASE, get_kernel_idtr_base()); + // The host RIP, used during VM-exit events, is only updated if HAXM + // is reloaded. Thus never changes during the lifetime of the VCPU. + vmwrite(vcpu, HOST_RIP, (mword)vmx_get_rip()); + #define WRITE_CONTROLS(vcpu, f, v) { \ uint32_t g = v & cpu_data->vmx_info.v##_1 | cpu_data->vmx_info.v##_0; \ vmwrite(vcpu, f, v = g); \ From 4b39e3550623fdd58d926c90fd2cf93f2795080f Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Wed, 21 Nov 2018 08:52:33 -0800 Subject: [PATCH 07/13] Fixed missing read-cache flush after guest-exit Signed-off-by: Alexandro Sanchez Bach --- core/cpu.c | 4 +++- core/include/vmx.h | 12 +++++++++++- core/intr_exc.c | 15 +++------------ core/vmx.c | 13 ++++++++++--- 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/core/cpu.c b/core/cpu.c index 448c09ca..ef466980 100644 --- a/core/cpu.c +++ b/core/cpu.c @@ -297,6 +297,8 @@ vmx_result_t cpu_vmx_run(struct vcpu_t *vcpu, struct hax_tunnel *htun) compare_host_state(vcpu); #endif + vcpu_vmcs_flush_cache_r(vcpu); + if (result != VMX_SUCCEED) { cpu_vmentry_failed(vcpu, result); htun->_exit_reason = 0; @@ -329,7 +331,7 @@ int cpu_vmx_execute(struct vcpu_t *vcpu, struct hax_tunnel *htun) hax_panic_log(vcpu); return 0; } - vcpu_handle_vmcs_pending(vcpu); + vcpu_vmcs_flush_cache_w(vcpu); vcpu_inject_intr(vcpu, htun); /* sometimes, the code segment type from qemu can be 10 (code segment), diff --git a/core/include/vmx.h b/core/include/vmx.h index 55ec0ba0..d2147674 100644 --- a/core/include/vmx.h +++ b/core/include/vmx.h @@ -960,7 +960,17 @@ VMCS_COMPS #define vmcs_write(vcpu, name, value) \ vmcs_write_##name(&vcpu->vmx, value) -void vcpu_handle_vmcs_pending(struct vcpu_t *vcpu); +/** + * Flush VMCS read-cache after guest-exit. + * @param vcpu VCPU object + */ +void vcpu_vmcs_flush_cache_r(struct vcpu_t *vcpu); + +/** + * Flush VMCS write-cache before guest-enter (calling vmwrite if required). + * @param vcpu VCPU object + */ +void vcpu_vmcs_flush_cache_w(struct vcpu_t *vcpu); #define VMREAD_SEG(vcpu, seg, val) \ ((val).selector = vmread(vcpu, GUEST_##seg##_SELECTOR), \ diff --git a/core/intr_exc.c b/core/intr_exc.c index 8287c74a..df25c347 100644 --- a/core/intr_exc.c +++ b/core/intr_exc.c @@ -217,21 +217,12 @@ void hax_inject_exception(struct vcpu_t *vcpu, uint8_t vector, uint32_t error_co intr_info = (1 << 31) | (EXCEPTION << 8) | vector; if (error_code != NO_ERROR_CODE) { intr_info |= 1 << 11; - if (vector == VECTOR_PF) { - vmcs_write(vcpu, VMX_ENTRY_EXCEPTION_ERROR_CODE, error_code); - } else { - vmwrite(vcpu, VMX_ENTRY_EXCEPTION_ERROR_CODE, error_code); - } + vmcs_write(vcpu, VMX_ENTRY_EXCEPTION_ERROR_CODE, error_code); } } - if (vector == VECTOR_PF) { - vmcs_write(vcpu, VMX_ENTRY_INSTRUCTION_LENGTH, exit_instr_length); - vmcs_write(vcpu, VMX_ENTRY_INTERRUPT_INFO, intr_info); - } else { - vmwrite(vcpu, VMX_ENTRY_INSTRUCTION_LENGTH, exit_instr_length); - vmwrite(vcpu, VMX_ENTRY_INTERRUPT_INFO, intr_info); - } + vmcs_write(vcpu, VMX_ENTRY_INSTRUCTION_LENGTH, exit_instr_length); + vmcs_write(vcpu, VMX_ENTRY_INTERRUPT_INFO, intr_info); hax_debug("Guest is injecting exception info:%x\n", intr_info); vcpu->event_injected = 1; diff --git a/core/vmx.c b/core/vmx.c index 5f7318f6..d790a6c7 100644 --- a/core/vmx.c +++ b/core/vmx.c @@ -298,14 +298,21 @@ void get_interruption_info_t(interruption_info_t *info, uint8_t v, uint8_t t) info->valid = 1; } +void vcpu_vmcs_flush_cache_r(struct vcpu_t *vcpu) +{ + memset(&vcpu->vmx.vmcs_cache_r, 0, sizeof(struct vmx_vmcs_cache_r_t)); +} + #define COMP_PENDING_0(name) #define COMP_PENDING_1(name) \ - if (vcpu->vmx.vmcs_cache_w.name##_dirty) \ - vmwrite(vcpu, name, vcpu->vmx.vmcs.name##_value); + if (vcpu->vmx.vmcs_cache_w.name##_dirty) { \ + vmwrite(vcpu, name, vcpu->vmx.vmcs.name##_value); \ + vcpu->vmx.vmcs_cache_w.name##_dirty = 0; \ + } #define COMP_PENDING(cache_r, cache_w, width, name) \ COMP_PENDING_##cache_w(name) -void vcpu_handle_vmcs_pending(struct vcpu_t* vcpu) +void vcpu_vmcs_flush_cache_w(struct vcpu_t *vcpu) { if (!vcpu || !vcpu->vmx.vmcs_cache_w.dirty) return; From 46fc754aca5fa448b89266b22277a916f9939da0 Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Thu, 22 Nov 2018 05:46:42 -0800 Subject: [PATCH 08/13] Optimization: Cached segment reads Signed-off-by: Alexandro Sanchez Bach --- core/cpu.c | 3 -- core/include/vcpu.h | 7 +++ core/include/vmx.h | 48 ++++++++--------- core/vcpu.c | 39 +++++--------- core/vmx.c | 124 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 168 insertions(+), 53 deletions(-) diff --git a/core/cpu.c b/core/cpu.c index ef466980..18a76a0d 100644 --- a/core/cpu.c +++ b/core/cpu.c @@ -386,9 +386,6 @@ int cpu_vmx_execute(struct vcpu_t *vcpu, struct hax_tunnel *htun) state->_rflags = vmread(vcpu, GUEST_RFLAGS); state->_rsp = vmread(vcpu, GUEST_RSP); - VMREAD_SEG(vcpu, CS, state->_cs); - VMREAD_SEG(vcpu, DS, state->_ds); - VMREAD_SEG(vcpu, ES, state->_es); vmread_cr(vcpu); if (vcpu->nr_pending_intrs > 0 || hax_intr_is_blocked(vcpu)) diff --git a/core/include/vcpu.h b/core/include/vcpu.h index 3e515959..14d93e89 100644 --- a/core/include/vcpu.h +++ b/core/include/vcpu.h @@ -254,4 +254,11 @@ bool vcpu_is_panic(struct vcpu_t *vcpu); void hax_panic_vcpu(struct vcpu_t *v, char *fmt, ...); #endif +// Extension-specific operations + +uint16_t vcpu_get_seg_selector(struct vcpu_t *vcpu, int seg); +mword vcpu_get_seg_base(struct vcpu_t *vcpu, int seg); +uint32_t vcpu_get_seg_limit(struct vcpu_t *vcpu, int seg); +uint32_t vcpu_get_seg_ar(struct vcpu_t *vcpu, int seg); + #endif // HAX_CORE_VCPU_H_ diff --git a/core/include/vmx.h b/core/include/vmx.h index d2147674..c64affe2 100644 --- a/core/include/vmx.h +++ b/core/include/vmx.h @@ -344,12 +344,12 @@ typedef enum component_index_t component_index_t; COMP(0, 0, W_UL, GUEST_CR0) \ COMP(0, 0, W_UL, GUEST_CR3) \ COMP(0, 0, W_UL, GUEST_CR4) \ - COMP(0, 0, W_UL, GUEST_ES_BASE) \ - COMP(0, 0, W_UL, GUEST_CS_BASE) \ - COMP(0, 0, W_UL, GUEST_SS_BASE) \ - COMP(0, 0, W_UL, GUEST_DS_BASE) \ - COMP(0, 0, W_UL, GUEST_FS_BASE) \ - COMP(0, 0, W_UL, GUEST_GS_BASE) \ + COMP(1, 0, W_UL, GUEST_ES_BASE) \ + COMP(1, 0, W_UL, GUEST_CS_BASE) \ + COMP(1, 0, W_UL, GUEST_SS_BASE) \ + COMP(1, 0, W_UL, GUEST_DS_BASE) \ + COMP(1, 0, W_UL, GUEST_FS_BASE) \ + COMP(1, 0, W_UL, GUEST_GS_BASE) \ COMP(0, 0, W_UL, GUEST_LDTR_BASE) \ COMP(0, 0, W_UL, GUEST_TR_BASE) \ COMP(0, 0, W_UL, GUEST_GDTR_BASE) \ @@ -385,20 +385,20 @@ typedef enum component_index_t component_index_t; COMP(1, 0, W_32, VM_EXIT_INFO_INSTRUCTION_LENGTH) \ COMP(0, 0, W_32, VM_EXIT_INFO_INSTRUCTION_INFO) \ COMP(0, 0, W_32, HOST_SYSENTER_CS) \ - COMP(0, 0, W_32, GUEST_ES_AR) \ - COMP(0, 0, W_32, GUEST_CS_AR) \ - COMP(0, 0, W_32, GUEST_SS_AR) \ - COMP(0, 0, W_32, GUEST_DS_AR) \ - COMP(0, 0, W_32, GUEST_FS_AR) \ - COMP(0, 0, W_32, GUEST_GS_AR) \ + COMP(1, 0, W_32, GUEST_ES_AR) \ + COMP(1, 0, W_32, GUEST_CS_AR) \ + COMP(1, 0, W_32, GUEST_SS_AR) \ + COMP(1, 0, W_32, GUEST_DS_AR) \ + COMP(1, 0, W_32, GUEST_FS_AR) \ + COMP(1, 0, W_32, GUEST_GS_AR) \ COMP(0, 0, W_32, GUEST_LDTR_AR) \ COMP(0, 0, W_32, GUEST_TR_AR) \ - COMP(0, 0, W_32, GUEST_ES_LIMIT) \ - COMP(0, 0, W_32, GUEST_CS_LIMIT) \ - COMP(0, 0, W_32, GUEST_SS_LIMIT) \ - COMP(0, 0, W_32, GUEST_DS_LIMIT) \ - COMP(0, 0, W_32, GUEST_FS_LIMIT) \ - COMP(0, 0, W_32, GUEST_GS_LIMIT) \ + COMP(1, 0, W_32, GUEST_ES_LIMIT) \ + COMP(1, 0, W_32, GUEST_CS_LIMIT) \ + COMP(1, 0, W_32, GUEST_SS_LIMIT) \ + COMP(1, 0, W_32, GUEST_DS_LIMIT) \ + COMP(1, 0, W_32, GUEST_FS_LIMIT) \ + COMP(1, 0, W_32, GUEST_GS_LIMIT) \ COMP(0, 0, W_32, GUEST_LDTR_LIMIT) \ COMP(0, 0, W_32, GUEST_TR_LIMIT) \ COMP(0, 0, W_32, GUEST_GDTR_LIMIT) \ @@ -416,12 +416,12 @@ typedef enum component_index_t component_index_t; COMP(0, 0, W_16, HOST_GS_SELECTOR) \ COMP(0, 0, W_16, HOST_SS_SELECTOR) \ COMP(0, 0, W_16, HOST_TR_SELECTOR) \ - COMP(0, 0, W_16, GUEST_ES_SELECTOR) \ - COMP(0, 0, W_16, GUEST_CS_SELECTOR) \ - COMP(0, 0, W_16, GUEST_SS_SELECTOR) \ - COMP(0, 0, W_16, GUEST_DS_SELECTOR) \ - COMP(0, 0, W_16, GUEST_FS_SELECTOR) \ - COMP(0, 0, W_16, GUEST_GS_SELECTOR) \ + COMP(1, 0, W_16, GUEST_ES_SELECTOR) \ + COMP(1, 0, W_16, GUEST_CS_SELECTOR) \ + COMP(1, 0, W_16, GUEST_SS_SELECTOR) \ + COMP(1, 0, W_16, GUEST_DS_SELECTOR) \ + COMP(1, 0, W_16, GUEST_FS_SELECTOR) \ + COMP(1, 0, W_16, GUEST_GS_SELECTOR) \ COMP(0, 0, W_16, GUEST_LDTR_SELECTOR) \ COMP(0, 0, W_16, GUEST_TR_SELECTOR) diff --git a/core/vcpu.c b/core/vcpu.c index 70f5f5f9..5c9bbc95 100644 --- a/core/vcpu.c +++ b/core/vcpu.c @@ -2096,8 +2096,8 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) em_context_t *em_ctxt = &vcpu->emulate_ctxt; uint8_t instr[INSTR_MAX_LEN] = {0}; uint32_t exit_instr_length = vmcs_read(vcpu, VM_EXIT_INFO_INSTRUCTION_LENGTH); - uint64_t cs_base = vcpu->state->_cs.base; uint64_t rip = vcpu->state->_rip; + segment_desc_t cs; uint64_t va; // Clean up the emulation context of the previous MMIO instruction, so that @@ -2105,11 +2105,12 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) vcpu_init_emulator(vcpu); // Detect guest mode + cs.ar = vcpu_get_seg_ar(vcpu, SEG_CS); if (!(vcpu->state->_cr0 & CR0_PE)) mode = EM_MODE_REAL; - else if (vcpu->state->_cs.long_mode == 1) + else if (cs.long_mode == 1) mode = EM_MODE_PROT64; - else if (vcpu->state->_cs.operand_size == 1) + else if (cs.operand_size == 1) mode = EM_MODE_PROT32; else mode = EM_MODE_PROT16; @@ -2117,12 +2118,13 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) // Fetch the instruction at guest CS:IP = CS.Base + IP, omitting segment // limit and privilege checks - va = (mode == EM_MODE_PROT64) ? rip : cs_base + rip; + cs.base = vcpu_get_seg_base(vcpu, SEG_CS); + va = (mode == EM_MODE_PROT64) ? rip : cs.base + rip; #ifdef CONFIG_HAX_EPT2 if (mmio_fetch_instruction(vcpu, va, instr, INSTR_MAX_LEN)) { hax_panic_vcpu(vcpu, "%s: mmio_fetch_instruction() failed: vcpu_id=%u," " gva=0x%llx (CS:IP=0x%llx:0x%llx)\n", - __func__, vcpu->vcpu_id, va, cs_base, rip); + __func__, vcpu->vcpu_id, va, cs.base, rip); dump_vmcs(vcpu); return -1; } @@ -2130,7 +2132,7 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) if (!vcpu_read_guest_virtual(vcpu, va, &instr, INSTR_MAX_LEN, INSTR_MAX_LEN, 0)) { hax_panic_vcpu(vcpu, "Error reading instruction at 0x%llx for decoding" - " (CS:IP=0x%llx:0x%llx)\n", va, cs_base, rip); + " (CS:IP=0x%llx:0x%llx)\n", va, cs.base, rip); dump_vmcs(vcpu); return -1; } @@ -2142,7 +2144,7 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) hax_panic_vcpu(vcpu, "em_decode_insn() failed: vcpu_id=%u," " len=%u, CS:IP=0x%llx:0x%llx, instr[0..5]=" "0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", vcpu->vcpu_id, - exit_instr_length, cs_base, rip, instr[0], instr[1], + exit_instr_length, cs.base, rip, instr[0], instr[1], instr[2], instr[3], instr[4], instr[5]); dump_vmcs(vcpu); return HAX_RESUME; @@ -2151,7 +2153,7 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) hax_debug("Inferred instruction length %u does not match VM-exit" " instruction length %u (CS:IP=0x%llx:0x%llx, instr[0..5]=" "0x%x 0x%x 0x%x 0x%x 0x%x 0x%x)\n", em_ctxt->len, - exit_instr_length, cs_base, rip, instr[0], instr[1], + exit_instr_length, cs.base, rip, instr[0], instr[1], instr[2], instr[3], instr[4], instr[5]); } rc = em_emulate_insn(em_ctxt); @@ -2159,7 +2161,7 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) hax_panic_vcpu(vcpu, "em_emulate_insn() failed: vcpu_id=%u," " len=%u, CS:IP=0x%llx:0x%llx, instr[0..5]=" "0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", vcpu->vcpu_id, - exit_instr_length, cs_base, rip, instr[0], instr[1], + exit_instr_length, cs.base, rip, instr[0], instr[1], instr[2], instr[3], instr[4], instr[5]); dump_vmcs(vcpu); return HAX_RESUME; @@ -2206,22 +2208,7 @@ void vcpu_write_rflags(void *obj, uint64_t value) static uint64_t vcpu_get_segment_base(void *obj, uint32_t segment) { struct vcpu_t *vcpu = obj; - switch (segment) { - case SEG_CS: - return vcpu->state->_cs.base; - case SEG_DS: - return vcpu->state->_ds.base; - case SEG_ES: - return vcpu->state->_es.base; - case SEG_FS: - return vcpu->state->_fs.base; - case SEG_GS: - return vcpu->state->_gs.base; - case SEG_SS: - return vcpu->state->_ss.base; - default: - return vcpu->state->_ds.base; - } + return vcpu_get_seg_base(vcpu, segment); } static void vcpu_advance_rip(void *obj, uint64_t len) @@ -3299,7 +3286,7 @@ static int handle_msr_read(struct vcpu_t *vcpu, uint32_t msr, uint64_t *val) } case IA32_FS_BASE: { if (vcpu->fs_base_dirty) - *val = vcpu->state->_fs.base; + *val = vcpu_get_seg_base(vcpu, SEG_FS); else *val = vmread(vcpu, GUEST_FS_BASE); break; diff --git a/core/vmx.c b/core/vmx.c index d790a6c7..abb55dd6 100644 --- a/core/vmx.c +++ b/core/vmx.c @@ -326,3 +326,127 @@ void vcpu_vmcs_flush_cache_w(struct vcpu_t *vcpu) } vcpu->vmx.vmcs_cache_w.dirty = 0; } + +uint16_t vcpu_get_seg_selector(struct vcpu_t *vcpu, int seg) +{ + uint16_t value; + + switch (seg) { + case SEG_CS: + value = vmcs_read(vcpu, GUEST_CS_SELECTOR); + break; + case SEG_SS: + value = vmcs_read(vcpu, GUEST_SS_SELECTOR); + break; + case SEG_DS: + value = vmcs_read(vcpu, GUEST_DS_SELECTOR); + break; + case SEG_ES: + value = vmcs_read(vcpu, GUEST_ES_SELECTOR); + break; + case SEG_FS: + value = vmcs_read(vcpu, GUEST_FS_SELECTOR); + break; + case SEG_GS: + value = vmcs_read(vcpu, GUEST_GS_SELECTOR); + break; + default: + hax_error("vcpu_get_seg_selector: Unexpected segment (%d)\n", seg); + value = 0; + } + return value; +} + +mword vcpu_get_seg_base(struct vcpu_t *vcpu, int seg) +{ + mword value; + + switch (seg) { + case SEG_CS: + value = vmcs_read(vcpu, GUEST_CS_BASE); + break; + case SEG_SS: + value = vmcs_read(vcpu, GUEST_SS_BASE); + break; + case SEG_DS: + value = vmcs_read(vcpu, GUEST_DS_BASE); + break; + case SEG_ES: + value = vmcs_read(vcpu, GUEST_ES_BASE); + break; + case SEG_FS: + value = vmcs_read(vcpu, GUEST_FS_BASE); + break; + case SEG_GS: + value = vmcs_read(vcpu, GUEST_GS_BASE); + break; + default: + hax_error("vcpu_get_seg_base: Unexpected segment (%d)\n", seg); + value = 0; + } + return value; +} + +uint32_t vcpu_get_seg_limit(struct vcpu_t *vcpu, int seg) +{ + uint32_t value; + + switch (seg) { + case SEG_CS: + value = vmcs_read(vcpu, GUEST_CS_LIMIT); + break; + case SEG_SS: + value = vmcs_read(vcpu, GUEST_SS_LIMIT); + break; + case SEG_DS: + value = vmcs_read(vcpu, GUEST_DS_LIMIT); + break; + case SEG_ES: + value = vmcs_read(vcpu, GUEST_ES_LIMIT); + break; + case SEG_FS: + value = vmcs_read(vcpu, GUEST_FS_LIMIT); + break; + case SEG_GS: + value = vmcs_read(vcpu, GUEST_GS_LIMIT); + break; + default: + hax_error("vcpu_get_seg_limit: Unexpected segment (%d)\n", seg); + value = 0; + } + return value; +} + +uint32_t vcpu_get_seg_ar(struct vcpu_t *vcpu, int seg) +{ + uint32_t value; + + switch (seg) { + case SEG_CS: + value = vmcs_read(vcpu, GUEST_CS_AR); + break; + case SEG_SS: + value = vmcs_read(vcpu, GUEST_SS_AR); + break; + case SEG_DS: + value = vmcs_read(vcpu, GUEST_DS_AR); + break; + case SEG_ES: + value = vmcs_read(vcpu, GUEST_ES_AR); + break; + case SEG_FS: + value = vmcs_read(vcpu, GUEST_FS_AR); + break; + case SEG_GS: + value = vmcs_read(vcpu, GUEST_GS_AR); + break; + default: + hax_error("vcpu_get_seg_ar: Unexpected segment (%d)\n", seg); + value = 0; + } + + if (value & (1 << 16) /* ar.null */) { + return 0; + } + return value; +} From 8e49240be4d8185fcbd0903968bd3c081bc278b6 Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Thu, 22 Nov 2018 06:07:13 -0800 Subject: [PATCH 09/13] Optimization: Cached RIP reads Signed-off-by: Alexandro Sanchez Bach --- core/cpu.c | 4 +--- core/include/vcpu.h | 1 + core/include/vmx.h | 2 +- core/vcpu.c | 13 +++++++------ core/vmx.c | 5 +++++ 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/core/cpu.c b/core/cpu.c index 18a76a0d..671595a7 100644 --- a/core/cpu.c +++ b/core/cpu.c @@ -375,8 +375,6 @@ int cpu_vmx_execute(struct vcpu_t *vcpu, struct hax_tunnel *htun) * reason is, we have no schedule hook to get notified of preemption * This should be changed later after get better idea */ - vcpu->state->_rip = vmread(vcpu, GUEST_RIP); - hax_handle_idt_vectoring(vcpu); vmx(vcpu, exit_qualification).raw = vmread( @@ -599,7 +597,7 @@ static void cpu_vmentry_failed(struct vcpu_t *vcpu, vmx_result_t result) uint64_t error, reason; hax_error("VM entry failed: RIP=%08lx\n", - (mword)vmread(vcpu, GUEST_RIP)); + (mword)vmcs_read(vcpu, GUEST_RIP)); //dump_vmcs(); diff --git a/core/include/vcpu.h b/core/include/vcpu.h index 14d93e89..205daa82 100644 --- a/core/include/vcpu.h +++ b/core/include/vcpu.h @@ -256,6 +256,7 @@ void hax_panic_vcpu(struct vcpu_t *v, char *fmt, ...); // Extension-specific operations +mword vcpu_get_rip(struct vcpu_t *vcpu); uint16_t vcpu_get_seg_selector(struct vcpu_t *vcpu, int seg); mword vcpu_get_seg_base(struct vcpu_t *vcpu, int seg); uint32_t vcpu_get_seg_limit(struct vcpu_t *vcpu, int seg); diff --git a/core/include/vmx.h b/core/include/vmx.h index c64affe2..040f4762 100644 --- a/core/include/vmx.h +++ b/core/include/vmx.h @@ -338,7 +338,7 @@ typedef enum component_index_t component_index_t; COMP(0, 0, W_UL, HOST_IDTR_BASE) \ COMP(0, 0, W_UL, HOST_SYSENTER_ESP) \ COMP(0, 0, W_UL, HOST_SYSENTER_EIP) \ - COMP(0, 0, W_UL, GUEST_RIP) \ + COMP(1, 0, W_UL, GUEST_RIP) \ COMP(0, 0, W_UL, GUEST_RFLAGS) \ COMP(0, 0, W_UL, GUEST_RSP) \ COMP(0, 0, W_UL, GUEST_CR0) \ diff --git a/core/vcpu.c b/core/vcpu.c index 5c9bbc95..4591e76a 100644 --- a/core/vcpu.c +++ b/core/vcpu.c @@ -1751,6 +1751,7 @@ static void advance_rip(struct vcpu_t *vcpu) vcpu->interruptibility_dirty = 1; } + state->_rip = vcpu_get_rip(vcpu); state->_rip += vmcs_read(vcpu, VM_EXIT_INFO_INSTRUCTION_LENGTH); vcpu->rip_dirty = 1; } @@ -1966,8 +1967,8 @@ static void vmwrite_cr(struct vcpu_t *vcpu) cr4_mask |= CR4_PAE; eptp = vm_get_eptp(vcpu->vm); hax_assert(eptp != INVALID_EPTP); - // hax_debug("Guest eip:%llx, EPT mode, eptp:%llx\n", vcpu->state->_rip, - // eptp); + // hax_debug("Guest eip:%llx, EPT mode, eptp:%llx\n", + // vcpu_get_rip(vcpu), eptp); vmwrite(vcpu, GUEST_CR3, state->_cr3); scpu_ctls |= ENABLE_EPT; // Set PDPTEs for vCPU if it's in or about to enter PAE paging mode @@ -2096,7 +2097,7 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) em_context_t *em_ctxt = &vcpu->emulate_ctxt; uint8_t instr[INSTR_MAX_LEN] = {0}; uint32_t exit_instr_length = vmcs_read(vcpu, VM_EXIT_INFO_INSTRUCTION_LENGTH); - uint64_t rip = vcpu->state->_rip; + uint64_t rip = vcpu_get_rip(vcpu); segment_desc_t cs; uint64_t va; @@ -2347,14 +2348,14 @@ static int exit_exc_nmi(struct vcpu_t *vcpu, struct hax_tunnel *htun) } case VECTOR_DB: { htun->_exit_status = HAX_EXIT_DEBUG; - htun->debug.rip = vcpu->state->_rip; + htun->debug.rip = vcpu_get_rip(vcpu); htun->debug.dr6 = vmx(vcpu, exit_qualification).raw; htun->debug.dr7 = vmread(vcpu, GUEST_DR7); return HAX_EXIT; } case VECTOR_BP: { htun->_exit_status = HAX_EXIT_DEBUG; - htun->debug.rip = vcpu->state->_rip; + htun->debug.rip = vcpu_get_rip(vcpu); htun->debug.dr6 = 0; htun->debug.dr7 = 0; return HAX_EXIT; @@ -2736,7 +2737,7 @@ static int exit_invlpg(struct vcpu_t *vcpu, struct hax_tunnel *htun) static int exit_rdtsc(struct vcpu_t *vcpu, struct hax_tunnel *htun) { - hax_debug("rdtsc exiting: rip: %llx\n", vcpu->state->_rip); + hax_debug("rdtsc exiting: rip: %lx\n", vcpu_get_rip(vcpu)); return HAX_RESUME; } diff --git a/core/vmx.c b/core/vmx.c index abb55dd6..51b64304 100644 --- a/core/vmx.c +++ b/core/vmx.c @@ -327,6 +327,11 @@ void vcpu_vmcs_flush_cache_w(struct vcpu_t *vcpu) vcpu->vmx.vmcs_cache_w.dirty = 0; } +mword vcpu_get_rip(struct vcpu_t *vcpu) +{ + return vmcs_read(vcpu, GUEST_RIP); +} + uint16_t vcpu_get_seg_selector(struct vcpu_t *vcpu, int seg) { uint16_t value; From 989eea99a260c5b70d20ecebca49f083582a7038 Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Thu, 22 Nov 2018 06:56:05 -0800 Subject: [PATCH 10/13] Optimization: Cached RSP/RFLAGS reads Signed-off-by: Alexandro Sanchez Bach --- core/cpu.c | 2 -- core/include/vcpu.h | 2 ++ core/include/vmx.h | 4 ++-- core/intr_exc.c | 5 +++-- core/vcpu.c | 30 ++++++++++++++++++------------ core/vmx.c | 10 ++++++++++ 6 files changed, 35 insertions(+), 18 deletions(-) diff --git a/core/cpu.c b/core/cpu.c index 671595a7..b1861163 100644 --- a/core/cpu.c +++ b/core/cpu.c @@ -382,8 +382,6 @@ int cpu_vmx_execute(struct vcpu_t *vcpu, struct hax_tunnel *htun) vmx(vcpu, interruptibility_state).raw = vmread( vcpu, GUEST_INTERRUPTIBILITY); - state->_rflags = vmread(vcpu, GUEST_RFLAGS); - state->_rsp = vmread(vcpu, GUEST_RSP); vmread_cr(vcpu); if (vcpu->nr_pending_intrs > 0 || hax_intr_is_blocked(vcpu)) diff --git a/core/include/vcpu.h b/core/include/vcpu.h index 205daa82..af0777ce 100644 --- a/core/include/vcpu.h +++ b/core/include/vcpu.h @@ -256,6 +256,8 @@ void hax_panic_vcpu(struct vcpu_t *v, char *fmt, ...); // Extension-specific operations +mword vcpu_get_rflags(struct vcpu_t *vcpu); +mword vcpu_get_rsp(struct vcpu_t *vcpu); mword vcpu_get_rip(struct vcpu_t *vcpu); uint16_t vcpu_get_seg_selector(struct vcpu_t *vcpu, int seg); mword vcpu_get_seg_base(struct vcpu_t *vcpu, int seg); diff --git a/core/include/vmx.h b/core/include/vmx.h index 040f4762..a6bdd9ca 100644 --- a/core/include/vmx.h +++ b/core/include/vmx.h @@ -339,8 +339,8 @@ typedef enum component_index_t component_index_t; COMP(0, 0, W_UL, HOST_SYSENTER_ESP) \ COMP(0, 0, W_UL, HOST_SYSENTER_EIP) \ COMP(1, 0, W_UL, GUEST_RIP) \ - COMP(0, 0, W_UL, GUEST_RFLAGS) \ - COMP(0, 0, W_UL, GUEST_RSP) \ + COMP(1, 0, W_UL, GUEST_RFLAGS) \ + COMP(1, 0, W_UL, GUEST_RSP) \ COMP(0, 0, W_UL, GUEST_CR0) \ COMP(0, 0, W_UL, GUEST_CR3) \ COMP(0, 0, W_UL, GUEST_CR4) \ diff --git a/core/intr_exc.c b/core/intr_exc.c index df25c347..32a85b80 100644 --- a/core/intr_exc.c +++ b/core/intr_exc.c @@ -119,10 +119,11 @@ static void hax_enable_intr_window(struct vcpu_t *vcpu) */ uint hax_intr_is_blocked(struct vcpu_t *vcpu) { - struct vcpu_state_t *state = vcpu->state; uint32_t intr_status; + uint64_t rflags; - if (!(state->_eflags & EFLAGS_IF)) + rflags = vcpu_get_rflags(vcpu); + if (!(rflags & EFLAGS_IF)) return 1; intr_status = vmx(vcpu, interruptibility_state).raw; diff --git a/core/vcpu.c b/core/vcpu.c index 4591e76a..f08e4efc 100644 --- a/core/vcpu.c +++ b/core/vcpu.c @@ -1062,6 +1062,7 @@ static void load_dirty_vmcs_fields(struct vcpu_t *vcpu) // rflags if (vcpu->debug_control_dirty) { // Single-stepping + state->_rflags = vcpu_get_rflags(vcpu); if (vcpu->debug_control & HAX_DEBUG_STEP) { state->_rflags |= EFLAGS_TF; } else { @@ -2173,12 +2174,20 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) static uint64_t vcpu_read_gpr(void *obj, uint32_t reg_index, uint32_t size) { struct vcpu_t *vcpu = obj; + uint64_t value; + if (reg_index >= 16) { hax_panic_vcpu(vcpu, "vcpu_read_gpr: Invalid register index\n"); return 0; } - uint64_t value = 0; - memcpy(&value, &vcpu->state->_regs[reg_index], size); + switch (reg_index) { + case REG_RSP: + value = vcpu_get_rsp(vcpu); + break; + default: + value = 0; + memcpy(&value, &vcpu->state->_regs[reg_index], size); + } return value; } @@ -2196,7 +2205,7 @@ static void vcpu_write_gpr(void *obj, uint32_t reg_index, uint64_t value, uint64_t vcpu_read_rflags(void *obj) { struct vcpu_t *vcpu = obj; - return vcpu->state->_rflags; + return vcpu_get_rflags(vcpu); } void vcpu_write_rflags(void *obj, uint64_t value) @@ -3042,6 +3051,7 @@ static int handle_string_io(struct vcpu_t *vcpu, exit_qualification_t *qual, { struct vcpu_state_t *state = vcpu->state; uint64_t count, total_size; + uint64_t rflags; uint elem_size, n, copy_size; hax_vaddr_t gla, start_gva; @@ -3065,7 +3075,8 @@ static int handle_string_io(struct vcpu_t *vcpu, exit_qualification_t *qual, // attribute of the instruction, or to check the presence of a segment // override prefix that can make OUTS read from ES:ESI instead of DS:ESI). gla = vmread(vcpu, VM_EXIT_INFO_GUEST_LINEAR_ADDRESS); - if (state->_rflags & EFLAGS_DF) { + rflags = vcpu_get_rflags(vcpu); + if (rflags & EFLAGS_DF) { start_gva = gla - (n - 1) * elem_size; htun->io._df = 1; } else { @@ -3091,7 +3102,7 @@ static int handle_string_io(struct vcpu_t *vcpu, exit_qualification_t *qual, advance_rip(vcpu); } - if (state->_rflags & EFLAGS_DF) { + if (rflags & EFLAGS_DF) { if (qual->io.direction == HAX_IO_OUT) { state->_rsi -= copy_size; } else { @@ -3843,13 +3854,8 @@ int vcpu_set_regs(struct vcpu_t *vcpu, struct vcpu_state_t *ustate) int i; int cr_dirty = 0, dr_dirty = 0; preempt_flag flags; - int rsp_dirty = 0; uint32_t vmcs_err = 0; - if (state->_rsp != ustate->_rsp) { - rsp_dirty = 1; - } - for (i = 0; i < 16; i++) { state->_regs[i] = ustate->_regs[i]; } @@ -3865,11 +3871,11 @@ int vcpu_set_regs(struct vcpu_t *vcpu, struct vcpu_state_t *ustate) state->_rip = ustate->_rip; vcpu->rip_dirty = 1; } - if (state->_rflags != ustate->_rflags) { + if (vcpu_get_rflags(vcpu) != ustate->_rflags) { state->_rflags = ustate->_rflags; vcpu->rflags_dirty = 1; } - if (rsp_dirty) { + if (vcpu_get_rsp(vcpu) != ustate->_rsp) { state->_rsp = ustate->_rsp; vmwrite(vcpu, GUEST_RSP, state->_rsp); } diff --git a/core/vmx.c b/core/vmx.c index 51b64304..eac0576f 100644 --- a/core/vmx.c +++ b/core/vmx.c @@ -327,6 +327,16 @@ void vcpu_vmcs_flush_cache_w(struct vcpu_t *vcpu) vcpu->vmx.vmcs_cache_w.dirty = 0; } +mword vcpu_get_rflags(struct vcpu_t *vcpu) +{ + return vmcs_read(vcpu, GUEST_RFLAGS); +} + +mword vcpu_get_rsp(struct vcpu_t *vcpu) +{ + return vmcs_read(vcpu, GUEST_RSP); +} + mword vcpu_get_rip(struct vcpu_t *vcpu) { return vmcs_read(vcpu, GUEST_RIP); From 61779ca8d81c5dda88f7488231453281040c5dd8 Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Fri, 23 Nov 2018 04:33:33 -0800 Subject: [PATCH 11/13] Optimization: Cached guest interruptibility/exit qualifier reads Signed-off-by: Alexandro Sanchez Bach --- core/cpu.c | 10 ++---- core/include/vmx.h | 6 ++-- core/include/vtlb.h | 6 ++-- core/intr_exc.c | 2 +- core/vcpu.c | 84 ++++++++++++++++++++++++++------------------- core/vtlb.c | 4 +-- 6 files changed, 59 insertions(+), 53 deletions(-) diff --git a/core/cpu.c b/core/cpu.c index b1861163..aff5e505 100644 --- a/core/cpu.c +++ b/core/cpu.c @@ -313,7 +313,6 @@ int cpu_vmx_execute(struct vcpu_t *vcpu, struct hax_tunnel *htun) vmx_result_t res = 0; int ret; preempt_flag flags; - struct vcpu_state_t *state = vcpu->state; uint32_t vmcs_err = 0; while (1) { @@ -367,7 +366,7 @@ int cpu_vmx_execute(struct vcpu_t *vcpu, struct hax_tunnel *htun) return -EINVAL; } - exit_reason.raw = vmread(vcpu, VM_EXIT_INFO_REASON); + exit_reason.raw = vmcs_read(vcpu, VM_EXIT_INFO_REASON); hax_debug("....exit_reason.raw %x, cpu %d %d\n", exit_reason.raw, vcpu->cpu_id, hax_cpuid()); @@ -377,11 +376,6 @@ int cpu_vmx_execute(struct vcpu_t *vcpu, struct hax_tunnel *htun) */ hax_handle_idt_vectoring(vcpu); - vmx(vcpu, exit_qualification).raw = vmread( - vcpu, VM_EXIT_INFO_QUALIFICATION); - vmx(vcpu, interruptibility_state).raw = vmread( - vcpu, GUEST_INTERRUPTIBILITY); - vmread_cr(vcpu); if (vcpu->nr_pending_intrs > 0 || hax_intr_is_blocked(vcpu)) @@ -599,7 +593,7 @@ static void cpu_vmentry_failed(struct vcpu_t *vcpu, vmx_result_t result) //dump_vmcs(); - reason = vmread(vcpu, VM_EXIT_INFO_REASON); + reason = vmcs_read(vcpu, VM_EXIT_INFO_REASON); if (result == VMX_FAIL_VALID) { error = vmread(vcpu, VMX_INSTRUCTION_ERROR_CODE); hax_error("VMfailValid. Prev exit: %llx. Error code: %llu (%s)\n", diff --git a/core/include/vmx.h b/core/include/vmx.h index a6bdd9ca..394351c6 100644 --- a/core/include/vmx.h +++ b/core/include/vmx.h @@ -320,7 +320,7 @@ typedef enum component_index_t component_index_t; COMP(0, 0, W_UL, VMX_CR0_READ_SHADOW) \ COMP(0, 0, W_UL, VMX_CR4_READ_SHADOW) \ COMP(0, 0, W_UL, VMX_CR3_TARGET_VAL_BASE) \ - COMP(0, 0, W_UL, VM_EXIT_INFO_QUALIFICATION) \ + COMP(1, 0, W_UL, VM_EXIT_INFO_QUALIFICATION) \ COMP(0, 0, W_UL, VM_EXIT_INFO_IO_ECX) \ COMP(0, 0, W_UL, VM_EXIT_INFO_IO_ESI) \ COMP(0, 0, W_UL, VM_EXIT_INFO_IO_EDI) \ @@ -377,7 +377,7 @@ typedef enum component_index_t component_index_t; COMP(0, 0, W_32, VMX_CR3_TARGET_COUNT) \ COMP(0, 0, W_32, VMX_PREEMPTION_TIMER) \ COMP(0, 0, W_32, VMX_INSTRUCTION_ERROR_CODE) \ - COMP(0, 0, W_32, VM_EXIT_INFO_REASON) \ + COMP(1, 0, W_32, VM_EXIT_INFO_REASON) \ COMP(1, 0, W_32, VM_EXIT_INFO_INTERRUPT_INFO) \ COMP(1, 0, W_32, VM_EXIT_INFO_EXCEPTION_ERROR_CODE) \ COMP(1, 0, W_32, VM_EXIT_INFO_IDT_VECTORING) \ @@ -405,7 +405,7 @@ typedef enum component_index_t component_index_t; COMP(0, 0, W_32, GUEST_IDTR_LIMIT) \ COMP(0, 0, W_32, GUEST_SYSENTER_CS) \ COMP(0, 0, W_32, GUEST_SMBASE) \ - COMP(0, 0, W_32, GUEST_INTERRUPTIBILITY) \ + COMP(1, 0, W_32, GUEST_INTERRUPTIBILITY) \ COMP(0, 0, W_32, GUEST_ACTIVITY_STATE) \ /* 16-bit components */ \ COMP(0, 0, W_16, VMX_VPID) \ diff --git a/core/include/vtlb.h b/core/include/vtlb.h index 630e773b..19e96551 100644 --- a/core/include/vtlb.h +++ b/core/include/vtlb.h @@ -100,10 +100,10 @@ void vcpu_invalidate_tlb_addr(struct vcpu_t *vcpu, hax_vaddr_t va); uint vcpu_vtlb_alloc(struct vcpu_t *vcpu); void vcpu_vtlb_free(struct vcpu_t *vcpu); -bool handle_vtlb(struct vcpu_t *vcpu); +bool handle_vtlb(struct vcpu_t *vcpu, hax_vaddr_t addr); -uint vcpu_translate(struct vcpu_t *vcpu, hax_vaddr_t va, uint access, hax_paddr_t *pa, - uint64_t *len, bool update); +uint vcpu_translate(struct vcpu_t *vcpu, hax_vaddr_t va, uint access, + hax_paddr_t *pa, uint64_t *len, bool update); uint32_t vcpu_read_guest_virtual(struct vcpu_t *vcpu, hax_vaddr_t addr, void *dst, uint32_t dst_buflen, uint32_t size, uint flag); diff --git a/core/intr_exc.c b/core/intr_exc.c index 32a85b80..48ea9448 100644 --- a/core/intr_exc.c +++ b/core/intr_exc.c @@ -126,7 +126,7 @@ uint hax_intr_is_blocked(struct vcpu_t *vcpu) if (!(rflags & EFLAGS_IF)) return 1; - intr_status = vmx(vcpu, interruptibility_state).raw; + intr_status = vmcs_read(vcpu, GUEST_INTERRUPTIBILITY); if (intr_status & 3) return 1; return 0; diff --git a/core/vcpu.c b/core/vcpu.c index f08e4efc..25240273 100644 --- a/core/vcpu.c +++ b/core/vcpu.c @@ -107,7 +107,7 @@ static void handle_machine_check(struct vcpu_t *vcpu); static void handle_cpuid_virtual(struct vcpu_t *vcpu, uint32_t eax, uint32_t ecx); static void handle_mem_fault(struct vcpu_t *vcpu, struct hax_tunnel *htun); -static void check_flush(struct vcpu_t *vcpu, uint32_t bits); +static void check_flush(struct vcpu_t *vcpu, int cr, uint32_t bits); static void vmwrite_efer(struct vcpu_t *vcpu); static int handle_msr_read(struct vcpu_t *vcpu, uint32_t msr, uint64_t *val); @@ -1272,8 +1272,6 @@ void vcpu_save_host_state(struct vcpu_t *vcpu) vmwrite(vcpu, HOST_IDTR_BASE, get_kernel_idtr_base()); // Handle SYSENTER/SYSEXIT MSR - vmwrite(vcpu, HOST_SYSENTER_CS, ia32_rdmsr(IA32_SYSENTER_CS)); - vmwrite(vcpu, HOST_SYSENTER_EIP, ia32_rdmsr(IA32_SYSENTER_EIP)); vmwrite(vcpu, HOST_SYSENTER_ESP, ia32_rdmsr(IA32_SYSENTER_ESP)); // LDTR is unusable from spec, do we need ldt for host? @@ -1414,6 +1412,9 @@ static void fill_common_vmcs(struct vcpu_t *vcpu) vmwrite(vcpu, HOST_GDTR_BASE, get_kernel_gdtr_base()); vmwrite(vcpu, HOST_IDTR_BASE, get_kernel_idtr_base()); + vmwrite(vcpu, HOST_SYSENTER_CS, ia32_rdmsr(IA32_SYSENTER_CS)); + vmwrite(vcpu, HOST_SYSENTER_EIP, ia32_rdmsr(IA32_SYSENTER_EIP)); + // The host RIP, used during VM-exit events, is only updated if HAXM // is reloaded. Thus never changes during the lifetime of the VCPU. vmwrite(vcpu, HOST_RIP, (mword)vmx_get_rip()); @@ -1744,7 +1745,7 @@ int vtlb_active(struct vcpu_t *vcpu) static void advance_rip(struct vcpu_t *vcpu) { struct vcpu_state_t *state = vcpu->state; - uint32_t interruptibility = vmx(vcpu, interruptibility_state).raw; + uint32_t interruptibility = vmcs_read(vcpu, GUEST_INTERRUPTIBILITY); if (interruptibility & 3u) { interruptibility &= ~3u; @@ -2320,6 +2321,8 @@ static void vcpu_init_emulator(struct vcpu_t *vcpu) static int exit_exc_nmi(struct vcpu_t *vcpu, struct hax_tunnel *htun) { struct vcpu_state_t *state = vcpu->state; + exit_qualification_t qual = { .raw = + vmcs_read(vcpu, VM_EXIT_INFO_QUALIFICATION) }; interruption_info_t exit_intr_info; exit_intr_info.raw = vmcs_read(vcpu, VM_EXIT_INFO_INTERRUPT_INFO); @@ -2333,7 +2336,7 @@ static int exit_exc_nmi(struct vcpu_t *vcpu, struct hax_tunnel *htun) } case VECTOR_PF: { if (vtlb_active(vcpu)) { - if (handle_vtlb(vcpu)) + if (handle_vtlb(vcpu, qual.address)) return HAX_RESUME; return vcpu_emulate_insn(vcpu); @@ -2358,7 +2361,7 @@ static int exit_exc_nmi(struct vcpu_t *vcpu, struct hax_tunnel *htun) case VECTOR_DB: { htun->_exit_status = HAX_EXIT_DEBUG; htun->debug.rip = vcpu_get_rip(vcpu); - htun->debug.dr6 = vmx(vcpu, exit_qualification).raw; + htun->debug.dr6 = qual.raw; htun->debug.dr7 = vmread(vcpu, GUEST_DR7); return HAX_EXIT; } @@ -2372,7 +2375,7 @@ static int exit_exc_nmi(struct vcpu_t *vcpu, struct hax_tunnel *htun) } if (exit_intr_info.vector == VECTOR_PF) { - state->_cr2 = vmx(vcpu, exit_qualification.address); + state->_cr2 = qual.address; } return HAX_RESUME; @@ -2738,8 +2741,11 @@ static int exit_hlt(struct vcpu_t *vcpu, struct hax_tunnel *htun) static int exit_invlpg(struct vcpu_t *vcpu, struct hax_tunnel *htun) { + exit_qualification_t qual = { .raw = + vmcs_read(vcpu, VM_EXIT_INFO_QUALIFICATION) }; + advance_rip(vcpu); - vcpu_invalidate_tlb_addr(vcpu, vmx(vcpu, exit_qualification).address); + vcpu_invalidate_tlb_addr(vcpu, qual.address); htun->_exit_reason = vmx(vcpu, exit_reason).basic_reason; return HAX_RESUME; } @@ -2750,9 +2756,9 @@ static int exit_rdtsc(struct vcpu_t *vcpu, struct hax_tunnel *htun) return HAX_RESUME; } -static void check_flush(struct vcpu_t *vcpu, uint32_t bits) +static void check_flush(struct vcpu_t *vcpu, int cr, uint32_t bits) { - switch (vmx(vcpu, exit_qualification).cr.creg) { + switch (cr) { case 0: { if (bits & (CR0_PE | CR0_PG)) { vcpu_invalidate_tlb(vcpu, 1); @@ -2783,18 +2789,20 @@ static int exit_cr_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) uint64_t cr_ptr; int cr; struct vcpu_state_t *state = vcpu->state; + exit_qualification_t qual = { .raw = + vmcs_read(vcpu, VM_EXIT_INFO_QUALIFICATION) }; bool is_ept_pae = false; preempt_flag flags; uint32_t vmcs_err = 0; htun->_exit_reason = vmx(vcpu, exit_reason).basic_reason; - cr = vmx(vcpu, exit_qualification).cr.creg; + cr = qual.cr.creg; cr_ptr = vcpu_read_cr(state, cr); - switch (vmx(vcpu, exit_qualification).cr.type) { + switch (qual.cr.type) { case 0: { // MOV CR <- GPR - uint64_t val = state->_regs[(vmx(vcpu, exit_qualification).cr.gpr)]; + uint64_t val = state->_regs[qual.cr.gpr]; hax_debug("cr_access W CR%d: %08llx -> %08llx\n", cr, cr_ptr, val); if (cr == 0) { @@ -2859,7 +2867,7 @@ static int exit_cr_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) hax_error("Unsupported CR%d access\n", cr); break; } - check_flush(vcpu, cr_ptr ^ val); + check_flush(vcpu, cr, cr_ptr ^ val); vcpu_write_cr(state, cr, val); if (is_ept_pae) { @@ -2899,7 +2907,7 @@ static int exit_cr_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) case 1: { // MOV CR -> GPR hax_info("cr_access R CR%d\n", cr); - state->_regs[vmx(vcpu, exit_qualification).cr.gpr] = cr_ptr; + state->_regs[qual.cr.gpr] = cr_ptr; if (cr == 8) { hax_info("Unsupported CR%d access\n", cr); break; @@ -2926,7 +2934,7 @@ static int exit_cr_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) case 3: { // LMSW hax_info("LMSW\n"); state->_cr0 = (state->_cr0 & ~0xfULL) | - (vmx(vcpu, exit_qualification).cr.lmsw_source & 0xf); + (qual.cr.lmsw_source & 0xf); if ((vmcs_err = load_vmcs(vcpu, &flags))) { hax_panic_vcpu(vcpu, "load_vmcs failed while LMSW %x\n", vmcs_err); @@ -2954,8 +2962,10 @@ static int exit_cr_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) static int exit_dr_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) { uint64_t *dr = NULL; - int dreg = vmx(vcpu, exit_qualification.dr.dreg); - int gpr_reg = vmx(vcpu, exit_qualification).dr.gpr; + exit_qualification_t qual = { .raw = + vmcs_read(vcpu, VM_EXIT_INFO_QUALIFICATION) }; + int dreg = qual.dr.dreg; + int gpr_reg = qual.dr.gpr; bool hbreak_enabled = !!(vcpu->debug_control & HAX_DEBUG_USE_HW_BP); struct vcpu_state_t *state = vcpu->state; @@ -3017,7 +3027,7 @@ static int exit_dr_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) } } - if (vmx(vcpu, exit_qualification.dr.direction)) { + if (qual.dr.direction) { // MOV DR -> GPR if (hbreak_enabled) { // HAX hardware breakpoint enabled, return dr default value @@ -3046,7 +3056,7 @@ static int exit_dr_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) return HAX_RESUME; } -static int handle_string_io(struct vcpu_t *vcpu, exit_qualification_t *qual, +static int handle_string_io(struct vcpu_t *vcpu, exit_qualification_t qual, struct hax_tunnel *htun) { struct vcpu_state_t *state = vcpu->state; @@ -3058,7 +3068,7 @@ static int handle_string_io(struct vcpu_t *vcpu, exit_qualification_t *qual, // 1 indicates string I/O (i.e. OUTS or INS) htun->io._flags = 1; - count = qual->io.rep ? state->_rcx : 1; + count = qual.io.rep ? state->_rcx : 1; elem_size = htun->io._size; total_size = count * elem_size; @@ -3086,7 +3096,7 @@ static int handle_string_io(struct vcpu_t *vcpu, exit_qualification_t *qual, // For INS (see handle_io_post()) htun->io._vaddr = start_gva; - if (qual->io.direction == HAX_IO_OUT) { + if (qual.io.direction == HAX_IO_OUT) { if (!vcpu_read_guest_virtual(vcpu, start_gva, vcpu->io_buf, IOS_MAX_BUFFER, copy_size, 0)) { hax_panic_vcpu(vcpu, "%s: vcpu_read_guest_virtual() failed," @@ -3103,13 +3113,13 @@ static int handle_string_io(struct vcpu_t *vcpu, exit_qualification_t *qual, } if (rflags & EFLAGS_DF) { - if (qual->io.direction == HAX_IO_OUT) { + if (qual.io.direction == HAX_IO_OUT) { state->_rsi -= copy_size; } else { state->_rdi -= copy_size; } } else { - if (qual->io.direction == HAX_IO_OUT) { + if (qual.io.direction == HAX_IO_OUT) { state->_rsi += copy_size; } else { state->_rdi += copy_size; @@ -3120,15 +3130,15 @@ static int handle_string_io(struct vcpu_t *vcpu, exit_qualification_t *qual, return HAX_EXIT; } -static int handle_io(struct vcpu_t *vcpu, exit_qualification_t *qual, +static int handle_io(struct vcpu_t *vcpu, exit_qualification_t qual, struct hax_tunnel *htun) { struct vcpu_state_t *state = vcpu->state; htun->io._count = 1; htun->io._flags = 0; - if (qual->io.direction == HAX_IO_OUT) { - switch (qual->io.size + 1) { + if (qual.io.direction == HAX_IO_OUT) { + switch (qual.io.size + 1) { case 1: { *((uint8_t *)vcpu->io_buf) = state->_al; break; @@ -3154,7 +3164,8 @@ static int handle_io(struct vcpu_t *vcpu, exit_qualification_t *qual, static int exit_io_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) { struct vcpu_state_t *state = vcpu->state; - exit_qualification_t *qual = &vmx(vcpu, exit_qualification); + exit_qualification_t qual = { .raw = + vmcs_read(vcpu, VM_EXIT_INFO_QUALIFICATION) }; htun->_exit_reason = vmx(vcpu, exit_reason).basic_reason; @@ -3165,14 +3176,14 @@ static int exit_io_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) htun->io._size = 0; htun->io._count = 0; - htun->io._port = qual->io.encoding ? qual->io.port : state->_dx; - htun->io._size = qual->io.size + 1; - htun->io._direction = qual->io.direction; + htun->io._port = qual.io.encoding ? qual.io.port : state->_dx; + htun->io._size = qual.io.size + 1; + htun->io._direction = qual.io.direction; hax_debug("exit_io_access port %x, size %d\n", htun->io._port, htun->io._size); - if (qual->io.string) + if (qual.io.string) return handle_string_io(vcpu, qual, htun); return handle_io(vcpu, qual, htun); @@ -3700,7 +3711,8 @@ static int exit_ept_misconfiguration(struct vcpu_t *vcpu, static int exit_ept_violation(struct vcpu_t *vcpu, struct hax_tunnel *htun) { - exit_qualification_t *qual = &vmx(vcpu, exit_qualification); + exit_qualification_t qual = { .raw = + vmcs_read(vcpu, VM_EXIT_INFO_QUALIFICATION) }; hax_paddr_t gpa; int ret = 0; #ifdef CONFIG_HAX_EPT2 @@ -3709,7 +3721,7 @@ static int exit_ept_violation(struct vcpu_t *vcpu, struct hax_tunnel *htun) htun->_exit_reason = vmx(vcpu, exit_reason).basic_reason; - if (qual->ept.gla1 == 0 && qual->ept.gla2 == 1) { + if (qual.ept.gla1 == 0 && qual.ept.gla2 == 1) { hax_panic_vcpu(vcpu, "Incorrect EPT seting\n"); dump_vmcs(vcpu); return HAX_RESUME; @@ -3719,12 +3731,12 @@ static int exit_ept_violation(struct vcpu_t *vcpu, struct hax_tunnel *htun) #ifdef CONFIG_HAX_EPT2 ret = ept_handle_access_violation(&vcpu->vm->gpa_space, &vcpu->vm->ept_tree, - *qual, gpa, &fault_gfn); + qual, gpa, &fault_gfn); if (ret == -EFAULT) { // Extract bits 5..0 from Exit Qualification. They indicate the type of // the faulting access (HAX_PAGEFAULT_ACC_R/W/X) and the types of access // allowed (HAX_PAGEFAULT_PERM_R/W/X). - htun->pagefault.flags = qual->raw & 0x3f; + htun->pagefault.flags = qual.raw & 0x3f; htun->pagefault.gpa = fault_gfn << PG_ORDER_4K; htun->pagefault.reserved1 = 0; htun->pagefault.reserved2 = 0; diff --git a/core/vtlb.c b/core/vtlb.c index 617a9c21..52b92fc0 100644 --- a/core/vtlb.c +++ b/core/vtlb.c @@ -779,12 +779,12 @@ static uint32_t vcpu_mmu_walk(struct vcpu_t *vcpu, hax_vaddr_t va, uint32_t acce return TF_OK; } -bool handle_vtlb(struct vcpu_t *vcpu) +bool handle_vtlb(struct vcpu_t *vcpu, hax_vaddr_t addr) { uint32_t access = vmcs_read(vcpu, VM_EXIT_INFO_EXCEPTION_ERROR_CODE); pagemode_t mode = vcpu_get_pagemode(vcpu); hax_paddr_t pdir = vcpu->state->_cr3 & (mode == PM_PAE ? ~0x1fULL : ~0xfffULL); - hax_vaddr_t cr2 = vmx(vcpu, exit_qualification).address; + hax_vaddr_t cr2 = addr; uint32_t ret = vtlb_handle_page_fault(vcpu, mode, pdir, cr2, access); From b6733f455e71446c66e5e6020a1ad8f0c8cf9f91 Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Fri, 23 Nov 2018 05:46:01 -0800 Subject: [PATCH 12/13] Optimization: Cached CR reads Signed-off-by: Alexandro Sanchez Bach --- core/cpu.c | 25 ---------------- core/include/vcpu.h | 3 ++ core/include/vmx.h | 10 +++---- core/page_walker.c | 6 ++-- core/vcpu.c | 70 ++++++++++++++++++++++++--------------------- core/vmx.c | 34 ++++++++++++++++++++++ core/vtlb.c | 7 +++-- 7 files changed, 87 insertions(+), 68 deletions(-) diff --git a/core/cpu.c b/core/cpu.c index aff5e505..8e91d23e 100644 --- a/core/cpu.c +++ b/core/cpu.c @@ -186,29 +186,6 @@ void cpu_pmu_init(void *arg) pmu_info->cpuid_edx = cpuid_args.edx; } -static void vmread_cr(struct vcpu_t *vcpu) -{ - struct vcpu_state_t *state = vcpu->state; - mword cr4, cr4_mask; - - // Update only the bits the guest is allowed to change - // This must use the actual cr0 mask, not _cr0_mask. - mword cr0 = vmread(vcpu, GUEST_CR0); - mword cr0_mask = vmread(vcpu, VMX_CR0_MASK); // should cache this - hax_debug("vmread_cr cr0 %lx, cr0_mask %lx, state->_cr0 %llx\n", cr0, - cr0_mask, state->_cr0); - state->_cr0 = (cr0 & ~cr0_mask) | (state->_cr0 & cr0_mask); - hax_debug("vmread_cr, state->_cr0 %llx\n", state->_cr0); - - // update CR3 only if guest is allowed to change it - if (!(vmx(vcpu, pcpu_ctls) & CR3_LOAD_EXITING)) - state->_cr3 = vmread(vcpu, GUEST_CR3); - - cr4 = vmread(vcpu, GUEST_CR4); - cr4_mask = vmread(vcpu, VMX_CR4_MASK); // should cache this - state->_cr4 = (cr4 & ~cr4_mask) | (state->_cr4 & cr4_mask); -} - vmx_result_t cpu_vmx_vmptrld(struct per_cpu_data *cpu_data, hax_paddr_t vmcs, struct vcpu_t *vcpu) { @@ -376,8 +353,6 @@ int cpu_vmx_execute(struct vcpu_t *vcpu, struct hax_tunnel *htun) */ hax_handle_idt_vectoring(vcpu); - vmread_cr(vcpu); - if (vcpu->nr_pending_intrs > 0 || hax_intr_is_blocked(vcpu)) htun->ready_for_interrupt_injection = 0; else diff --git a/core/include/vcpu.h b/core/include/vcpu.h index af0777ce..9d77763a 100644 --- a/core/include/vcpu.h +++ b/core/include/vcpu.h @@ -256,6 +256,9 @@ void hax_panic_vcpu(struct vcpu_t *v, char *fmt, ...); // Extension-specific operations +mword vcpu_get_cr0(struct vcpu_t *vcpu); +mword vcpu_get_cr3(struct vcpu_t *vcpu); +mword vcpu_get_cr4(struct vcpu_t *vcpu); mword vcpu_get_rflags(struct vcpu_t *vcpu); mword vcpu_get_rsp(struct vcpu_t *vcpu); mword vcpu_get_rip(struct vcpu_t *vcpu); diff --git a/core/include/vmx.h b/core/include/vmx.h index 394351c6..8daf3725 100644 --- a/core/include/vmx.h +++ b/core/include/vmx.h @@ -315,8 +315,8 @@ typedef enum component_index_t component_index_t; COMP(0, 0, W_64, GUEST_PDPTE2) \ COMP(0, 0, W_64, GUEST_PDPTE3) \ /* Natural-width components */ \ - COMP(0, 0, W_UL, VMX_CR0_MASK) \ - COMP(0, 0, W_UL, VMX_CR4_MASK) \ + COMP(1, 0, W_UL, VMX_CR0_MASK) \ + COMP(1, 0, W_UL, VMX_CR4_MASK) \ COMP(0, 0, W_UL, VMX_CR0_READ_SHADOW) \ COMP(0, 0, W_UL, VMX_CR4_READ_SHADOW) \ COMP(0, 0, W_UL, VMX_CR3_TARGET_VAL_BASE) \ @@ -341,9 +341,9 @@ typedef enum component_index_t component_index_t; COMP(1, 0, W_UL, GUEST_RIP) \ COMP(1, 0, W_UL, GUEST_RFLAGS) \ COMP(1, 0, W_UL, GUEST_RSP) \ - COMP(0, 0, W_UL, GUEST_CR0) \ - COMP(0, 0, W_UL, GUEST_CR3) \ - COMP(0, 0, W_UL, GUEST_CR4) \ + COMP(1, 0, W_UL, GUEST_CR0) \ + COMP(1, 0, W_UL, GUEST_CR3) \ + COMP(1, 0, W_UL, GUEST_CR4) \ COMP(1, 0, W_UL, GUEST_ES_BASE) \ COMP(1, 0, W_UL, GUEST_CS_BASE) \ COMP(1, 0, W_UL, GUEST_SS_BASE) \ diff --git a/core/page_walker.c b/core/page_walker.c index 68deeb8f..a9e80de6 100644 --- a/core/page_walker.c +++ b/core/page_walker.c @@ -569,9 +569,9 @@ uint32_t pw_perform_page_walk( uint64_t efer_value = vcpu->state->_efer; bool is_nxe = ((efer_value & IA32_EFER_XD) != 0); bool is_lme = ((efer_value & IA32_EFER_LME) != 0); - uint64_t cr0 = vcpu->state->_cr0; - uint64_t cr3 = vcpu->state->_cr3; - uint64_t cr4 = vcpu->state->_cr4; + uint64_t cr0 = vcpu_get_cr0(vcpu); + uint64_t cr3 = vcpu_get_cr3(vcpu); + uint64_t cr4 = vcpu_get_cr4(vcpu); bool is_wp = ((cr0 & CR0_WP) != 0); bool is_pae = ((cr4 & CR4_PAE) != 0); bool is_pse = ((cr4 & CR4_PSE) != 0); diff --git a/core/vcpu.c b/core/vcpu.c index 25240273..f11303b4 100644 --- a/core/vcpu.c +++ b/core/vcpu.c @@ -172,35 +172,28 @@ static inline void set_idt(struct vcpu_state_t *state, uint64_t base, state->_idt.limit = limit; } -static uint64_t vcpu_read_cr(struct vcpu_state_t *state, uint32_t n) +static uint64_t vcpu_read_cr(struct vcpu_t *vcpu, uint32_t n) { uint64_t val = 0; switch (n) { - case 0: { - val = state->_cr0; - break; - } - case 2: { - val = state->_cr2; - break; - } - case 3: { - val = state->_cr3; - break; - } - case 4: { - val = state->_cr4; - break; - } - default: { - hax_error("Unsupported CR%d access\n", n); - break; - } + case 0: + val = vcpu_get_cr0(vcpu); + break; + case 2: + val = vcpu->state->_cr2; + break; + case 3: + val = vcpu_get_cr3(vcpu); + break; + case 4: + val = vcpu_get_cr4(vcpu); + break; + default: + hax_error("Unsupported CR%d access\n", n); + break; } - hax_debug("vcpu_read_cr cr %x val %llx\n", n, val); - return val; } @@ -1665,7 +1658,7 @@ int vcpu_execute(struct vcpu_t *vcpu) hax_mutex_lock(vcpu->tmutex); hax_debug("vcpu begin to run....\n"); // QEMU will do realmode stuff for us - if (!hax->ug_enable_flag && !(vcpu->state->_cr0 & CR0_PE)) { + if (!hax->ug_enable_flag && !(vcpu_get_cr0(vcpu) & CR0_PE)) { htun->_exit_reason = 0; htun->_exit_status = HAX_EXIT_REALMODE; hax_debug("Guest is in realmode.\n"); @@ -1723,14 +1716,14 @@ int vcpu_vmexit_handler(struct vcpu_t *vcpu, exit_reason_t exit_reason, int vtlb_active(struct vcpu_t *vcpu) { - struct vcpu_state_t *state = vcpu->state; struct per_cpu_data *cpu_data = current_cpu_data(); + uint64_t cr0 = vcpu_get_cr0(vcpu); if (hax->ug_enable_flag) return 0; - hax_debug("vtlb active: cr0, %llx\n", state->_cr0); - if ((state->_cr0 & CR0_PG) == 0) + hax_debug("vtlb active: cr0, %llx\n", cr0); + if ((cr0 & CR0_PG) == 0) return 1; if (config.disable_ept) @@ -2099,6 +2092,7 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) em_context_t *em_ctxt = &vcpu->emulate_ctxt; uint8_t instr[INSTR_MAX_LEN] = {0}; uint32_t exit_instr_length = vmcs_read(vcpu, VM_EXIT_INFO_INSTRUCTION_LENGTH); + uint64_t cr0 = vcpu_get_cr0(vcpu); uint64_t rip = vcpu_get_rip(vcpu); segment_desc_t cs; uint64_t va; @@ -2109,7 +2103,7 @@ static int vcpu_emulate_insn(struct vcpu_t *vcpu) // Detect guest mode cs.ar = vcpu_get_seg_ar(vcpu, SEG_CS); - if (!(vcpu->state->_cr0 & CR0_PE)) + if (!(cr0 & CR0_PE)) mode = EM_MODE_REAL; else if (cs.long_mode == 1) mode = EM_MODE_PROT64; @@ -2794,11 +2788,14 @@ static int exit_cr_access(struct vcpu_t *vcpu, struct hax_tunnel *htun) bool is_ept_pae = false; preempt_flag flags; uint32_t vmcs_err = 0; + state->_cr0 = vcpu_get_cr0(vcpu); + state->_cr3 = vcpu_get_cr3(vcpu); + state->_cr4 = vcpu_get_cr4(vcpu); htun->_exit_reason = vmx(vcpu, exit_reason).basic_reason; cr = qual.cr.creg; - cr_ptr = vcpu_read_cr(state, cr); + cr_ptr = vcpu_read_cr(vcpu, cr); switch (qual.cr.type) { case 0: { // MOV CR <- GPR @@ -3259,6 +3256,7 @@ static int handle_msr_read(struct vcpu_t *vcpu, uint32_t msr, uint64_t *val) int index, r = 0; struct vcpu_state_t *state = vcpu->state; struct gstate *gstate = &vcpu->gstate; + uint64_t cr0, cr4; switch (msr) { case IA32_TSC: { @@ -3278,7 +3276,9 @@ static int handle_msr_read(struct vcpu_t *vcpu, uint32_t msr, uint64_t *val) break; } case IA32_EFER: { - if (!(state->_cr4 & CR4_PAE) && (state->_cr0 & CR0_PG)) { + cr0 = vcpu_get_cr0(vcpu); + cr4 = vcpu_get_cr4(vcpu); + if (!(cr4 & CR4_PAE) && (cr0 & CR0_PG)) { r = 1; } else { *val = state->_efer; @@ -3462,8 +3462,9 @@ static int handle_msr_read(struct vcpu_t *vcpu, uint32_t msr, uint64_t *val) static void vmwrite_efer(struct vcpu_t *vcpu) { struct vcpu_state_t *state = vcpu->state; + uint64_t cr0 = vcpu_get_cr0(vcpu); - if ((state->_cr0 & CR0_PG) && (state->_efer & IA32_EFER_LME)) { + if ((state->_efer & IA32_EFER_LME) && (cr0 & CR0_PG)) { state->_efer |= IA32_EFER_LMA; vmwrite(vcpu, VMX_ENTRY_CONTROLS, vmread(vcpu, VMX_ENTRY_CONTROLS) | @@ -3515,6 +3516,7 @@ static int handle_msr_write(struct vcpu_t *vcpu, uint32_t msr, uint64_t val) int index, r = 0; struct vcpu_state_t *state = vcpu->state; struct gstate *gstate = &vcpu->gstate; + uint64_t cr0, cr4; switch (msr) { case IA32_TSC: { @@ -3539,10 +3541,12 @@ static int handle_msr_write(struct vcpu_t *vcpu, uint32_t msr, uint64_t val) break; } case IA32_EFER: { + cr0 = vcpu_get_cr0(vcpu); + cr4 = vcpu_get_cr4(vcpu); hax_info("Guest writing to EFER[%u]: 0x%x -> 0x%llx, _cr0=0x%llx," " _cr4=0x%llx\n", vcpu->vcpu_id, state->_efer, val, - state->_cr0, state->_cr4); - if ((state->_cr0 & CR0_PG) && !(state->_cr4 & CR4_PAE)) { + cr0, cr4); + if ((cr0 & CR0_PG) && !(cr4 & CR4_PAE)) { state->_efer = 0; } else { state->_efer = val; diff --git a/core/vmx.c b/core/vmx.c index eac0576f..6c54e0ab 100644 --- a/core/vmx.c +++ b/core/vmx.c @@ -327,6 +327,40 @@ void vcpu_vmcs_flush_cache_w(struct vcpu_t *vcpu) vcpu->vmx.vmcs_cache_w.dirty = 0; } +mword vcpu_get_cr0(struct vcpu_t *vcpu) +{ + struct vcpu_state_t *state = vcpu->state; + mword cr0, cr0_mask; + + // Update only the bits the guest is allowed to change + // This must use the actual cr0 mask, not _cr0_mask. + cr0 = vmcs_read(vcpu, GUEST_CR0); + cr0_mask = vmcs_read(vcpu, VMX_CR0_MASK); // should cache this + state->_cr0 = (cr0 & ~cr0_mask) | (state->_cr0 & cr0_mask); + return state->_cr0; +} + +mword vcpu_get_cr3(struct vcpu_t *vcpu) +{ + struct vcpu_state_t *state = vcpu->state; + + // update CR3 only if guest is allowed to change it + if (!(vmx(vcpu, pcpu_ctls) & CR3_LOAD_EXITING)) + state->_cr3 = vmread(vcpu, GUEST_CR3); + return state->_cr3; +} + +mword vcpu_get_cr4(struct vcpu_t *vcpu) +{ + struct vcpu_state_t *state = vcpu->state; + mword cr4, cr4_mask; + + cr4 = vmread(vcpu, GUEST_CR4); + cr4_mask = vmread(vcpu, VMX_CR4_MASK); // should cache this + state->_cr4 = (cr4 & ~cr4_mask) | (state->_cr4 & cr4_mask); + return state->_cr4; +} + mword vcpu_get_rflags(struct vcpu_t *vcpu) { return vmcs_read(vcpu, GUEST_RFLAGS); diff --git a/core/vtlb.c b/core/vtlb.c index 52b92fc0..3585393c 100644 --- a/core/vtlb.c +++ b/core/vtlb.c @@ -1162,10 +1162,13 @@ uint vcpu_translate(struct vcpu_t *vcpu, hax_vaddr_t va, uint access, hax_paddr_ pagemode_t vcpu_get_pagemode(struct vcpu_t *vcpu) { - if (!(vcpu->state->_cr0 & CR0_PG)) + uint64_t cr0 = vcpu_get_cr0(vcpu); + uint64_t cr4 = vcpu_get_cr4(vcpu); + + if (!(cr0 & CR0_PG)) return PM_FLAT; - if (!(vcpu->state->_cr4 & CR4_PAE)) + if (!(cr4 & CR4_PAE)) return PM_2LVL; // Only support pure 32-bit paging. May support PAE paging in future. From b00b798f2ef8a65efe5295b9c8ace6ee4ff36ccd Mon Sep 17 00:00:00 2001 From: Alexandro Sanchez Bach Date: Fri, 23 Nov 2018 06:25:27 -0800 Subject: [PATCH 13/13] Optimization: Moved tunnel updates outside guest-execution loop Signed-off-by: Alexandro Sanchez Bach --- core/cpu.c | 5 ----- core/vcpu.c | 10 +++++++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/core/cpu.c b/core/cpu.c index 8e91d23e..3799a7eb 100644 --- a/core/cpu.c +++ b/core/cpu.c @@ -353,11 +353,6 @@ int cpu_vmx_execute(struct vcpu_t *vcpu, struct hax_tunnel *htun) */ hax_handle_idt_vectoring(vcpu); - if (vcpu->nr_pending_intrs > 0 || hax_intr_is_blocked(vcpu)) - htun->ready_for_interrupt_injection = 0; - else - htun->ready_for_interrupt_injection = 1; - vcpu->cur_state = GS_STALE; vmcs_err = put_vmcs(vcpu, &flags); if (vmcs_err) { diff --git a/core/vcpu.c b/core/vcpu.c index f11303b4..78a6de72 100644 --- a/core/vcpu.c +++ b/core/vcpu.c @@ -1685,6 +1685,15 @@ int vcpu_execute(struct vcpu_t *vcpu) } err = cpu_vmx_execute(vcpu, htun); vcpu_is_panic(vcpu); + + if (vcpu->nr_pending_intrs > 0 || hax_intr_is_blocked(vcpu)) + htun->ready_for_interrupt_injection = 0; + else + htun->ready_for_interrupt_injection = 1; + + if (htun->_exit_status == HAX_EXIT_HLT) + htun->ready_for_interrupt_injection = 1; + out: if (err) { vcpu->cur_state = GS_STALE; @@ -2729,7 +2738,6 @@ static int exit_hlt(struct vcpu_t *vcpu, struct hax_tunnel *htun) if (hax_valid_vector(vector)) return HAX_RESUME; - htun->ready_for_interrupt_injection = 1; return HAX_EXIT; }