From 9c36bb90730e5ab84d0085900ee79e81ef7f3de8 Mon Sep 17 00:00:00 2001 From: Daniel Pouzzner Date: Fri, 5 Jan 2024 23:21:24 -0600 Subject: [PATCH] linuxkm: add WOLFSSL_COMMERCIAL_LICENSE alternative to kernel_fpu_begin()/kernel_fpu_end() in save_vector_registers_x86()/restore_vector_registers_x86(): allocate wc_linuxkm_fpu_savebufs as a buffer for os_xsave()/os_xrstor(), and use fpregs_lock()/fpregs_unlock() to inhibit softirqs/preemption. --- linuxkm/linuxkm_memory.c | 112 +++++++++++++++++++++++++++++++++----- linuxkm/linuxkm_wc_port.h | 44 +++++++++------ linuxkm/module_hooks.c | 40 +++++++++++--- 3 files changed, 157 insertions(+), 39 deletions(-) diff --git a/linuxkm/linuxkm_memory.c b/linuxkm/linuxkm_memory.c index 31a7b93553..58131606e2 100644 --- a/linuxkm/linuxkm_memory.c +++ b/linuxkm/linuxkm_memory.c @@ -80,6 +80,25 @@ struct wc_thread_fpu_count_ent { unsigned int fpu_state; }; struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_states = NULL; + +#ifdef WOLFSSL_COMMERCIAL_LICENSE + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wnested-externs" +/* avoid dependence on "alternatives_patched" and "xfd_validate_state()". */ +#undef CONFIG_X86_DEBUG_FPU +#include "../kernel/fpu/internal.h" +#include "../kernel/fpu/xstate.h" +#pragma GCC diagnostic pop + +static union wc_linuxkm_fpu_savebuf { + byte buf[1024]; /* must be 64-byte-aligned */ + struct fpstate fpstate; +} *wc_linuxkm_fpu_savebufs = NULL; + +#endif /* WOLFSSL_COMMERCIAL_LICENSE */ + #define WC_FPU_COUNT_MASK 0x7fffffffU #define WC_FPU_SAVED_MASK 0x80000000U @@ -111,7 +130,37 @@ WARN_UNUSED_RESULT int allocate_wolfcrypt_linuxkm_fpu_states(void) return MEMORY_E; } - memset(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked * sizeof(wc_linuxkm_fpu_states[0])); + memset(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked + * sizeof(wc_linuxkm_fpu_states[0])); + +#ifdef WOLFSSL_COMMERCIAL_LICENSE + wc_linuxkm_fpu_savebufs = (union wc_linuxkm_fpu_savebuf *)malloc( + wc_linuxkm_fpu_states_n_tracked * sizeof(*wc_linuxkm_fpu_savebufs)); + if (! wc_linuxkm_fpu_savebufs) { + pr_err("allocation of %lu bytes for " + "wc_linuxkm_fpu_savebufs failed.\n", + WC_LINUXKM_ROUND_UP_P_OF_2(wc_linuxkm_fpu_states_n_tracked) + * sizeof(*wc_linuxkm_fpu_savebufs)); + free(wc_linuxkm_fpu_states); + wc_linuxkm_fpu_states = NULL; + return MEMORY_E; + } + if ((uintptr_t)wc_linuxkm_fpu_savebufs + & (WC_LINUXKM_ROUND_UP_P_OF_2(sizeof(*wc_linuxkm_fpu_savebufs)) - 1)) + { + pr_err("allocation of %lu bytes for " + "wc_linuxkm_fpu_savebufs allocated with wrong alignment 0x%lx.\n", + WC_LINUXKM_ROUND_UP_P_OF_2(wc_linuxkm_fpu_states_n_tracked) + * sizeof(*wc_linuxkm_fpu_savebufs), + (uintptr_t)wc_linuxkm_fpu_savebufs); + free(wc_linuxkm_fpu_savebufs); + wc_linuxkm_fpu_savebufs = NULL; + free(wc_linuxkm_fpu_states); + wc_linuxkm_fpu_states = NULL; + return MEMORY_E; + } + +#endif return 0; } @@ -141,11 +190,17 @@ void free_wolfcrypt_linuxkm_fpu_states(void) { } } +#ifdef WOLFSSL_COMMERCIAL_LICENSE + free(wc_linuxkm_fpu_savebufs); + wc_linuxkm_fpu_savebufs = NULL; +#endif free(wc_linuxkm_fpu_states); wc_linuxkm_fpu_states = NULL; } -/* lock-(mostly)-free thread-local storage facility for tracking recursive fpu pushing/popping */ +/* lock-(mostly)-free thread-local storage facility for tracking recursive fpu + * pushing/popping + */ static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p) { struct wc_thread_fpu_count_ent *i, *i_endptr, *i_empty; pid_t my_pid = task_pid_nr(current), i_pid; @@ -194,7 +249,16 @@ static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p) } } -static void wc_linuxkm_fpu_state_free(struct wc_thread_fpu_count_ent *ent) { +#ifdef WOLFSSL_COMMERCIAL_LICENSE +static struct fpstate *wc_linuxkm_fpstate_buf_from_fpu_state( + struct wc_thread_fpu_count_ent *state) +{ + size_t i = (size_t)(state - wc_linuxkm_fpu_states) / sizeof(*state); + return &wc_linuxkm_fpu_savebufs[i].fpstate; +} +#endif + +static void wc_linuxkm_fpu_state_release(struct wc_thread_fpu_count_ent *ent) { if (ent->fpu_state != 0) { static int warned_nonzero_fpu_state = 0; if (! warned_nonzero_fpu_state) { @@ -228,25 +292,39 @@ WARN_UNUSED_RESULT int save_vector_registers_x86(void) } if (irq_fpu_usable()) { -#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) - /* inhibit migration, which gums up the algorithm in kernel_fpu_{begin,end}(). */ +#ifdef WOLFSSL_COMMERCIAL_LICENSE + struct fpstate *fpstate = wc_linuxkm_fpstate_buf_from_fpu_state(pstate); + fpregs_lock(); + fpstate->xfeatures = ~0UL; + os_xsave(fpstate); +#else /* !WOLFSSL_COMMERCIAL_LICENSE */ +#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + /* inhibit migration, which gums up the algorithm in + * kernel_fpu_{begin,end}(). + */ migrate_disable(); #endif kernel_fpu_begin(); - pstate->fpu_state = 1U; /* set msb 0 to trigger kernel_fpu_end() at cleanup. */ +#endif /* !WOLFSSL_COMMERCIAL_LICENSE */ + /* set msb 0 to trigger kernel_fpu_end() at cleanup. */ + pstate->fpu_state = 1U; } else if (in_nmi() || (hardirq_count() > 0) || (softirq_count() > 0)) { static int warned_fpu_forbidden = 0; if (! warned_fpu_forbidden) pr_err("save_vector_registers_x86 called from IRQ handler.\n"); - wc_linuxkm_fpu_state_free(pstate); + wc_linuxkm_fpu_state_release(pstate); return EPERM; } else { -#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) +#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \ + !defined(WOLFSSL_COMMERCIAL_LICENSE) migrate_disable(); #endif /* assume already safely in_kernel_fpu. */ + /* set msb 1 to inhibit kernel_fpu_end() at cleanup. */ pstate->fpu_state = - WC_FPU_SAVED_MASK + 1U; /* set msb 1 to inhibit kernel_fpu_end() at cleanup. */ + WC_FPU_SAVED_MASK + 1U; } return 0; @@ -265,15 +343,23 @@ void restore_vector_registers_x86(void) return; } - if (pstate->fpu_state == 0U) + if (pstate->fpu_state == 0U) { +#ifdef WOLFSSL_COMMERCIAL_LICENSE + struct fpstate *fpstate = wc_linuxkm_fpstate_buf_from_fpu_state(pstate); + os_xrstor(fpstate, fpstate->xfeatures); + fpregs_unlock(); +#else kernel_fpu_end(); - else +#endif + } else pstate->fpu_state = 0U; -#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) +#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \ + !defined(WOLFSSL_COMMERCIAL_LICENSE) migrate_enable(); #endif - wc_linuxkm_fpu_state_free(pstate); + wc_linuxkm_fpu_state_release(pstate); return; } diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h index c155f52838..f815ec354f 100644 --- a/linuxkm/linuxkm_wc_port.h +++ b/linuxkm/linuxkm_wc_port.h @@ -333,7 +333,7 @@ #endif typeof(nr_cpu_ids) *nr_cpu_ids; - #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && !defined(WOLFSSL_COMMERCIAL_LICENSE) /* note the current and needed version of these were added in af449901b8 (2020-Sep-17) */ typeof(migrate_disable) *migrate_disable; typeof(migrate_enable) *migrate_enable; @@ -341,15 +341,20 @@ #ifdef CONFIG_X86 typeof(irq_fpu_usable) *irq_fpu_usable; - /* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289, - * released in kernel 5.11, backported to 5.4.93 - */ - #ifdef kernel_fpu_begin - typeof(kernel_fpu_begin_mask) *kernel_fpu_begin_mask; - #else - typeof(kernel_fpu_begin) *kernel_fpu_begin; - #endif - typeof(kernel_fpu_end) *kernel_fpu_end; + #ifdef WOLFSSL_COMMERCIAL_LICENSE + typeof(fpregs_lock) *fpregs_lock; + typeof(fpregs_lock) *fpregs_unlock; + #else /* !WOLFSSL_COMMERCIAL_LICENSE */ + /* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289, + * released in kernel 5.11, backported to 5.4.93 + */ + #ifdef kernel_fpu_begin + typeof(kernel_fpu_begin_mask) *kernel_fpu_begin_mask; + #else + typeof(kernel_fpu_begin) *kernel_fpu_begin; + #endif + typeof(kernel_fpu_end) *kernel_fpu_end; + #endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */ #else /* !CONFIG_X86 */ #error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture. #endif /* arch */ @@ -472,19 +477,24 @@ #endif #define nr_cpu_ids (*(wolfssl_linuxkm_get_pie_redirect_table()->nr_cpu_ids)) - #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && !defined(WOLFSSL_COMMERCIAL_LICENSE) #define migrate_disable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_disable)) #define migrate_enable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_enable)) #endif #ifdef CONFIG_X86 #define irq_fpu_usable (wolfssl_linuxkm_get_pie_redirect_table()->irq_fpu_usable) - #ifdef kernel_fpu_begin - #define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask) - #else - #define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin) - #endif - #define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end) + #ifdef WOLFSSL_COMMERCIAL_LICENSE + #define fpregs_lock() (wolfssl_linuxkm_get_pie_redirect_table()->fpregs_lock()) + #define fpregs_unlock() (wolfssl_linuxkm_get_pie_redirect_table()->fpregs_unlock()) + #else /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */ + #ifdef kernel_fpu_begin + #define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask) + #else + #define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin) + #endif + #define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end) + #endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */ #else /* !CONFIG_X86 */ #error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture. #endif /* archs */ diff --git a/linuxkm/module_hooks.c b/linuxkm/module_hooks.c index 5b51337421..d270edac7a 100644 --- a/linuxkm/module_hooks.c +++ b/linuxkm/module_hooks.c @@ -20,8 +20,12 @@ */ #ifndef WOLFSSL_LICENSE +#ifdef WOLFSSL_COMMERCIAL_LICENSE +#define WOLFSSL_LICENSE "wolfSSL Commercial" +#else #define WOLFSSL_LICENSE "GPL v2" #endif +#endif #define FIPS_NO_WRAPPERS @@ -346,6 +350,17 @@ static int my_preempt_count(void) { return preempt_count(); } +/* ditto for fpregs_lock/fpregs_unlock */ +#ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS +static void my_fpregs_lock(void) { + fpregs_lock(); +} + +static void my_fpregs_unlock(void) { + fpregs_unlock(); +} +#endif + static int set_up_wolfssl_linuxkm_pie_redirect_table(void) { memset( &wolfssl_linuxkm_pie_redirect_table, @@ -442,21 +457,28 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) { #endif wolfssl_linuxkm_pie_redirect_table.nr_cpu_ids = &nr_cpu_ids; - #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) + #if defined(CONFIG_SMP) && \ + (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \ + !defined(WOLFSSL_COMMERCIAL_LICENSE) wolfssl_linuxkm_pie_redirect_table.migrate_disable = &migrate_disable; wolfssl_linuxkm_pie_redirect_table.migrate_enable = &migrate_enable; #endif #ifdef WOLFSSL_LINUXKM_SIMD_X86 wolfssl_linuxkm_pie_redirect_table.irq_fpu_usable = irq_fpu_usable; - #ifdef kernel_fpu_begin - wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin_mask = - kernel_fpu_begin_mask; - #else - wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin = - kernel_fpu_begin; - #endif - wolfssl_linuxkm_pie_redirect_table.kernel_fpu_end = kernel_fpu_end; + #ifdef WOLFSSL_COMMERCIAL_LICENSE + wolfssl_linuxkm_pie_redirect_table.fpregs_lock = my_fpregs_lock; + wolfssl_linuxkm_pie_redirect_table.fpregs_unlock = my_fpregs_unlock; + #else /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */ + #ifdef kernel_fpu_begin + wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin_mask = + kernel_fpu_begin_mask; + #else + wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin = + kernel_fpu_begin; + #endif + wolfssl_linuxkm_pie_redirect_table.kernel_fpu_end = kernel_fpu_end; + #endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */ #endif /* WOLFSSL_LINUXKM_SIMD_X86 */ #endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */