mirrors/wolfssl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

linuxkm: add WOLFSSL_COMMERCIAL_LICENSE alternative to kernel_fpu_begin()/kernel_fpu_end() in save_vector_registers_x86()/restore_vector_registers_x86(): allocate wc_linuxkm_fpu_savebufs as a buffer for os_xsave()/os_xrstor(), and use fpregs_lock()/fpregs_unlock() to inhibit softirqs/preemption.

Browse Source
This commit is contained in:
Daniel Pouzzner 2024-01-05 23:21:24 -06:00
parent d5d476a3a1
commit 9c36bb9073
3 changed files with 157 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
linuxkm/linuxkm_memory.c
View File

@ -80,6 +80,25 @@ struct wc_thread_fpu_count_ent {
unsigned int fpu_state;
};
struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_states = NULL;
#ifdef WOLFSSL_COMMERCIAL_LICENSE
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
#pragma GCC diagnostic ignored "-Wnested-externs"
/* avoid dependence on "alternatives_patched" and "xfd_validate_state()". */
#undef CONFIG_X86_DEBUG_FPU
#include "../kernel/fpu/internal.h"
#include "../kernel/fpu/xstate.h"
#pragma GCC diagnostic pop
static union wc_linuxkm_fpu_savebuf {
byte buf[1024]; /* must be 64-byte-aligned */
struct fpstate fpstate;
} *wc_linuxkm_fpu_savebufs = NULL;
#endif /* WOLFSSL_COMMERCIAL_LICENSE */
#define WC_FPU_COUNT_MASK 0x7fffffffU
#define WC_FPU_SAVED_MASK 0x80000000U
@ -111,7 +130,37 @@ WARN_UNUSED_RESULT int allocate_wolfcrypt_linuxkm_fpu_states(void)
return MEMORY_E;
}
memset(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked * sizeof(wc_linuxkm_fpu_states[0]));
memset(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked
* sizeof(wc_linuxkm_fpu_states[0]));
#ifdef WOLFSSL_COMMERCIAL_LICENSE
wc_linuxkm_fpu_savebufs = (union wc_linuxkm_fpu_savebuf *)malloc(
wc_linuxkm_fpu_states_n_tracked * sizeof(*wc_linuxkm_fpu_savebufs));
if (! wc_linuxkm_fpu_savebufs) {
pr_err("allocation of %lu bytes for "
"wc_linuxkm_fpu_savebufs failed.\n",
WC_LINUXKM_ROUND_UP_P_OF_2(wc_linuxkm_fpu_states_n_tracked)
* sizeof(*wc_linuxkm_fpu_savebufs));
free(wc_linuxkm_fpu_states);
wc_linuxkm_fpu_states = NULL;
return MEMORY_E;
}
if ((uintptr_t)wc_linuxkm_fpu_savebufs
& (WC_LINUXKM_ROUND_UP_P_OF_2(sizeof(*wc_linuxkm_fpu_savebufs)) - 1))
{
pr_err("allocation of %lu bytes for "
"wc_linuxkm_fpu_savebufs allocated with wrong alignment 0x%lx.\n",
WC_LINUXKM_ROUND_UP_P_OF_2(wc_linuxkm_fpu_states_n_tracked)
* sizeof(*wc_linuxkm_fpu_savebufs),
(uintptr_t)wc_linuxkm_fpu_savebufs);
free(wc_linuxkm_fpu_savebufs);
wc_linuxkm_fpu_savebufs = NULL;
free(wc_linuxkm_fpu_states);
wc_linuxkm_fpu_states = NULL;
return MEMORY_E;
}
#endif
return 0;
}
@ -141,11 +190,17 @@ void free_wolfcrypt_linuxkm_fpu_states(void) {
}
}
#ifdef WOLFSSL_COMMERCIAL_LICENSE
free(wc_linuxkm_fpu_savebufs);
wc_linuxkm_fpu_savebufs = NULL;
#endif
free(wc_linuxkm_fpu_states);
wc_linuxkm_fpu_states = NULL;
}
/* lock-(mostly)-free thread-local storage facility for tracking recursive fpu pushing/popping */
/* lock-(mostly)-free thread-local storage facility for tracking recursive fpu
* pushing/popping
*/
static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p) {
struct wc_thread_fpu_count_ent *i, *i_endptr, *i_empty;
pid_t my_pid = task_pid_nr(current), i_pid;
@ -194,7 +249,16 @@ static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p)
}
}
static void wc_linuxkm_fpu_state_free(struct wc_thread_fpu_count_ent *ent) {
#ifdef WOLFSSL_COMMERCIAL_LICENSE
static struct fpstate *wc_linuxkm_fpstate_buf_from_fpu_state(
struct wc_thread_fpu_count_ent *state)
{
size_t i = (size_t)(state - wc_linuxkm_fpu_states) / sizeof(*state);
return &wc_linuxkm_fpu_savebufs[i].fpstate;
}
#endif
static void wc_linuxkm_fpu_state_release(struct wc_thread_fpu_count_ent *ent) {
if (ent->fpu_state != 0) {
static int warned_nonzero_fpu_state = 0;
if (! warned_nonzero_fpu_state) {
@ -228,25 +292,39 @@ WARN_UNUSED_RESULT int save_vector_registers_x86(void)
}
if (irq_fpu_usable()) {
#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
/* inhibit migration, which gums up the algorithm in kernel_fpu_{begin,end}(). */
#ifdef WOLFSSL_COMMERCIAL_LICENSE
struct fpstate *fpstate = wc_linuxkm_fpstate_buf_from_fpu_state(pstate);
fpregs_lock();
fpstate->xfeatures = ~0UL;
os_xsave(fpstate);
#else /* !WOLFSSL_COMMERCIAL_LICENSE */
#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
(LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
/* inhibit migration, which gums up the algorithm in
* kernel_fpu_{begin,end}().
*/
migrate_disable();
#endif
kernel_fpu_begin();
pstate->fpu_state = 1U; /* set msb 0 to trigger kernel_fpu_end() at cleanup. */
#endif /* !WOLFSSL_COMMERCIAL_LICENSE */
/* set msb 0 to trigger kernel_fpu_end() at cleanup. */
pstate->fpu_state = 1U;
} else if (in_nmi() || (hardirq_count() > 0) || (softirq_count() > 0)) {
static int warned_fpu_forbidden = 0;
if (! warned_fpu_forbidden)
pr_err("save_vector_registers_x86 called from IRQ handler.\n");
wc_linuxkm_fpu_state_free(pstate);
wc_linuxkm_fpu_state_release(pstate);
return EPERM;
} else {
#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
(LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
!defined(WOLFSSL_COMMERCIAL_LICENSE)
migrate_disable();
#endif
/* assume already safely in_kernel_fpu. */
/* set msb 1 to inhibit kernel_fpu_end() at cleanup. */
pstate->fpu_state =
WC_FPU_SAVED_MASK + 1U; /* set msb 1 to inhibit kernel_fpu_end() at cleanup. */
WC_FPU_SAVED_MASK + 1U;
}
return 0;
@ -265,15 +343,23 @@ void restore_vector_registers_x86(void)
return;
}
if (pstate->fpu_state == 0U)
if (pstate->fpu_state == 0U) {
#ifdef WOLFSSL_COMMERCIAL_LICENSE
struct fpstate *fpstate = wc_linuxkm_fpstate_buf_from_fpu_state(pstate);
os_xrstor(fpstate, fpstate->xfeatures);
fpregs_unlock();
#else
kernel_fpu_end();
else
#endif
} else
pstate->fpu_state = 0U;
#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
(LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
!defined(WOLFSSL_COMMERCIAL_LICENSE)
migrate_enable();
#endif
wc_linuxkm_fpu_state_free(pstate);
wc_linuxkm_fpu_state_release(pstate);
return;
}

44
linuxkm/linuxkm_wc_port.h
View File

@ -333,7 +333,7 @@
#endif
typeof(nr_cpu_ids) *nr_cpu_ids;
#if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
#if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && !defined(WOLFSSL_COMMERCIAL_LICENSE)
/* note the current and needed version of these were added in af449901b8 (2020-Sep-17) */
typeof(migrate_disable) *migrate_disable;
typeof(migrate_enable) *migrate_enable;
@ -341,15 +341,20 @@
#ifdef CONFIG_X86
typeof(irq_fpu_usable) *irq_fpu_usable;
/* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289,
* released in kernel 5.11, backported to 5.4.93
*/
#ifdef kernel_fpu_begin
typeof(kernel_fpu_begin_mask) *kernel_fpu_begin_mask;
#else
typeof(kernel_fpu_begin) *kernel_fpu_begin;
#endif
typeof(kernel_fpu_end) *kernel_fpu_end;
#ifdef WOLFSSL_COMMERCIAL_LICENSE
typeof(fpregs_lock) *fpregs_lock;
typeof(fpregs_lock) *fpregs_unlock;
#else /* !WOLFSSL_COMMERCIAL_LICENSE */
/* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289,
* released in kernel 5.11, backported to 5.4.93
*/
#ifdef kernel_fpu_begin
typeof(kernel_fpu_begin_mask) *kernel_fpu_begin_mask;
#else
typeof(kernel_fpu_begin) *kernel_fpu_begin;
#endif
typeof(kernel_fpu_end) *kernel_fpu_end;
#endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
#else /* !CONFIG_X86 */
#error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture.
#endif /* arch */
@ -472,19 +477,24 @@
#endif
#define nr_cpu_ids (*(wolfssl_linuxkm_get_pie_redirect_table()->nr_cpu_ids))
#if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
#if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && !defined(WOLFSSL_COMMERCIAL_LICENSE)
#define migrate_disable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_disable))
#define migrate_enable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_enable))
#endif
#ifdef CONFIG_X86
#define irq_fpu_usable (wolfssl_linuxkm_get_pie_redirect_table()->irq_fpu_usable)
#ifdef kernel_fpu_begin
#define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask)
#else
#define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin)
#endif
#define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end)
#ifdef WOLFSSL_COMMERCIAL_LICENSE
#define fpregs_lock() (wolfssl_linuxkm_get_pie_redirect_table()->fpregs_lock())
#define fpregs_unlock() (wolfssl_linuxkm_get_pie_redirect_table()->fpregs_unlock())
#else /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
#ifdef kernel_fpu_begin
#define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask)
#else
#define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin)
#endif
#define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end)
#endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
#else /* !CONFIG_X86 */
#error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture.
#endif /* archs */

40
linuxkm/module_hooks.c
View File

@ -20,8 +20,12 @@
*/
#ifndef WOLFSSL_LICENSE
#ifdef WOLFSSL_COMMERCIAL_LICENSE
#define WOLFSSL_LICENSE "wolfSSL Commercial"
#else
#define WOLFSSL_LICENSE "GPL v2"
#endif
#endif
#define FIPS_NO_WRAPPERS
@ -346,6 +350,17 @@ static int my_preempt_count(void) {
return preempt_count();
}
/* ditto for fpregs_lock/fpregs_unlock */
#ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
static void my_fpregs_lock(void) {
fpregs_lock();
}
static void my_fpregs_unlock(void) {
fpregs_unlock();
}
#endif
static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
memset(
&wolfssl_linuxkm_pie_redirect_table,
@ -442,21 +457,28 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
#endif
wolfssl_linuxkm_pie_redirect_table.nr_cpu_ids = &nr_cpu_ids;
#if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
#if defined(CONFIG_SMP) && \
(LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
!defined(WOLFSSL_COMMERCIAL_LICENSE)
wolfssl_linuxkm_pie_redirect_table.migrate_disable = &migrate_disable;
wolfssl_linuxkm_pie_redirect_table.migrate_enable = &migrate_enable;
#endif
#ifdef WOLFSSL_LINUXKM_SIMD_X86
wolfssl_linuxkm_pie_redirect_table.irq_fpu_usable = irq_fpu_usable;
#ifdef kernel_fpu_begin
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin_mask =
kernel_fpu_begin_mask;
#else
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin =
kernel_fpu_begin;
#endif
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_end = kernel_fpu_end;
#ifdef WOLFSSL_COMMERCIAL_LICENSE
wolfssl_linuxkm_pie_redirect_table.fpregs_lock = my_fpregs_lock;
wolfssl_linuxkm_pie_redirect_table.fpregs_unlock = my_fpregs_unlock;
#else /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
#ifdef kernel_fpu_begin
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin_mask =
kernel_fpu_begin_mask;
#else
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin =
kernel_fpu_begin;
#endif
wolfssl_linuxkm_pie_redirect_table.kernel_fpu_end = kernel_fpu_end;
#endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
#endif /* WOLFSSL_LINUXKM_SIMD_X86 */
#endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */