diff --git a/wolfcrypt/src/memory.c b/wolfcrypt/src/memory.c index 314a3a82f..945f86fbc 100644 --- a/wolfcrypt/src/memory.c +++ b/wolfcrypt/src/memory.c @@ -1140,26 +1140,23 @@ void __attribute__((no_instrument_function)) } #endif -#ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED -union fpregs_state **wolfcrypt_irq_fpu_states = NULL; -#endif - -#if defined(WOLFSSL_LINUXKM_SIMD_X86) && defined(WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED) +#if defined(WOLFSSL_LINUXKM_SIMD_X86) + static union fpregs_state **wolfcrypt_linuxkm_fpu_states = NULL; static WARN_UNUSED_RESULT inline int am_in_hard_interrupt_handler(void) { return (preempt_count() & (NMI_MASK | HARDIRQ_MASK)) != 0; } - WARN_UNUSED_RESULT int allocate_wolfcrypt_irq_fpu_states(void) + WARN_UNUSED_RESULT int allocate_wolfcrypt_linuxkm_fpu_states(void) { - wolfcrypt_irq_fpu_states = + wolfcrypt_linuxkm_fpu_states = (union fpregs_state **)kzalloc(nr_cpu_ids * sizeof(struct fpu_state *), GFP_KERNEL); - if (! wolfcrypt_irq_fpu_states) { + if (! wolfcrypt_linuxkm_fpu_states) { pr_err("warning, allocation of %lu bytes for " - "wolfcrypt_irq_fpu_states failed.\n", + "wolfcrypt_linuxkm_fpu_states failed.\n", nr_cpu_ids * sizeof(struct fpu_state *)); return MEMORY_E; } @@ -1168,27 +1165,27 @@ union fpregs_state **wolfcrypt_irq_fpu_states = NULL; for (i=0; i 0) { - return; - } else { - #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0) - copy_kernel_to_fpregs(wolfcrypt_irq_fpu_states[processor_id]); - #else - __restore_fpregs_from_fpstate(wolfcrypt_irq_fpu_states[processor_id], - xfeatures_mask_all); - #endif - ((char *)wolfcrypt_irq_fpu_states[processor_id])[PAGE_SIZE-1] = 0; - preempt_enable(); - return; - } } - kernel_fpu_end(); + + if (((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] == 0) + { + pr_err("restore_vector_registers_x86 called for cpu id %d " + "without saved context.\n", processor_id); + return; + } + + if (--((unsigned char *)wolfcrypt_linuxkm_fpu_states[processor_id])[PAGE_SIZE-1] > 0) { + preempt_enable(); /* preempt_disable count will still be nonzero after this decrement. */ + return; + } + + if (am_in_hard_interrupt_handler()) { + #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0) + copy_kernel_to_fpregs(wolfcrypt_linuxkm_fpu_states[processor_id]); + #else + __restore_fpregs_from_fpstate(wolfcrypt_linuxkm_fpu_states[processor_id], + xfeatures_mask_all); + #endif + preempt_enable(); + } else { + kernel_fpu_end(); + } + return; } #endif /* WOLFSSL_LINUXKM_SIMD_X86 && WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */ diff --git a/wolfcrypt/src/wc_port.c b/wolfcrypt/src/wc_port.c index fbf836515..5d27ebd1e 100644 --- a/wolfcrypt/src/wc_port.c +++ b/wolfcrypt/src/wc_port.c @@ -162,11 +162,10 @@ int wolfCrypt_Init(void) } #endif - #if defined(WOLFSSL_LINUXKM_SIMD_X86) \ - && defined(WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED) - ret = allocate_wolfcrypt_irq_fpu_states(); + #if defined(WOLFSSL_LINUXKM_SIMD_X86) + ret = allocate_wolfcrypt_linuxkm_fpu_states(); if (ret != 0) { - WOLFSSL_MSG("allocate_wolfcrypt_irq_fpu_states failed"); + WOLFSSL_MSG("allocate_wolfcrypt_linuxkm_fpu_states failed"); return ret; } #endif @@ -374,9 +373,8 @@ int wolfCrypt_Cleanup(void) rpcmem_deinit(); wolfSSL_CleanupHandle(); #endif - #if defined(WOLFSSL_LINUXKM_SIMD_X86) \ - && defined(WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED) - free_wolfcrypt_irq_fpu_states(); + #if defined(WOLFSSL_LINUXKM_SIMD_X86) + free_wolfcrypt_linuxkm_fpu_states(); #endif } diff --git a/wolfssl/wolfcrypt/wc_port.h b/wolfssl/wolfcrypt/wc_port.h index b371629dc..dfd595b80 100644 --- a/wolfssl/wolfcrypt/wc_port.h +++ b/wolfssl/wolfcrypt/wc_port.h @@ -282,7 +282,6 @@ #endif typeof(kernel_fpu_end) *kernel_fpu_end; - #ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0) typeof(copy_fpregs_to_fpstate) *copy_fpregs_to_fpstate; typeof(copy_kernel_to_fpregs) *copy_kernel_to_fpregs; @@ -293,7 +292,6 @@ #endif typeof(cpu_number) *cpu_number; typeof(nr_cpu_ids) *nr_cpu_ids; - #endif /* WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */ #endif /* WOLFSSL_LINUXKM_SIMD_X86 */ @@ -399,18 +397,16 @@ #define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin) #endif #define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end) - #ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED - #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0) - #define copy_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->copy_fpregs_to_fpstate) - #define copy_kernel_to_fpregs (wolfssl_linuxkm_get_pie_redirect_table()->copy_kernel_to_fpregs) - #else - #define save_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->save_fpregs_to_fpstate) - #define __restore_fpregs_from_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->__restore_fpregs_from_fpstate) - #define xfeatures_mask_all (*(wolfssl_linuxkm_get_pie_redirect_table()->xfeatures_mask_all)) - #endif - #define cpu_number (*(wolfssl_linuxkm_get_pie_redirect_table()->cpu_number)) - #define nr_cpu_ids (*(wolfssl_linuxkm_get_pie_redirect_table()->nr_cpu_ids)) - #endif /* WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */ + #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0) + #define copy_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->copy_fpregs_to_fpstate) + #define copy_kernel_to_fpregs (wolfssl_linuxkm_get_pie_redirect_table()->copy_kernel_to_fpregs) + #else + #define save_fpregs_to_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->save_fpregs_to_fpstate) + #define __restore_fpregs_from_fpstate (wolfssl_linuxkm_get_pie_redirect_table()->__restore_fpregs_from_fpstate) + #define xfeatures_mask_all (*(wolfssl_linuxkm_get_pie_redirect_table()->xfeatures_mask_all)) + #endif + #define cpu_number (*(wolfssl_linuxkm_get_pie_redirect_table()->cpu_number)) + #define nr_cpu_ids (*(wolfssl_linuxkm_get_pie_redirect_table()->nr_cpu_ids)) #endif #define __mutex_init (wolfssl_linuxkm_get_pie_redirect_table()->__mutex_init) @@ -448,30 +444,15 @@ #ifdef WOLFSSL_LINUXKM_SIMD_X86 -#ifdef WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED - extern __must_check int allocate_wolfcrypt_irq_fpu_states(void); - extern void free_wolfcrypt_irq_fpu_states(void); + extern __must_check int allocate_wolfcrypt_linuxkm_fpu_states(void); + extern void free_wolfcrypt_linuxkm_fpu_states(void); extern __must_check int save_vector_registers_x86(void); extern void restore_vector_registers_x86(void); -#else /* !WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */ - #define save_vector_registers_x86() ({ \ - int _ret; \ - preempt_disable(); \ - if (! irq_fpu_usable()) { \ - preempt_enable(); \ - _ret = BAD_STATE_E; \ - } else { \ - kernel_fpu_begin(); \ - preempt_enable(); /* kernel_fpu_begin() does its own preempt_disable(). decrement ours. */ \ - _ret = 0; \ - } \ - _ret; \ - }) - #define restore_vector_registers_x86() kernel_fpu_end() -#endif /* !WOLFSSL_LINUXKM_SIMD_X86_IRQ_ALLOWED */ #elif defined(CONFIG_ARM) || defined(CONFIG_ARM64) + #error kernel module ARM SIMD is not yet tested or usable. + static WARN_UNUSED_RESULT inline int save_vector_registers_arm(void) { preempt_disable();