linuxkm: add WOLFSSL_COMMERCIAL_LICENSE alternative to kernel_fpu_begin()/kernel_fpu_end() in save_vector_registers_x86()/restore_vector_registers_x86(): allocate wc_linuxkm_fpu_savebufs as a buffer for os_xsave()/os_xrstor(), and use fpregs_lock()/fpregs_unlock() to inhibit softirqs/preemption.

douzzer · douzzer · commit 9c36bb90730e · 2024-01-05T23:21:24.000-06:00
diff --git a/linuxkm/linuxkm_memory.c b/linuxkm/linuxkm_memory.c
@@ -80,6 +80,25 @@ struct wc_thread_fpu_count_ent {
     unsigned int fpu_state;
 };
 struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_states = NULL;
+
+#ifdef WOLFSSL_COMMERCIAL_LICENSE
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#pragma GCC diagnostic ignored "-Wnested-externs"
+/* avoid dependence on "alternatives_patched" and "xfd_validate_state()". */
+#undef CONFIG_X86_DEBUG_FPU
+#include "../kernel/fpu/internal.h"
+#include "../kernel/fpu/xstate.h"
+#pragma GCC diagnostic pop
+
+static union wc_linuxkm_fpu_savebuf {
+    byte buf[1024]; /* must be 64-byte-aligned */
+    struct fpstate fpstate;
+} *wc_linuxkm_fpu_savebufs = NULL;
+
+#endif /* WOLFSSL_COMMERCIAL_LICENSE */
+
 #define WC_FPU_COUNT_MASK 0x7fffffffU
 #define WC_FPU_SAVED_MASK 0x80000000U
 
@@ -111,7 +130,37 @@ WARN_UNUSED_RESULT int allocate_wolfcrypt_linuxkm_fpu_states(void)
         return MEMORY_E;
     }
 
-    memset(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked * sizeof(wc_linuxkm_fpu_states[0]));
+    memset(wc_linuxkm_fpu_states, 0, wc_linuxkm_fpu_states_n_tracked
+           * sizeof(wc_linuxkm_fpu_states[0]));
+
+#ifdef WOLFSSL_COMMERCIAL_LICENSE
+    wc_linuxkm_fpu_savebufs = (union wc_linuxkm_fpu_savebuf *)malloc(
+        wc_linuxkm_fpu_states_n_tracked * sizeof(*wc_linuxkm_fpu_savebufs));
+    if (! wc_linuxkm_fpu_savebufs) {
+        pr_err("allocation of %lu bytes for "
+               "wc_linuxkm_fpu_savebufs failed.\n",
+               WC_LINUXKM_ROUND_UP_P_OF_2(wc_linuxkm_fpu_states_n_tracked)
+               * sizeof(*wc_linuxkm_fpu_savebufs));
+        free(wc_linuxkm_fpu_states);
+        wc_linuxkm_fpu_states = NULL;
+        return MEMORY_E;
+    }
+    if ((uintptr_t)wc_linuxkm_fpu_savebufs
+        & (WC_LINUXKM_ROUND_UP_P_OF_2(sizeof(*wc_linuxkm_fpu_savebufs)) - 1))
+    {
+        pr_err("allocation of %lu bytes for "
+               "wc_linuxkm_fpu_savebufs allocated with wrong alignment 0x%lx.\n",
+               WC_LINUXKM_ROUND_UP_P_OF_2(wc_linuxkm_fpu_states_n_tracked)
+               * sizeof(*wc_linuxkm_fpu_savebufs),
+               (uintptr_t)wc_linuxkm_fpu_savebufs);
+        free(wc_linuxkm_fpu_savebufs);
+        wc_linuxkm_fpu_savebufs = NULL;
+        free(wc_linuxkm_fpu_states);
+        wc_linuxkm_fpu_states = NULL;
+        return MEMORY_E;
+    }
+
+#endif
 
     return 0;
 }
@@ -141,11 +190,17 @@ void free_wolfcrypt_linuxkm_fpu_states(void) {
         }
     }
 
+#ifdef WOLFSSL_COMMERCIAL_LICENSE
+    free(wc_linuxkm_fpu_savebufs);
+    wc_linuxkm_fpu_savebufs = NULL;
+#endif
     free(wc_linuxkm_fpu_states);
     wc_linuxkm_fpu_states = NULL;
 }
 
-/* lock-(mostly)-free thread-local storage facility for tracking recursive fpu pushing/popping */
+/* lock-(mostly)-free thread-local storage facility for tracking recursive fpu
+ * pushing/popping
+ */
 static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p) {
     struct wc_thread_fpu_count_ent *i, *i_endptr, *i_empty;
     pid_t my_pid = task_pid_nr(current), i_pid;
@@ -194,7 +249,16 @@ static struct wc_thread_fpu_count_ent *wc_linuxkm_fpu_state_assoc(int create_p)
     }
 }
 
-static void wc_linuxkm_fpu_state_free(struct wc_thread_fpu_count_ent *ent) {
+#ifdef WOLFSSL_COMMERCIAL_LICENSE
+static struct fpstate *wc_linuxkm_fpstate_buf_from_fpu_state(
+    struct wc_thread_fpu_count_ent *state)
+{
+    size_t i = (size_t)(state - wc_linuxkm_fpu_states) / sizeof(*state);
+    return &wc_linuxkm_fpu_savebufs[i].fpstate;
+}
+#endif
+
+static void wc_linuxkm_fpu_state_release(struct wc_thread_fpu_count_ent *ent) {
     if (ent->fpu_state != 0) {
         static int warned_nonzero_fpu_state = 0;
         if (! warned_nonzero_fpu_state) {
@@ -228,25 +292,39 @@ WARN_UNUSED_RESULT int save_vector_registers_x86(void)
     }
 
     if (irq_fpu_usable()) {
-#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
-        /* inhibit migration, which gums up the algorithm in kernel_fpu_{begin,end}(). */
+#ifdef WOLFSSL_COMMERCIAL_LICENSE
+        struct fpstate *fpstate = wc_linuxkm_fpstate_buf_from_fpu_state(pstate);
+        fpregs_lock();
+        fpstate->xfeatures = ~0UL;
+        os_xsave(fpstate);
+#else /* !WOLFSSL_COMMERCIAL_LICENSE */
+#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
+    (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+        /* inhibit migration, which gums up the algorithm in
+         * kernel_fpu_{begin,end}().
+         */
         migrate_disable();
 #endif
         kernel_fpu_begin();
-        pstate->fpu_state = 1U; /* set msb 0 to trigger kernel_fpu_end() at cleanup. */
+#endif /* !WOLFSSL_COMMERCIAL_LICENSE */
+        /* set msb 0 to trigger kernel_fpu_end() at cleanup. */
+        pstate->fpu_state = 1U;
     } else if (in_nmi() || (hardirq_count() > 0) || (softirq_count() > 0)) {
         static int warned_fpu_forbidden = 0;
         if (! warned_fpu_forbidden)
             pr_err("save_vector_registers_x86 called from IRQ handler.\n");
-        wc_linuxkm_fpu_state_free(pstate);
+        wc_linuxkm_fpu_state_release(pstate);
         return EPERM;
     } else {
-#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
+    (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
+    !defined(WOLFSSL_COMMERCIAL_LICENSE)
         migrate_disable();
 #endif
         /* assume already safely in_kernel_fpu. */
+        /* set msb 1 to inhibit kernel_fpu_end() at cleanup. */
         pstate->fpu_state =
-            WC_FPU_SAVED_MASK + 1U; /* set msb 1 to inhibit kernel_fpu_end() at cleanup. */
+            WC_FPU_SAVED_MASK + 1U;
     }
 
     return 0;
@@ -265,15 +343,23 @@ void restore_vector_registers_x86(void)
         return;
     }
 
-    if (pstate->fpu_state == 0U)
+    if (pstate->fpu_state == 0U) {
+#ifdef WOLFSSL_COMMERCIAL_LICENSE
+        struct fpstate *fpstate = wc_linuxkm_fpstate_buf_from_fpu_state(pstate);
+        os_xrstor(fpstate, fpstate->xfeatures);
+        fpregs_unlock();
+#else
         kernel_fpu_end();
-    else
+#endif
+    } else
         pstate->fpu_state = 0U;
-#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+#if defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_COUNT) && \
+    (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
+    !defined(WOLFSSL_COMMERCIAL_LICENSE)
     migrate_enable();
 #endif
 
-    wc_linuxkm_fpu_state_free(pstate);
+    wc_linuxkm_fpu_state_release(pstate);
 
     return;
 }
diff --git a/linuxkm/linuxkm_wc_port.h b/linuxkm/linuxkm_wc_port.h
@@ -333,23 +333,28 @@
             #endif
             typeof(nr_cpu_ids) *nr_cpu_ids;
 
-            #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+            #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && !defined(WOLFSSL_COMMERCIAL_LICENSE)
                 /* note the current and needed version of these were added in af449901b8 (2020-Sep-17) */
                 typeof(migrate_disable) *migrate_disable;
                 typeof(migrate_enable) *migrate_enable;
             #endif
 
             #ifdef CONFIG_X86
                 typeof(irq_fpu_usable) *irq_fpu_usable;
-                /* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289,
-                 * released in kernel 5.11, backported to 5.4.93
-                 */
-                #ifdef kernel_fpu_begin
-                    typeof(kernel_fpu_begin_mask) *kernel_fpu_begin_mask;
-                #else
-                    typeof(kernel_fpu_begin) *kernel_fpu_begin;
-                #endif
-                typeof(kernel_fpu_end) *kernel_fpu_end;
+                #ifdef WOLFSSL_COMMERCIAL_LICENSE
+                    typeof(fpregs_lock) *fpregs_lock;
+                    typeof(fpregs_lock) *fpregs_unlock;
+                #else /* !WOLFSSL_COMMERCIAL_LICENSE */
+                    /* kernel_fpu_begin() replaced by kernel_fpu_begin_mask() in commit e4512289,
+                     * released in kernel 5.11, backported to 5.4.93
+                     */
+                    #ifdef kernel_fpu_begin
+                        typeof(kernel_fpu_begin_mask) *kernel_fpu_begin_mask;
+                    #else
+                        typeof(kernel_fpu_begin) *kernel_fpu_begin;
+                    #endif
+                    typeof(kernel_fpu_end) *kernel_fpu_end;
+                #endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
             #else /* !CONFIG_X86 */
                 #error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture.
             #endif /* arch */
@@ -472,19 +477,24 @@
         #endif
         #define nr_cpu_ids (*(wolfssl_linuxkm_get_pie_redirect_table()->nr_cpu_ids))
 
-        #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+        #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && !defined(WOLFSSL_COMMERCIAL_LICENSE)
             #define migrate_disable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_disable))
             #define migrate_enable (*(wolfssl_linuxkm_get_pie_redirect_table()->migrate_enable))
         #endif
 
         #ifdef CONFIG_X86
             #define irq_fpu_usable (wolfssl_linuxkm_get_pie_redirect_table()->irq_fpu_usable)
-            #ifdef kernel_fpu_begin
-                #define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask)
-            #else
-                #define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin)
-            #endif
-            #define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end)
+            #ifdef WOLFSSL_COMMERCIAL_LICENSE
+                #define fpregs_lock() (wolfssl_linuxkm_get_pie_redirect_table()->fpregs_lock())
+                #define fpregs_unlock() (wolfssl_linuxkm_get_pie_redirect_table()->fpregs_unlock())
+            #else /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
+                #ifdef kernel_fpu_begin
+                    #define kernel_fpu_begin_mask (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin_mask)
+                #else
+                    #define kernel_fpu_begin (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_begin)
+                #endif
+                #define kernel_fpu_end (wolfssl_linuxkm_get_pie_redirect_table()->kernel_fpu_end)
+            #endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
         #else /* !CONFIG_X86 */
             #error WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture.
         #endif /* archs */
diff --git a/linuxkm/module_hooks.c b/linuxkm/module_hooks.c
@@ -20,8 +20,12 @@
  */
 
 #ifndef WOLFSSL_LICENSE
+#ifdef WOLFSSL_COMMERCIAL_LICENSE
+#define WOLFSSL_LICENSE "wolfSSL Commercial"
+#else
 #define WOLFSSL_LICENSE "GPL v2"
 #endif
+#endif
 
 #define FIPS_NO_WRAPPERS
 
@@ -346,6 +350,17 @@ static int my_preempt_count(void) {
     return preempt_count();
 }
 
+/* ditto for fpregs_lock/fpregs_unlock */
+#ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
+static void my_fpregs_lock(void) {
+    fpregs_lock();
+}
+
+static void my_fpregs_unlock(void) {
+    fpregs_unlock();
+}
+#endif
+
 static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
     memset(
         &wolfssl_linuxkm_pie_redirect_table,
@@ -442,21 +457,28 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
     #endif
     wolfssl_linuxkm_pie_redirect_table.nr_cpu_ids = &nr_cpu_ids;
 
-    #if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+    #if defined(CONFIG_SMP) && \
+        (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)) && \
+        !defined(WOLFSSL_COMMERCIAL_LICENSE)
         wolfssl_linuxkm_pie_redirect_table.migrate_disable = &migrate_disable;
         wolfssl_linuxkm_pie_redirect_table.migrate_enable = &migrate_enable;
     #endif
 
 #ifdef WOLFSSL_LINUXKM_SIMD_X86
     wolfssl_linuxkm_pie_redirect_table.irq_fpu_usable = irq_fpu_usable;
-    #ifdef kernel_fpu_begin
-    wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin_mask =
-        kernel_fpu_begin_mask;
-    #else
-    wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin =
-        kernel_fpu_begin;
-    #endif
-    wolfssl_linuxkm_pie_redirect_table.kernel_fpu_end = kernel_fpu_end;
+    #ifdef WOLFSSL_COMMERCIAL_LICENSE
+        wolfssl_linuxkm_pie_redirect_table.fpregs_lock = my_fpregs_lock;
+        wolfssl_linuxkm_pie_redirect_table.fpregs_unlock = my_fpregs_unlock;
+    #else /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
+        #ifdef kernel_fpu_begin
+        wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin_mask =
+            kernel_fpu_begin_mask;
+        #else
+        wolfssl_linuxkm_pie_redirect_table.kernel_fpu_begin =
+            kernel_fpu_begin;
+        #endif
+        wolfssl_linuxkm_pie_redirect_table.kernel_fpu_end = kernel_fpu_end;
+    #endif /* !defined(WOLFSSL_COMMERCIAL_LICENSE) */
 #endif /* WOLFSSL_LINUXKM_SIMD_X86 */
 
 #endif /* WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS */
