linuxkm/lkcapi_sha_glue.c: implement mutex-free sync mechanism for wc_linuxkm_drbg_ctx in new get_drbg(), get_drbg_n(), and put_drbg();

linuxkm/x86_vector_register_glue.c: implement support for WC_FPU_INHIBITED_FLAG, and an `int inhibit_p` argument to save_vector_registers_x86();

wolfcrypt/src/random.c: implement linuxkm support for RDSEED and HAVE_ENTROPY_MEMUSE;

wolfssl/wolfcrypt/error-crypt.h and wolfcrypt/src/error.c: add WC_ACCEL_INHIBIT_E "Crypto acceleration is currently inhibited";

linuxkm/module_hooks.c and linuxkm/x86_vector_register_glue.c: remove broken and bit-rotten WOLFSSL_COMMERCIAL_LICENSE and LINUXKM_FPU_STATES_FOLLOW_THREADS code paths.

Author: douzzer

.wolfssl_known_macro_extras

@@ -294,7 +294,6 @@ LIBWOLFSSL_VERSION_GIT_ORIGIN
 LIBWOLFSSL_VERSION_GIT_SHORT_HASH
 LIBWOLFSSL_VERSION_GIT_TAG
 LINUXKM_DONT_FORCE_FIPS_ENABLED
-LINUXKM_FPU_STATES_FOLLOW_THREADS
 LINUXKM_LKCAPI_PRIORITY_ALLOW_MASKING
 LINUX_CYCLE_COUNT
 LINUX_RUSAGE_UTIME
@@ -652,7 +651,6 @@ WOLFSSL_CHECK_MEM_ZERO
 WOLFSSL_CHIBIOS
 WOLFSSL_CLANG_TIDY
 WOLFSSL_CLIENT_EXAMPLE
-WOLFSSL_COMMERCIAL_LICENSE
 WOLFSSL_CONTIKI
 WOLFSSL_CRL_ALLOW_MISSING_CDP
 WOLFSSL_CUSTOM_CONFIG

linuxkm/linuxkm_wc_port.h

@@ -367,7 +367,7 @@
         extern __must_check int allocate_wolfcrypt_linuxkm_fpu_states(void);
         extern void free_wolfcrypt_linuxkm_fpu_states(void);
         extern __must_check int can_save_vector_registers_x86(void);
-        extern __must_check int save_vector_registers_x86(void);
+        extern __must_check int save_vector_registers_x86(int inhibit_p);
         extern void restore_vector_registers_x86(void);
 
         #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
@@ -383,29 +383,36 @@
             #endif
         #endif
         #ifndef SAVE_VECTOR_REGISTERS
-            #define SAVE_VECTOR_REGISTERS(fail_clause) {    \
-                int _svr_ret = save_vector_registers_x86(); \
-                if (_svr_ret != 0) {                        \
-                    fail_clause                             \
-                }                                           \
+            #define SAVE_VECTOR_REGISTERS(fail_clause) {     \
+                int _svr_ret = save_vector_registers_x86(0); \
+                if (_svr_ret != 0) {                         \
+                    fail_clause                              \
+                }                                            \
             }
         #endif
         #ifndef SAVE_VECTOR_REGISTERS2
             #ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING
                 #define SAVE_VECTOR_REGISTERS2() ({                    \
                     int _fuzzer_ret = SAVE_VECTOR_REGISTERS2_fuzzer(); \
                     (_fuzzer_ret == 0) ?                               \
-                     save_vector_registers_x86() :                     \
+                     save_vector_registers_x86(0) :                    \
                      _fuzzer_ret;                                      \
                 })
             #else
-                #define SAVE_VECTOR_REGISTERS2() save_vector_registers_x86()
+                #define SAVE_VECTOR_REGISTERS2() save_vector_registers_x86(0)
             #endif
         #endif
         #ifndef RESTORE_VECTOR_REGISTERS
             #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_x86()
         #endif
 
+        #ifndef DISABLE_VECTOR_REGISTERS
+            #define DISABLE_VECTOR_REGISTERS() save_vector_registers_x86(1)
+        #endif
+        #ifndef REENABLE_VECTOR_REGISTERS
+            #define REENABLE_VECTOR_REGISTERS() restore_vector_registers_x86()
+        #endif
+
     #elif defined(WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS) && (defined(CONFIG_ARM) || defined(CONFIG_ARM64))
 
         #error kernel module ARM SIMD is not yet tested or usable.

linuxkm/lkcapi_sha_glue.c

@@ -941,7 +941,7 @@ struct wc_swallow_the_semicolon
 
 struct wc_linuxkm_drbg_ctx {
     struct wc_rng_inst {
-        wolfSSL_Mutex lock;
+        wolfSSL_Atomic_Int lock;
         WC_RNG rng;
     } *rngs; /* one per CPU ID */
 };
@@ -952,7 +952,6 @@ static inline void wc_linuxkm_drbg_ctx_clear(struct wc_linuxkm_drbg_ctx * ctx)
 
     if (ctx->rngs) {
         for (i = 0; i < nr_cpu_ids; ++i) {
-            (void)wc_FreeMutex(&ctx->rngs[i].lock);
             wc_FreeRng(&ctx->rngs[i].rng);
         }
         free(ctx->rngs);
@@ -974,11 +973,7 @@ static int wc_linuxkm_drbg_init_tfm(struct crypto_tfm *tfm)
     XMEMSET(ctx->rngs, 0, sizeof(*ctx->rngs) * nr_cpu_ids);
 
     for (i = 0; i < nr_cpu_ids; ++i) {
-        ret = wc_InitMutex(&ctx->rngs[i].lock);
-        if (ret != 0) {
-            ret = -EINVAL;
-            break;
-        }
+        ctx->rngs[i].lock = 0;
 
         /* Note the new DRBG instance is seeded, and later reseeded, from system
          * get_random_bytes() via wc_GenerateSeed().
@@ -1006,39 +1001,96 @@ static void wc_linuxkm_drbg_exit_tfm(struct crypto_tfm *tfm)
     return;
 }
 
+static int wc_linuxkm_drbg_default_instance_registered = 0;
+
+static inline struct wc_rng_inst *get_drbg(struct crypto_rng *tfm) {
+    struct wc_linuxkm_drbg_ctx *ctx = (struct wc_linuxkm_drbg_ctx *)crypto_rng_ctx(tfm);
+    int n;
+
+#ifdef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
+    int disabled_vector_registers = 0;
+
+    if (tfm == crypto_default_rng) {
+        /* The persistent default stdrng needs to use interrupt-safe mechanisms,
+         * because we use it as the back end for get_random_bytes().  With
+         * SMALL_STACK_CACHE, this only affects the ephemeral SHA256 instances used
+         * by the seed generator, but without SMALL_STACK_CACHE it affects every
+         * DRBG call.
+         */
+        int ret = DISABLE_VECTOR_REGISTERS();
+        if (ret != 0) {
+            pr_err("get_drbg DISABLE_VECTOR_REGISTERS returned %d", ret);
+        } else {
+            disabled_vector_registers = 1;
+        }
+    }
+#endif
+
+    n = raw_smp_processor_id();
+
+    for (;;) {
+        if (likely(__atomic_test_and_set(&ctx->rngs[n].lock,__ATOMIC_ACQUIRE))) {
+#ifdef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
+            if (disabled_vector_registers)
+                ctx->rngs[n].lock = 2;
+            else
+                ctx->rngs[n].lock = 1; /* be sure. */
+#endif
+            return &ctx->rngs[n];
+        }
+        ++n;
+        if (n >= (int)nr_cpu_ids)
+            n = 0;
+    }
+
+    __builtin_unreachable();
+}
+
+static inline struct wc_rng_inst *get_drbg_n(struct wc_linuxkm_drbg_ctx *ctx, int n) {
+    for (;;) {
+        if (likely(__atomic_test_and_set(&ctx->rngs[n].lock,__ATOMIC_ACQUIRE)))
+            return &ctx->rngs[n];
+        cond_resched();
+    }
+
+    __builtin_unreachable();
+}
+
+static inline void put_drbg(struct wc_rng_inst *drbg) {
+#ifdef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
+    if (drbg->lock == 2)
+        REENABLE_VECTOR_REGISTERS();
+#endif
+    __atomic_store_n(&(drbg->lock),0,__ATOMIC_RELEASE);
+}
+
 static int wc_linuxkm_drbg_generate(struct crypto_rng *tfm,
                         const u8 *src, unsigned int slen,
                         u8 *dst, unsigned int dlen)
 {
-    struct wc_linuxkm_drbg_ctx *ctx = (struct wc_linuxkm_drbg_ctx *)crypto_rng_ctx(tfm);
     int ret;
     /* Note, core is not locked, so the actual core ID may change while
-     * executing, hence the mutex.
-     * The mutex is also needed to coordinate with wc_linuxkm_drbg_seed(), which
+     * executing, hence the lock.
+     * The lock is also needed to coordinate with wc_linuxkm_drbg_seed(), which
      * seeds all instances.
      */
-    int my_cpu = raw_smp_processor_id();
-    wolfSSL_Mutex *lock = &ctx->rngs[my_cpu].lock;
-    WC_RNG *rng = &ctx->rngs[my_cpu].rng;
-
-    if (wc_LockMutex(lock) != 0)
-        return -EINVAL;
+    struct wc_rng_inst *drbg = get_drbg(tfm);
 
     if (slen > 0) {
-        ret = wc_RNG_DRBG_Reseed(rng, src, slen);
+        ret = wc_RNG_DRBG_Reseed(&drbg->rng, src, slen);
         if (ret != 0) {
             ret = -EINVAL;
             goto out;
         }
     }
 
-    ret = wc_RNG_GenerateBlock(rng, dst, dlen);
+    ret = wc_RNG_GenerateBlock(&drbg->rng, dst, dlen);
     if (ret != 0)
         ret = -EINVAL;
 
 out:
 
-    wc_UnLockMutex(lock);
+    put_drbg(drbg);
 
     return ret;
 }
@@ -1049,7 +1101,7 @@ static int wc_linuxkm_drbg_seed(struct crypto_rng *tfm,
     struct wc_linuxkm_drbg_ctx *ctx = (struct wc_linuxkm_drbg_ctx *)crypto_rng_ctx(tfm);
     u8 *seed_copy = NULL;
     int ret;
-    unsigned int i;
+    int n;
 
     if (slen == 0)
         return 0;
@@ -1059,25 +1111,21 @@ static int wc_linuxkm_drbg_seed(struct crypto_rng *tfm,
         return -ENOMEM;
     XMEMCPY(seed_copy + 2, seed, slen);
 
-    for (i = 0; i < nr_cpu_ids; ++i) {
-        wolfSSL_Mutex *lock = &ctx->rngs[i].lock;
-        WC_RNG *rng = &ctx->rngs[i].rng;
+    for (n = nr_cpu_ids - 1; n >= 0; --n) {
+        struct wc_rng_inst *drbg = get_drbg_n(ctx, n);
 
         /* perturb the seed with the CPU ID, so that no DRBG has the exact same
          * seed.
          */
-        seed_copy[0] = (u8)(i >> 8);
-        seed_copy[1] = (u8)i;
-
-        if (wc_LockMutex(lock) != 0)
-            return -EINVAL;
+        seed_copy[0] = (u8)(n >> 8);
+        seed_copy[1] = (u8)n;
 
-        ret = wc_RNG_DRBG_Reseed(rng, seed_copy, slen + 2);
+        ret = wc_RNG_DRBG_Reseed(&drbg->rng, seed_copy, slen + 2);
         if (ret != 0) {
             ret = -EINVAL;
         }
 
-        wc_UnLockMutex(lock);
+        put_drbg(drbg);
 
         if (ret != 0)
             break;
@@ -1103,7 +1151,6 @@ static struct rng_alg wc_linuxkm_drbg = {
     }
 };
 static int wc_linuxkm_drbg_loaded = 0;
-static int wc_linuxkm_drbg_default_instance_registered = 0;
 
 WC_MAYBE_UNUSED static int wc_linuxkm_drbg_startup(void)
 {
@@ -1222,7 +1269,18 @@ WC_MAYBE_UNUSED static int wc_linuxkm_drbg_startup(void)
         return ret;
     }
 
+    #ifdef DISABLE_VECTOR_REGISTERS
+    ret = DISABLE_VECTOR_REGISTERS();
+    if (ret != 0) {
+        pr_err("DISABLE_VECTOR_REGISTERS() returned %d", ret);
+        return -EINVAL;
+    }
+    #endif
     ret = crypto_get_default_rng();
+    #ifdef DISABLE_VECTOR_REGISTERS
+    REENABLE_VECTOR_REGISTERS();
+    #endif
+
     if (ret) {
         pr_err("crypto_get_default_rng returned %d", ret);
         return ret;

linuxkm/module_hooks.c

@@ -20,12 +20,8 @@
  */
 
 #ifndef WOLFSSL_LICENSE
-#ifdef WOLFSSL_COMMERCIAL_LICENSE
-#define WOLFSSL_LICENSE "wolfSSL Commercial"
-#else
 #define WOLFSSL_LICENSE "GPL v2"
 #endif
-#endif
 
 #define WOLFSSL_LINUXKM_NEED_LINUX_CURRENT
 
@@ -454,22 +450,6 @@ static struct task_struct *my_get_current_thread(void) {
     return get_current();
 }
 
-#if defined(WOLFSSL_LINUXKM_SIMD_X86) && defined(WOLFSSL_COMMERCIAL_LICENSE)
-
-/* ditto for fpregs_lock/fpregs_unlock */
-#ifdef WOLFSSL_LINUXKM_USE_SAVE_VECTOR_REGISTERS
-static void my_fpregs_lock(void) {
-    fpregs_lock();
-}
-
-static void my_fpregs_unlock(void) {
-    fpregs_unlock();
-}
-
-#endif /* WOLFSSL_LINUXKM_SIMD_X86 && WOLFSSL_COMMERCIAL_LICENSE */
-
-#endif /* USE_WOLFSSL_LINUXKM_PIE_REDIRECT_TABLE */
-
 static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
     memset(
         &wolfssl_linuxkm_pie_redirect_table,