address undefined shift behavior and overflow

Author: rlm2002

wolfcrypt/src/pwdbased.c

@@ -816,9 +816,16 @@ int wc_scrypt(byte* output, const byte* passwd, int passLen,
         ret = MEMORY_E;
         goto end;
     }
+
+    /* Check that (1 << cost) * bSz won't overflow or exceed allowed max */
+    if (((size_t)1 << cost) * (size_t)bSz > SCRYPT_WORD32_MAX) {
+        ret = BAD_FUNC_ARG;
+        goto end;
+    }
+
     /* Temporary for scryptROMix. */
-    v = (byte*)XMALLOC((size_t)((1U << cost) * bSz), NULL,
-                       DYNAMIC_TYPE_TMP_BUFFER);
+    v = (byte*)XMALLOC(((size_t)1 << cost) * (size_t)bSz, NULL,
+                         DYNAMIC_TYPE_TMP_BUFFER);
     if (v == NULL) {
         ret = MEMORY_E;
         goto end;
@@ -841,7 +848,8 @@ int wc_scrypt(byte* output, const byte* passwd, int passLen,
 
     /* Step 2. */
     for (i = 0; i < parallel; i++)
-        scryptROMix(blocks + i * (int)bSz, v, y, (int)blockSize, 1U << cost);
+        scryptROMix(blocks + i * (int)bSz, v, y, (int)blockSize,
+                    (word32)((size_t)1 << cost));
 
     /* Step 3. */
     ret = wc_PBKDF2(output, passwd, passLen, blocks, (int)blocksSz, 1, dkLen,