libvncserver: check before shifting in table init

As it can turn into undefined behavior like
shift exponent 32 is too large for 32-bit type 'uint32_t'

Author: catenacyber

libvncserver/tableinittctemplate.c

@@ -126,7 +126,11 @@ rfbInitOneRGBTableOUT (OUT_T *table, int inMax, int outMax, int outShift,
     int nEntries = inMax + 1;
 
     for (i = 0; i < nEntries; i++) {
-        table[i] = ((OUT_T)((i * outMax + inMax / 2) / inMax)) << outShift;
+        if (outShift < 32) {
+            table[i] = ((OUT_T)((i * outMax + inMax / 2) / inMax)) << outShift;
+        } else {
+            table[i] = 0;
+        }
 #if (OUT != 8)
         if (swap) {
             table[i] = SwapOUT(table[i]);