examples,test: fix integer overflows in calculating alloc sizes

Allocating memory with a size controlled by an external user can result
in integer overflow.

Author: szsam

examples/server/pnmshow.c

@@ -81,6 +81,12 @@ int main(int argc,char** argv)
   rfbScreen->httpDir = "../webclients";
 
   /* allocate picture and read it */
+  if (bytesPerPixel!=0 && paddedWidth>SIZE_MAX/bytesPerPixel) {
+    exit(1);
+  }
+  if (height!=0 && paddedWidth*bytesPerPixel>SIZE_MAX/height) {
+    exit(1);
+  }
   rfbScreen->frameBuffer = (char*)malloc(paddedWidth*bytesPerPixel*height);
   if(!rfbScreen->frameBuffer)
       exit(1);

examples/server/pnmshow24.c

@@ -80,6 +80,8 @@ int main(int argc,char** argv)
   rfbScreen->httpDir = "../webclients";
 
   /* allocate picture and read it */
+  if (paddedWidth>SIZE_MAX/3 || (height!=0 && paddedWidth*3>SIZE_MAX/height))
+    return 1;
   rfbScreen->frameBuffer = (char*)malloc(paddedWidth*3*height);
   if(!rfbScreen->frameBuffer)
     return 1;

test/bmp.c

@@ -20,6 +20,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <stdint.h>
 #ifdef _WIN32
  #include <io.h>
 #else
@@ -139,7 +140,7 @@ int loadppm(int *fd, unsigned char **buf, int *w, int *h,
 	if((*w)<1 || (*h)<1 || scalefactor<1) _throw("Corrupt PPM header");
 
 	dstpitch=(((*w)*ps[f])+(align-1))&(~(align-1));
-	if((*buf=(unsigned char *)malloc(dstpitch*(*h)))==NULL)
+	if((*buf=(unsigned char *)calloc(dstpitch, *h))==NULL)
 		_throw("Memory allocation error");
 	if(ascii)
 	{
@@ -159,7 +160,7 @@ int loadppm(int *fd, unsigned char **buf, int *w, int *h,
 	{
 		if(scalefactor!=255)
 			_throw("Binary PPMs must have 8-bit components");
-		if((tempbuf=(unsigned char *)malloc((*w)*(*h)*3))==NULL)
+		if(*h>SIZE_MAX/3 || (tempbuf=(unsigned char *)calloc(*w, (*h)*3))==NULL)
 			_throw("Memory allocation error");
 		if(fread(tempbuf, (*w)*(*h)*3, 1, fs)!=1) _throw("Read error");
 		pixelconvert(tempbuf, BMP_RGB, (*w)*3, *buf, f, dstpitch, *w, *h, dstbottomup);
@@ -249,8 +250,8 @@ int loadbmp(char *filename, unsigned char **buf, int *w, int *h,
 	dstpitch=(((*w)*ps[f])+(align-1))&(~(align-1));
 
 	if(srcpitch*(*h)+bh.bfOffBits!=bh.bfSize) _throw("Corrupt bitmap header");
-	if((tempbuf=(unsigned char *)malloc(srcpitch*(*h)))==NULL
-	|| (*buf=(unsigned char *)malloc(dstpitch*(*h)))==NULL)
+	if((tempbuf=(unsigned char *)calloc(srcpitch, *h))==NULL
+	|| (*buf=(unsigned char *)calloc(dstpitch, *h))==NULL)
 		_throw("Memory allocation error");
 	if(lseek(fd, (long)bh.bfOffBits, SEEK_SET)!=(long)bh.bfOffBits)
 		_throw(strerror(errno));