Fix several places in tiff2rgba and rgb2ycbcr that were being careless about
possible integer overflow in calculation of buffer sizes.

CVE-2009-2347


diff -Naur tiff-3.8.2.orig/tools/rgb2ycbcr.c tiff-3.8.2/tools/rgb2ycbcr.c
--- tiff-3.8.2.orig/tools/rgb2ycbcr.c	2004-09-03 03:57:13.000000000 -0400
+++ tiff-3.8.2/tools/rgb2ycbcr.c	2009-07-10 17:12:32.000000000 -0400
@@ -202,6 +202,17 @@
 #undef LumaBlue
 #undef V2Code
 
+static tsize_t
+multiply(tsize_t m1, tsize_t m2)
+{
+    tsize_t prod = m1 * m2;
+
+    if (m1 && prod / m1 != m2)
+        prod = 0;		/* overflow */
+
+    return prod;
+}
+
 /*
  * Convert a strip of RGB data to YCbCr and
  * sample to generate the output data.
@@ -278,10 +289,19 @@
 	float floatv;
 	char *stringv;
 	uint32 longv;
+	tsize_t raster_size;
 
 	TIFFGetField(in, TIFFTAG_IMAGEWIDTH, &width);
 	TIFFGetField(in, TIFFTAG_IMAGELENGTH, &height);
-	raster = (uint32*)_TIFFmalloc(width * height * sizeof (uint32));
+
+	raster_size = multiply(multiply(width, height), sizeof (uint32));
+	if (!raster_size) {
+		TIFFError(TIFFFileName(in),
+			  "Can't allocate buffer for raster of size %lux%lu",
+			  (unsigned long) width, (unsigned long) height);
+		return (0);
+	}
+	raster = (uint32*)_TIFFmalloc(raster_size);
 	if (raster == 0) {
 		TIFFError(TIFFFileName(in), "No space for raster buffer");
 		return (0);
diff -Naur tiff-3.8.2.orig/tools/tiff2rgba.c tiff-3.8.2/tools/tiff2rgba.c
--- tiff-3.8.2.orig/tools/tiff2rgba.c	2004-11-07 06:08:37.000000000 -0500
+++ tiff-3.8.2/tools/tiff2rgba.c	2009-07-10 17:06:42.000000000 -0400
@@ -124,6 +124,17 @@
     return (0);
 }
 
+static tsize_t
+multiply(tsize_t m1, tsize_t m2)
+{
+    tsize_t prod = m1 * m2;
+
+    if (m1 && prod / m1 != m2)
+        prod = 0;		/* overflow */
+
+    return prod;
+}
+
 static int
 cvt_by_tile( TIFF *in, TIFF *out )
 
@@ -133,6 +144,7 @@
     uint32  tile_width, tile_height;
     uint32  row, col;
     uint32  *wrk_line;
+    tsize_t raster_size;
     int	    ok = 1;
 
     TIFFGetField(in, TIFFTAG_IMAGEWIDTH, &width);
@@ -150,7 +162,14 @@
     /*
      * Allocate tile buffer
      */
-    raster = (uint32*)_TIFFmalloc(tile_width * tile_height * sizeof (uint32));
+    raster_size = multiply(multiply(tile_width, tile_height), sizeof (uint32));
+    if (!raster_size) {
+	TIFFError(TIFFFileName(in),
+		  "Can't allocate buffer for raster of size %lux%lu",
+		  (unsigned long) tile_width, (unsigned long) tile_height);
+	return (0);
+    }
+    raster = (uint32*)_TIFFmalloc(raster_size);
     if (raster == 0) {
         TIFFError(TIFFFileName(in), "No space for raster buffer");
         return (0);
@@ -158,7 +177,7 @@
 
     /*
      * Allocate a scanline buffer for swapping during the vertical
-     * mirroring pass.
+     * mirroring pass.  (Request can't overflow given prior checks.)
      */
     wrk_line = (uint32*)_TIFFmalloc(tile_width * sizeof (uint32));
     if (!wrk_line) {
@@ -226,6 +245,7 @@
     uint32  width, height;		/* image width & height */
     uint32  row;
     uint32  *wrk_line;
+    tsize_t raster_size;
     int	    ok = 1;
 
     TIFFGetField(in, TIFFTAG_IMAGEWIDTH, &width);
@@ -241,7 +261,14 @@
     /*
      * Allocate strip buffer
      */
-    raster = (uint32*)_TIFFmalloc(width * rowsperstrip * sizeof (uint32));
+    raster_size = multiply(multiply(width, rowsperstrip), sizeof (uint32));
+    if (!raster_size) {
+	TIFFError(TIFFFileName(in),
+		  "Can't allocate buffer for raster of size %lux%lu",
+		  (unsigned long) width, (unsigned long) rowsperstrip);
+	return (0);
+    }
+    raster = (uint32*)_TIFFmalloc(raster_size);
     if (raster == 0) {
         TIFFError(TIFFFileName(in), "No space for raster buffer");
         return (0);
@@ -249,7 +276,7 @@
 
     /*
      * Allocate a scanline buffer for swapping during the vertical
-     * mirroring pass.
+     * mirroring pass.  (Request can't overflow given prior checks.)
      */
     wrk_line = (uint32*)_TIFFmalloc(width * sizeof (uint32));
     if (!wrk_line) {
@@ -328,14 +355,22 @@
     uint32* raster;			/* retrieve RGBA image */
     uint32  width, height;		/* image width & height */
     uint32  row;
-        
+    tsize_t raster_size;
+
     TIFFGetField(in, TIFFTAG_IMAGEWIDTH, &width);
     TIFFGetField(in, TIFFTAG_IMAGELENGTH, &height);
 
     rowsperstrip = TIFFDefaultStripSize(out, rowsperstrip);
     TIFFSetField(out, TIFFTAG_ROWSPERSTRIP, rowsperstrip);
 
-    raster = (uint32*)_TIFFmalloc(width * height * sizeof (uint32));
+    raster_size = multiply(multiply(width, height), sizeof (uint32));
+    if (!raster_size) {
+	TIFFError(TIFFFileName(in),
+		  "Can't allocate buffer for raster of size %lux%lu",
+		  (unsigned long) width, (unsigned long) height);
+	return (0);
+    }
+    raster = (uint32*)_TIFFmalloc(raster_size);
     if (raster == 0) {
         TIFFError(TIFFFileName(in), "No space for raster buffer");
         return (0);
@@ -353,7 +388,7 @@
     */
     if( no_alpha )
     {
-        int	pixel_count = width * height;
+        tsize_t  pixel_count = (tsize_t) width * (tsize_t) height;
         unsigned char *src, *dst;
 
         src = (unsigned char *) raster;