CVE-2019-7637: Fix in integer overflow in SDL_CalculatePitch
If a too large width is passed to SDL_SetVideoMode() the width travels
to SDL_CalculatePitch() where the width (e.g. 65535) is multiplied by
BytesPerPixel (e.g. 4) and the result is stored into Uint16 pitch
variable. During this arithmetics an integer overflow can happen (e.g.
the value is clamped as 65532). As a result SDL_Surface with a pitch
smaller than width * BytesPerPixel is created, too small pixel buffer
is allocated and when the SDL_Surface is processed in SDL_FillRect()
a buffer overflow occurs.

This can be reproduced with "./graywin -width 21312312313123213213213"
command.

This patch fixes is by using a very careful arithmetics in
SDL_CalculatePitch(). If an overflow is detected, an error is reported
back as a special 0 value. We assume that 0-width surfaces do not
occur in the wild. Since SDL_CalculatePitch() is a private function,
we can change the semantics.

CVE-2019-7637
https://bugzilla.libsdl.org/show_bug.cgi?id=4497

Signed-off-by: Petr Písař <ppisar@redhat.com>

diff -r 4646533663ae -r 9b0e5c555c0f src/video/SDL_pixels.c
--- a/src/video/SDL_pixels.c	Sat Mar 16 18:35:33 2019 -0700
+++ b/src/video/SDL_pixels.c	Sat Mar 16 19:16:24 2019 -0700
@@ -286,26 +286,53 @@
 	}
 }
 /* 
- * Calculate the pad-aligned scanline width of a surface
+ * Calculate the pad-aligned scanline width of a surface. Return 0 in case of
+ * an error.
  */
 Uint16 SDL_CalculatePitch(SDL_Surface *surface)
 {
-	Uint16 pitch;
+	unsigned int pitch = 0;
 
 	/* Surface should be 4-byte aligned for speed */
-	pitch = surface->w*surface->format->BytesPerPixel;
+	/* The code tries to prevent from an Uint16 overflow. */;
+	for (Uint8 byte = surface->format->BytesPerPixel; byte; byte--) {
+		pitch += (unsigned int)surface->w;
+		if (pitch < surface->w) {
+			SDL_SetError("A scanline is too wide");
+			return(0);
+		}
+	}
 	switch (surface->format->BitsPerPixel) {
 		case 1:
-			pitch = (pitch+7)/8;
+			if (pitch % 8) {
+				pitch = pitch / 8 + 1;
+			} else {
+				pitch = pitch / 8;
+			}
 			break;
 		case 4:
-			pitch = (pitch+1)/2;
+			if (pitch % 2) {
+				pitch = pitch / 2 + 1;
+			} else {
+				pitch = pitch / 2;
+			}
 			break;
 		default:
 			break;
 	}
-	pitch = (pitch + 3) & ~3;	/* 4-byte aligning */
-	return(pitch);
+	/* 4-byte aligning */
+	if (pitch & 3) {
+		if (pitch + 3 < pitch) {
+			SDL_SetError("A scanline is too wide");
+			return(0);
+		}
+		pitch = (pitch + 3) & ~3;
+	}
+	if (pitch > 0xFFFF) {
+		SDL_SetError("A scanline is too wide");
+		return(0);
+	}
+	return((Uint16)pitch);
 }
 /*
  * Match an RGB value to a particular palette index
diff -r 4646533663ae -r 9b0e5c555c0f src/video/gapi/SDL_gapivideo.c
--- a/src/video/gapi/SDL_gapivideo.c	Sat Mar 16 18:35:33 2019 -0700
+++ b/src/video/gapi/SDL_gapivideo.c	Sat Mar 16 19:16:24 2019 -0700
@@ -733,6 +733,9 @@
 	video->w = gapi->w = width;
 	video->h = gapi->h = height;
 	video->pitch = SDL_CalculatePitch(video); 
+	if (!current->pitch) {
+		return(NULL);
+	}
 
 	/* Small fix for WinCE/Win32 - when activating window
 	   SDL_VideoSurface is equal to zero, so activating code
diff -r 4646533663ae -r 9b0e5c555c0f src/video/nanox/SDL_nxvideo.c
--- a/src/video/nanox/SDL_nxvideo.c	Sat Mar 16 18:35:33 2019 -0700
+++ b/src/video/nanox/SDL_nxvideo.c	Sat Mar 16 19:16:24 2019 -0700
@@ -378,6 +378,10 @@
         current -> w = width ;
         current -> h = height ;
         current -> pitch = SDL_CalculatePitch (current) ;
+        if (!current->pitch) {
+            current = NULL;
+            goto done;
+        }
         NX_ResizeImage (this, current, flags) ;
     }
 
diff -r 4646533663ae -r 9b0e5c555c0f src/video/ps2gs/SDL_gsvideo.c
--- a/src/video/ps2gs/SDL_gsvideo.c	Sat Mar 16 18:35:33 2019 -0700
+++ b/src/video/ps2gs/SDL_gsvideo.c	Sat Mar 16 19:16:24 2019 -0700
@@ -479,6 +479,9 @@
 	current->w = width;
 	current->h = height;
 	current->pitch = SDL_CalculatePitch(current);
+	if (!current->pitch) {
+		return(NULL);
+	}
 
 	/* Memory map the DMA area for block memory transfer */
 	if ( ! mapped_mem ) {
diff -r 4646533663ae -r 9b0e5c555c0f src/video/ps3/SDL_ps3video.c
--- a/src/video/ps3/SDL_ps3video.c	Sat Mar 16 18:35:33 2019 -0700
+++ b/src/video/ps3/SDL_ps3video.c	Sat Mar 16 19:16:24 2019 -0700
@@ -339,6 +339,9 @@
 	current->w = width;
 	current->h = height;
 	current->pitch = SDL_CalculatePitch(current);
+	if (!current->pitch) {
+		return(NULL);
+	}
 
 	/* Alloc aligned mem for current->pixels */
 	s_pixels = memalign(16, current->h * current->pitch);
diff -r 4646533663ae -r 9b0e5c555c0f src/video/windib/SDL_dibvideo.c
--- a/src/video/windib/SDL_dibvideo.c	Sat Mar 16 18:35:33 2019 -0700
+++ b/src/video/windib/SDL_dibvideo.c	Sat Mar 16 19:16:24 2019 -0700
@@ -675,6 +675,9 @@
 	video->w = width;
 	video->h = height;
 	video->pitch = SDL_CalculatePitch(video);
+	if (!current->pitch) {
+		return(NULL);
+	}
 
 	/* Small fix for WinCE/Win32 - when activating window
 	   SDL_VideoSurface is equal to zero, so activating code
diff -r 4646533663ae -r 9b0e5c555c0f src/video/windx5/SDL_dx5video.c
--- a/src/video/windx5/SDL_dx5video.c	Sat Mar 16 18:35:33 2019 -0700
+++ b/src/video/windx5/SDL_dx5video.c	Sat Mar 16 19:16:24 2019 -0700
@@ -1127,6 +1127,9 @@
 		video->w = width;
 		video->h = height;
 		video->pitch = SDL_CalculatePitch(video);
+		if (!current->pitch) {
+			return(NULL);
+		}
 
 #ifndef NO_CHANGEDISPLAYSETTINGS
 		/* Set fullscreen mode if appropriate.
diff -r 4646533663ae -r 9b0e5c555c0f src/video/x11/SDL_x11video.c
--- a/src/video/x11/SDL_x11video.c	Sat Mar 16 18:35:33 2019 -0700
+++ b/src/video/x11/SDL_x11video.c	Sat Mar 16 19:16:24 2019 -0700
@@ -1225,6 +1225,10 @@
 		current->w = width;
 		current->h = height;
 		current->pitch = SDL_CalculatePitch(current);
+		if (!current->pitch) {
+			current = NULL;
+			goto done;
+		}
 		if (X11_ResizeImage(this, current, flags) < 0) {
 			current = NULL;
 			goto done;