CVE-2019-7637: Fix in integer overflow in SDL_CalculatePitch SDL-1.2
authorPetr Písař <ppisar@redhat.com>
Sat, 16 Mar 2019 19:16:24 -0700
branchSDL-1.2
changeset 126539b0e5c555c0f
parent 12646 4646533663ae
child 12721 0c327f0edd70
CVE-2019-7637: Fix in integer overflow in SDL_CalculatePitch
If a too large width is passed to SDL_SetVideoMode() the width travels
to SDL_CalculatePitch() where the width (e.g. 65535) is multiplied by
BytesPerPixel (e.g. 4) and the result is stored into Uint16 pitch
variable. During this arithmetics an integer overflow can happen (e.g.
the value is clamped as 65532). As a result SDL_Surface with a pitch
smaller than width * BytesPerPixel is created, too small pixel buffer
is allocated and when the SDL_Surface is processed in SDL_FillRect()
a buffer overflow occurs.

This can be reproduced with "./graywin -width 21312312313123213213213"
command.

This patch fixes is by using a very careful arithmetics in
SDL_CalculatePitch(). If an overflow is detected, an error is reported
back as a special 0 value. We assume that 0-width surfaces do not
occur in the wild. Since SDL_CalculatePitch() is a private function,
we can change the semantics.

CVE-2019-7637
https://bugzilla.libsdl.org/show_bug.cgi?id=4497

Signed-off-by: Petr Písař <ppisar@redhat.com>
src/video/SDL_pixels.c
src/video/gapi/SDL_gapivideo.c
src/video/nanox/SDL_nxvideo.c
src/video/ps2gs/SDL_gsvideo.c
src/video/ps3/SDL_ps3video.c
src/video/windib/SDL_dibvideo.c
src/video/windx5/SDL_dx5video.c
src/video/x11/SDL_x11video.c
     1.1 --- a/src/video/SDL_pixels.c	Sat Mar 16 18:35:33 2019 -0700
     1.2 +++ b/src/video/SDL_pixels.c	Sat Mar 16 19:16:24 2019 -0700
     1.3 @@ -286,26 +286,53 @@
     1.4  	}
     1.5  }
     1.6  /* 
     1.7 - * Calculate the pad-aligned scanline width of a surface
     1.8 + * Calculate the pad-aligned scanline width of a surface. Return 0 in case of
     1.9 + * an error.
    1.10   */
    1.11  Uint16 SDL_CalculatePitch(SDL_Surface *surface)
    1.12  {
    1.13 -	Uint16 pitch;
    1.14 +	unsigned int pitch = 0;
    1.15  
    1.16  	/* Surface should be 4-byte aligned for speed */
    1.17 -	pitch = surface->w*surface->format->BytesPerPixel;
    1.18 +	/* The code tries to prevent from an Uint16 overflow. */;
    1.19 +	for (Uint8 byte = surface->format->BytesPerPixel; byte; byte--) {
    1.20 +		pitch += (unsigned int)surface->w;
    1.21 +		if (pitch < surface->w) {
    1.22 +			SDL_SetError("A scanline is too wide");
    1.23 +			return(0);
    1.24 +		}
    1.25 +	}
    1.26  	switch (surface->format->BitsPerPixel) {
    1.27  		case 1:
    1.28 -			pitch = (pitch+7)/8;
    1.29 +			if (pitch % 8) {
    1.30 +				pitch = pitch / 8 + 1;
    1.31 +			} else {
    1.32 +				pitch = pitch / 8;
    1.33 +			}
    1.34  			break;
    1.35  		case 4:
    1.36 -			pitch = (pitch+1)/2;
    1.37 +			if (pitch % 2) {
    1.38 +				pitch = pitch / 2 + 1;
    1.39 +			} else {
    1.40 +				pitch = pitch / 2;
    1.41 +			}
    1.42  			break;
    1.43  		default:
    1.44  			break;
    1.45  	}
    1.46 -	pitch = (pitch + 3) & ~3;	/* 4-byte aligning */
    1.47 -	return(pitch);
    1.48 +	/* 4-byte aligning */
    1.49 +	if (pitch & 3) {
    1.50 +		if (pitch + 3 < pitch) {
    1.51 +			SDL_SetError("A scanline is too wide");
    1.52 +			return(0);
    1.53 +		}
    1.54 +		pitch = (pitch + 3) & ~3;
    1.55 +	}
    1.56 +	if (pitch > 0xFFFF) {
    1.57 +		SDL_SetError("A scanline is too wide");
    1.58 +		return(0);
    1.59 +	}
    1.60 +	return((Uint16)pitch);
    1.61  }
    1.62  /*
    1.63   * Match an RGB value to a particular palette index
     2.1 --- a/src/video/gapi/SDL_gapivideo.c	Sat Mar 16 18:35:33 2019 -0700
     2.2 +++ b/src/video/gapi/SDL_gapivideo.c	Sat Mar 16 19:16:24 2019 -0700
     2.3 @@ -733,6 +733,9 @@
     2.4  	video->w = gapi->w = width;
     2.5  	video->h = gapi->h = height;
     2.6  	video->pitch = SDL_CalculatePitch(video); 
     2.7 +	if (!current->pitch) {
     2.8 +		return(NULL);
     2.9 +	}
    2.10  
    2.11  	/* Small fix for WinCE/Win32 - when activating window
    2.12  	   SDL_VideoSurface is equal to zero, so activating code
     3.1 --- a/src/video/nanox/SDL_nxvideo.c	Sat Mar 16 18:35:33 2019 -0700
     3.2 +++ b/src/video/nanox/SDL_nxvideo.c	Sat Mar 16 19:16:24 2019 -0700
     3.3 @@ -378,6 +378,10 @@
     3.4          current -> w = width ;
     3.5          current -> h = height ;
     3.6          current -> pitch = SDL_CalculatePitch (current) ;
     3.7 +        if (!current->pitch) {
     3.8 +            current = NULL;
     3.9 +            goto done;
    3.10 +        }
    3.11          NX_ResizeImage (this, current, flags) ;
    3.12      }
    3.13  
     4.1 --- a/src/video/ps2gs/SDL_gsvideo.c	Sat Mar 16 18:35:33 2019 -0700
     4.2 +++ b/src/video/ps2gs/SDL_gsvideo.c	Sat Mar 16 19:16:24 2019 -0700
     4.3 @@ -479,6 +479,9 @@
     4.4  	current->w = width;
     4.5  	current->h = height;
     4.6  	current->pitch = SDL_CalculatePitch(current);
     4.7 +	if (!current->pitch) {
     4.8 +		return(NULL);
     4.9 +	}
    4.10  
    4.11  	/* Memory map the DMA area for block memory transfer */
    4.12  	if ( ! mapped_mem ) {
     5.1 --- a/src/video/ps3/SDL_ps3video.c	Sat Mar 16 18:35:33 2019 -0700
     5.2 +++ b/src/video/ps3/SDL_ps3video.c	Sat Mar 16 19:16:24 2019 -0700
     5.3 @@ -339,6 +339,9 @@
     5.4  	current->w = width;
     5.5  	current->h = height;
     5.6  	current->pitch = SDL_CalculatePitch(current);
     5.7 +	if (!current->pitch) {
     5.8 +		return(NULL);
     5.9 +	}
    5.10  
    5.11  	/* Alloc aligned mem for current->pixels */
    5.12  	s_pixels = memalign(16, current->h * current->pitch);
     6.1 --- a/src/video/windib/SDL_dibvideo.c	Sat Mar 16 18:35:33 2019 -0700
     6.2 +++ b/src/video/windib/SDL_dibvideo.c	Sat Mar 16 19:16:24 2019 -0700
     6.3 @@ -675,6 +675,9 @@
     6.4  	video->w = width;
     6.5  	video->h = height;
     6.6  	video->pitch = SDL_CalculatePitch(video);
     6.7 +	if (!current->pitch) {
     6.8 +		return(NULL);
     6.9 +	}
    6.10  
    6.11  	/* Small fix for WinCE/Win32 - when activating window
    6.12  	   SDL_VideoSurface is equal to zero, so activating code
     7.1 --- a/src/video/windx5/SDL_dx5video.c	Sat Mar 16 18:35:33 2019 -0700
     7.2 +++ b/src/video/windx5/SDL_dx5video.c	Sat Mar 16 19:16:24 2019 -0700
     7.3 @@ -1127,6 +1127,9 @@
     7.4  		video->w = width;
     7.5  		video->h = height;
     7.6  		video->pitch = SDL_CalculatePitch(video);
     7.7 +		if (!current->pitch) {
     7.8 +			return(NULL);
     7.9 +		}
    7.10  
    7.11  #ifndef NO_CHANGEDISPLAYSETTINGS
    7.12  		/* Set fullscreen mode if appropriate.
     8.1 --- a/src/video/x11/SDL_x11video.c	Sat Mar 16 18:35:33 2019 -0700
     8.2 +++ b/src/video/x11/SDL_x11video.c	Sat Mar 16 19:16:24 2019 -0700
     8.3 @@ -1225,6 +1225,10 @@
     8.4  		current->w = width;
     8.5  		current->h = height;
     8.6  		current->pitch = SDL_CalculatePitch(current);
     8.7 +		if (!current->pitch) {
     8.8 +			current = NULL;
     8.9 +			goto done;
    8.10 +		}
    8.11  		if (X11_ResizeImage(this, current, flags) < 0) {
    8.12  			current = NULL;
    8.13  			goto done;