Marcus von Appen fixed bug 1535: SDL_UpdateWindowRects() SIGSEGV on X11 with out-of-bound rects
authorSam Lantinga
Thu, 05 Jul 2012 07:26:18 -0400
changeset 63518f5535d05e34
parent 6350 19545983ac76
child 6352 a9bcd26e7105
Marcus von Appen fixed bug 1535: SDL_UpdateWindowRects() SIGSEGV on X11 with out-of-bound rects

Using SDL_UpdateWindowRects () with SDL_Rect instances, that do not clip
to the bounds of the SDL_Window, causes a BadValue error on X11, if shared
memory is enabled:

X Error of failed request: BadValue (integer parameter out of range for
operation)
Major opcode of failed request: 142 (MIT-SHM)
Minor opcode of failed request: 3 (X_ShmPutImage)
Value in failed request: 0x5
Serial number of failed request: 67
Current serial number in output stream: 70

Attached is a test program to recreate the behaviour along with a simple
patch for SDL_x11framebuffer.c, which reduces the rect area to the window
bounds.

I am not sure, if XPutImage() is more robust here, so I applied it to the
non-shared memory version, too.
src/video/x11/SDL_x11framebuffer.c
     1.1 --- a/src/video/x11/SDL_x11framebuffer.c	Tue Jul 03 23:52:02 2012 -0400
     1.2 +++ b/src/video/x11/SDL_x11framebuffer.c	Thu Jul 05 07:26:18 2012 -0400
     1.3 @@ -154,33 +154,68 @@
     1.4      SDL_WindowData *data = (SDL_WindowData *) window->driverdata;
     1.5      Display *display = data->videodata->display;
     1.6      int i;
     1.7 -    SDL_Rect *rect;
     1.8 -
     1.9 +    int x, y, w ,h;
    1.10  #ifndef NO_SHARED_MEMORY
    1.11      if (data->use_mitshm) {
    1.12          for (i = 0; i < numrects; ++i) {
    1.13 -            rect = &rects[i];
    1.14 +            x = rects[i].x;
    1.15 +            y = rects[i].y;
    1.16 +            w = rects[i].w;
    1.17 +            h = rects[i].h;
    1.18  
    1.19 -            if (rect->w == 0 || rect->h == 0) { /* Clipped? */
    1.20 +            if (w <= 0 || h <= 0 || (x + w) <= 0 || (y + h) <= 0) {
    1.21 +                /* Clipped? */
    1.22                  continue;
    1.23              }
    1.24 +            if (x < 0)
    1.25 +            {
    1.26 +                x += w;
    1.27 +                w += rects[i].x;
    1.28 +            }
    1.29 +            if (y < 0)
    1.30 +            {
    1.31 +                y += h;
    1.32 +                h += rects[i].y;
    1.33 +            }
    1.34 +            if (x + w > window->w)
    1.35 +                w = window->w - x;
    1.36 +            if (y + h > window->h)
    1.37 +                h = window->h - y;
    1.38 +
    1.39              XShmPutImage(display, data->xwindow, data->gc, data->ximage,
    1.40 -                    rect->x, rect->y,
    1.41 -                    rect->x, rect->y, rect->w, rect->h, False);
    1.42 +                x, y, x, y, w, h, False);
    1.43          }
    1.44      }
    1.45      else
    1.46  #endif /* !NO_SHARED_MEMORY */
    1.47      {
    1.48          for (i = 0; i < numrects; ++i) {
    1.49 -            rect = &rects[i];
    1.50 +            x = rects[i].x;
    1.51 +            y = rects[i].y;
    1.52 +            w = rects[i].w;
    1.53 +            h = rects[i].h;
    1.54  
    1.55 -            if (rect->w == 0 || rect->h == 0) { /* Clipped? */
    1.56 +            if (w <= 0 || h <= 0 || (x + w) <= 0 || (y + h) <= 0) {
    1.57 +                /* Clipped? */
    1.58                  continue;
    1.59              }
    1.60 +            if (x < 0)
    1.61 +            {
    1.62 +                x += w;
    1.63 +                w += rects[i].x;
    1.64 +            }
    1.65 +            if (y < 0)
    1.66 +            {
    1.67 +                y += h;
    1.68 +                h += rects[i].y;
    1.69 +            }
    1.70 +            if (x + w > window->w)
    1.71 +                w = window->w - x;
    1.72 +            if (y + h > window->h)
    1.73 +                h = window->h - y;
    1.74 +
    1.75              XPutImage(display, data->xwindow, data->gc, data->ximage,
    1.76 -                  rect->x, rect->y,
    1.77 -                  rect->x, rect->y, rect->w, rect->h);
    1.78 +                x, y, x, y, w, h);
    1.79          }
    1.80      }
    1.81