Fixed a couple of bugs in the general and X11 shape code, and fixed a bug in testshape that was keeping it from recognizing surfaces without alpha. Thanks to Andreas's bit-bashing tip, X11 shaped windows now work entirely, AFAICT.
authorEli Gottlieb <eligottlieb@gmail.com>
Sun, 01 Aug 2010 21:10:42 -0400
changeset 4817c68e7490e4cf
parent 4816 eb433f0d2ac5
child 4818 44889440eada
Fixed a couple of bugs in the general and X11 shape code, and fixed a bug in testshape that was keeping it from recognizing surfaces without alpha. Thanks to Andreas's bit-bashing tip, X11 shaped windows now work entirely, AFAICT.
src/video/SDL_shape.c
src/video/SDL_shape_internals.h
src/video/x11/SDL_x11shape.c
test/testshape.c
     1.1 --- a/src/video/SDL_shape.c	Fri Jul 30 18:04:21 2010 -0400
     1.2 +++ b/src/video/SDL_shape.c	Sun Aug 01 21:10:42 2010 -0400
     1.3 @@ -27,7 +27,7 @@
     1.4  #include "SDL_pixels.h"
     1.5  #include "SDL_surface.h"
     1.6  #include "SDL_shape.h"
     1.7 -#include "SDL_shape_internals.h"
     1.8 +#include "../src/video/SDL_shape_internals.h"
     1.9  
    1.10  SDL_Window* SDL_CreateShapedWindow(const char *title,unsigned int x,unsigned int y,unsigned int w,unsigned int h,Uint32 flags) {
    1.11  	SDL_Window *result = SDL_CreateWindow(title,x,y,w,h,SDL_WINDOW_BORDERLESS | flags & !SDL_WINDOW_FULLSCREEN & !SDL_WINDOW_SHOWN);
    1.12 @@ -57,12 +57,12 @@
    1.13  }
    1.14  
    1.15  /* REQUIRES that bitmap point to a w-by-h bitmap with ppb pixels-per-byte. */
    1.16 -void SDL_CalculateShapeBitmap(SDL_WindowShapeMode mode,SDL_Surface *shape,Uint8* bitmap,Uint8 ppb,Uint8 value) {
    1.17 +void SDL_CalculateShapeBitmap(SDL_WindowShapeMode mode,SDL_Surface *shape,Uint8* bitmap,Uint8 ppb) {
    1.18  	int x = 0;
    1.19  	int y = 0;
    1.20  	Uint8 r = 0,g = 0,b = 0,alpha = 0;
    1.21  	Uint8* pixel = NULL;
    1.22 -	Uint32 bitmap_pixel,pixel_value = 0;
    1.23 +	Uint32 bitmap_pixel,pixel_value = 0,mask_value = 0;
    1.24  	SDL_Color key;
    1.25  	if(SDL_MUSTLOCK(shape))
    1.26  		SDL_LockSurface(shape);
    1.27 @@ -79,6 +79,9 @@
    1.28  				case(2):
    1.29  					pixel_value = *(Uint16*)pixel;
    1.30  					break;
    1.31 +				case(3):
    1.32 +					pixel_value = *(Uint32*)pixel & (~shape->format->Amask);
    1.33 +					break;
    1.34  				case(4):
    1.35  					pixel_value = *(Uint32*)pixel;
    1.36  					break;
    1.37 @@ -87,19 +90,20 @@
    1.38  			bitmap_pixel = y*shape->w + x;
    1.39  			switch(mode.mode) {
    1.40  				case(ShapeModeDefault):
    1.41 -					bitmap[bitmap_pixel / ppb] |= (alpha >= 1 ? value : 0) << ((ppb - 1) - (bitmap_pixel % ppb));
    1.42 +					mask_value = (alpha >= 1 ? 1 : 0);
    1.43  					break;
    1.44  				case(ShapeModeBinarizeAlpha):
    1.45 -					bitmap[bitmap_pixel / ppb] |= (alpha >= mode.parameters.binarizationCutoff ? value : 0) << ((ppb - 1) - (bitmap_pixel % ppb));
    1.46 +					mask_value = (alpha >= mode.parameters.binarizationCutoff ? 1 : 0);
    1.47  					break;
    1.48  				case(ShapeModeReverseBinarizeAlpha):
    1.49 -					bitmap[bitmap_pixel / ppb] |= (alpha <= mode.parameters.binarizationCutoff ? value : 0) << ((ppb - 1) - (bitmap_pixel % ppb));
    1.50 +					mask_value = (alpha <= mode.parameters.binarizationCutoff ? 1 : 0);
    1.51  					break;
    1.52  				case(ShapeModeColorKey):
    1.53  					key = mode.parameters.colorKey;
    1.54 -					bitmap[bitmap_pixel / ppb] |= ((key.r == r && key.g == g && key.b == b) ? value : 0) << ((ppb - 1) - (bitmap_pixel % ppb));
    1.55 +					mask_value = ((key.r != r && key.g != g && key.b != b) ? 1 : 0);
    1.56  					break;
    1.57  			}
    1.58 +			bitmap[bitmap_pixel / ppb] |= mask_value << (7 - ((ppb - 1) - (bitmap_pixel % ppb)));
    1.59  		}
    1.60  	}
    1.61  	if(SDL_MUSTLOCK(shape))
     2.1 --- a/src/video/SDL_shape_internals.h	Fri Jul 30 18:04:21 2010 -0400
     2.2 +++ b/src/video/SDL_shape_internals.h	Sun Aug 01 21:10:42 2010 -0400
     2.3 @@ -19,6 +19,7 @@
     2.4      Eli Gottlieb
     2.5      eligottlieb@gmail.com
     2.6  */
     2.7 +#include "SDL_config.h"
     2.8  
     2.9  #ifndef _SDL_shape_internals_h
    2.10  #define _SDL_shape_internals_h
    2.11 @@ -51,7 +52,7 @@
    2.12  	SDL_ShapeUnion data;
    2.13  } SDL_ShapeTree;
    2.14  
    2.15 -extern void SDL_CalculateShapeBitmap(SDL_WindowShapeMode mode,SDL_Surface *shape,Uint8* bitmap,Uint8 ppb,Uint8 value);
    2.16 +extern void SDL_CalculateShapeBitmap(SDL_WindowShapeMode mode,SDL_Surface *shape,Uint8* bitmap,Uint8 ppb);
    2.17  extern SDL_ShapeTree* SDL_CalculateShapeTree(SDL_WindowShapeMode mode,SDL_Surface* shape,SDL_bool invert);
    2.18  extern void SDL_TraverseShapeTree(SDL_ShapeTree *tree,void(*function)(SDL_ShapeTree*,void*),void* closure);
    2.19  extern void SDL_FreeShapeTree(SDL_ShapeTree** shapeTree);
     3.1 --- a/src/video/x11/SDL_x11shape.c	Fri Jul 30 18:04:21 2010 -0400
     3.2 +++ b/src/video/x11/SDL_x11shape.c	Sun Aug 01 21:10:42 2010 -0400
     3.3 @@ -70,8 +70,7 @@
     3.4  			return -1;
     3.5  		}
     3.6  	}
     3.7 -	else
     3.8 -		memset(data->bitmap,0,data->bitmapsize);
     3.9 +	memset(data->bitmap,0,data->bitmapsize);
    3.10  	
    3.11  	window->shaper->usershownflag |= window->flags & SDL_WINDOW_SHOWN;
    3.12  	
    3.13 @@ -90,7 +89,7 @@
    3.14  	SDL_ShapeData *data = shaper->driverdata;
    3.15  	
    3.16  	/* Assume that shaper->alphacutoff already has a value, because SDL_SetWindowShape() should have given it one. */
    3.17 -	SDL_CalculateShapeBitmap(shaper->mode,shape,data->bitmap,8,1);
    3.18 +	SDL_CalculateShapeBitmap(shaper->mode,shape,data->bitmap,8);
    3.19  		
    3.20  	SDL_WindowData *windowdata = (SDL_WindowData*)(shaper->window->driverdata);
    3.21  	Pixmap shapemask = XCreateBitmapFromData(windowdata->videodata->display,windowdata->xwindow,data->bitmap,shaper->window->w,shaper->window->h);
     4.1 --- a/test/testshape.c	Fri Jul 30 18:04:21 2010 -0400
     4.2 +++ b/test/testshape.c	Sun Aug 01 21:10:42 2010 -0400
     4.3 @@ -43,7 +43,7 @@
     4.4  	Uint8 num_pictures;
     4.5  	LoadedPicture* pictures;
     4.6  	int i, j;
     4.7 -	SDL_PixelFormat* format;
     4.8 +	SDL_PixelFormat* format = NULL;
     4.9  	Uint32 format_enum;
    4.10  	SDL_Window *window;
    4.11  	SDL_Color black = {0,0,0,0xff};
    4.12 @@ -52,8 +52,8 @@
    4.13  	int should_exit = 0;
    4.14  	unsigned int current_picture;
    4.15  	int button_down;
    4.16 -	Uint32 pixelFormat;
    4.17 -	int access;
    4.18 +	Uint32 pixelFormat = 0;
    4.19 +	int access = 0;
    4.20  	SDL_Rect texture_dimensions;;
    4.21  
    4.22  	if(argc < 2) {
    4.23 @@ -84,8 +84,7 @@
    4.24  		}
    4.25  
    4.26  		format = pictures[i].surface->format;
    4.27 -		format_enum = SDL_MasksToPixelFormatEnum (format->BitsPerPixel,format->Rmask,format->Gmask, format->Bmask,format->Amask);
    4.28 -		if(SDL_ISPIXELFORMAT_ALPHA(format_enum)) {
    4.29 +		if(format->Amask != 0) {
    4.30  			pictures[i].mode.mode = ShapeModeBinarizeAlpha;
    4.31  			pictures[i].mode.parameters.binarizationCutoff = 1;
    4.32  		}
    4.33 @@ -139,7 +138,6 @@
    4.34  	event_pending = SDL_PollEvent(&event);
    4.35  	current_picture = 0;
    4.36  	button_down = 0;
    4.37 -	format = 0,access = 0;
    4.38  	texture_dimensions.h = 0;
    4.39  	texture_dimensions.w = 0;
    4.40  	texture_dimensions.x = 0;