dront78 implemented YUV texture support for OpenGL ES 2.0
authorSam Lantinga <slouken@libsdl.org>
Sat, 07 Jun 2014 11:36:08 -0700
changeset 8835bc5ec1b6904c
parent 8834 b300e097899d
child 8836 ab4cb2acb4d1
dront78 implemented YUV texture support for OpenGL ES 2.0
src/render/opengles2/SDL_render_gles2.c
src/render/opengles2/SDL_shaders_gles2.c
src/render/opengles2/SDL_shaders_gles2.h
     1.1 --- a/src/render/opengles2/SDL_render_gles2.c	Fri Jun 06 18:33:17 2014 -0300
     1.2 +++ b/src/render/opengles2/SDL_render_gles2.c	Sat Jun 07 11:36:08 2014 -0700
     1.3 @@ -79,6 +79,10 @@
     1.4      GLenum pixel_type;
     1.5      void *pixel_data;
     1.6      size_t pitch;
     1.7 +    /* YV12 texture support */
     1.8 +    SDL_bool yuv;
     1.9 +    GLenum texture_v;
    1.10 +    GLenum texture_u;
    1.11      GLES2_FBOList *fbo;
    1.12  } GLES2_TextureData;
    1.13  
    1.14 @@ -133,7 +137,9 @@
    1.15      GLES2_UNIFORM_PROJECTION,
    1.16      GLES2_UNIFORM_TEXTURE,
    1.17      GLES2_UNIFORM_MODULATION,
    1.18 -    GLES2_UNIFORM_COLOR
    1.19 +    GLES2_UNIFORM_COLOR,
    1.20 +    GLES2_UNIFORM_TEXTURE_U,
    1.21 +    GLES2_UNIFORM_TEXTURE_V
    1.22  } GLES2_Uniform;
    1.23  
    1.24  typedef enum
    1.25 @@ -142,7 +148,8 @@
    1.26      GLES2_IMAGESOURCE_TEXTURE_ABGR,
    1.27      GLES2_IMAGESOURCE_TEXTURE_ARGB,
    1.28      GLES2_IMAGESOURCE_TEXTURE_RGB,
    1.29 -    GLES2_IMAGESOURCE_TEXTURE_BGR
    1.30 +    GLES2_IMAGESOURCE_TEXTURE_BGR,
    1.31 +    GLES2_IMAGESOURCE_TEXTURE_YUV
    1.32  } GLES2_ImageSource;
    1.33  
    1.34  typedef struct GLES2_DriverContext
    1.35 @@ -433,6 +440,11 @@
    1.36  static int GLES2_CreateTexture(SDL_Renderer *renderer, SDL_Texture *texture);
    1.37  static int GLES2_UpdateTexture(SDL_Renderer *renderer, SDL_Texture *texture, const SDL_Rect *rect,
    1.38                                 const void *pixels, int pitch);
    1.39 +static int GLES2_UpdateTextureYUV(SDL_Renderer * renderer, SDL_Texture * texture,
    1.40 +                               const SDL_Rect * rect,
    1.41 +                               const Uint8 *Yplane, int Ypitch,
    1.42 +                               const Uint8 *Uplane, int Upitch,
    1.43 +                               const Uint8 *Vplane, int Vpitch);
    1.44  static int GLES2_LockTexture(SDL_Renderer *renderer, SDL_Texture *texture, const SDL_Rect *rect,
    1.45                               void **pixels, int *pitch);
    1.46  static void GLES2_UnlockTexture(SDL_Renderer *renderer, SDL_Texture *texture);
    1.47 @@ -472,6 +484,11 @@
    1.48          format = GL_RGBA;
    1.49          type = GL_UNSIGNED_BYTE;
    1.50          break;
    1.51 +    case SDL_PIXELFORMAT_IYUV:
    1.52 +    case SDL_PIXELFORMAT_YV12:
    1.53 +        format = GL_LUMINANCE;
    1.54 +        type = GL_UNSIGNED_BYTE;
    1.55 +        break;
    1.56      default:
    1.57          return SDL_SetError("Texture format not supported");
    1.58      }
    1.59 @@ -485,12 +502,21 @@
    1.60      data->texture_type = GL_TEXTURE_2D;
    1.61      data->pixel_format = format;
    1.62      data->pixel_type = type;
    1.63 +    data->yuv = ((texture->format == SDL_PIXELFORMAT_IYUV) || (texture->format == SDL_PIXELFORMAT_YV12));
    1.64 +    data->texture_u = 0;
    1.65 +    data->texture_v = 0;
    1.66      scaleMode = GetScaleQuality();
    1.67  
    1.68      /* Allocate a blob for image renderdata */
    1.69      if (texture->access == SDL_TEXTUREACCESS_STREAMING) {
    1.70 +        size_t size;
    1.71          data->pitch = texture->w * SDL_BYTESPERPIXEL(texture->format);
    1.72 -        data->pixel_data = SDL_calloc(1, data->pitch * texture->h);
    1.73 +        size = texture->h * data->pitch;
    1.74 +        if (data->yuv) {
    1.75 +            /* Need to add size for the U and V planes */
    1.76 +            size += (2 * (texture->h * data->pitch) / 4);
    1.77 +        }
    1.78 +        data->pixel_data = SDL_calloc(1, size);
    1.79          if (!data->pixel_data) {
    1.80              SDL_free(data);
    1.81              return SDL_OutOfMemory();
    1.82 @@ -499,11 +525,42 @@
    1.83  
    1.84      /* Allocate the texture */
    1.85      GL_CheckError("", renderer);
    1.86 +
    1.87 +    if (data->yuv) {
    1.88 +        renderdata->glGenTextures(1, &data->texture_v);
    1.89 +        if (GL_CheckError("glGenTexures()", renderer) < 0) {
    1.90 +            return -1;
    1.91 +        }
    1.92 +        renderdata->glActiveTexture(GL_TEXTURE2);
    1.93 +        renderdata->glBindTexture(data->texture_type, data->texture_v);
    1.94 +        renderdata->glTexParameteri(data->texture_type, GL_TEXTURE_MIN_FILTER, scaleMode);
    1.95 +        renderdata->glTexParameteri(data->texture_type, GL_TEXTURE_MAG_FILTER, scaleMode);
    1.96 +        renderdata->glTexParameteri(data->texture_type, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    1.97 +        renderdata->glTexParameteri(data->texture_type, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    1.98 +        renderdata->glTexImage2D(data->texture_type, 0, format, texture->w / 2, texture->h / 2, 0, format, type, NULL);
    1.99 +
   1.100 +        renderdata->glGenTextures(1, &data->texture_u);
   1.101 +        if (GL_CheckError("glGenTexures()", renderer) < 0) {
   1.102 +            return -1;
   1.103 +        }
   1.104 +        renderdata->glActiveTexture(GL_TEXTURE1);
   1.105 +        renderdata->glBindTexture(data->texture_type, data->texture_u);
   1.106 +        renderdata->glTexParameteri(data->texture_type, GL_TEXTURE_MIN_FILTER, scaleMode);
   1.107 +        renderdata->glTexParameteri(data->texture_type, GL_TEXTURE_MAG_FILTER, scaleMode);
   1.108 +        renderdata->glTexParameteri(data->texture_type, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
   1.109 +        renderdata->glTexParameteri(data->texture_type, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
   1.110 +        renderdata->glTexImage2D(data->texture_type, 0, format, texture->w / 2, texture->h / 2, 0, format, type, NULL);
   1.111 +        if (GL_CheckError("glTexImage2D()", renderer) < 0) {
   1.112 +            return -1;
   1.113 +        }
   1.114 +    }
   1.115 +
   1.116      renderdata->glGenTextures(1, &data->texture);
   1.117      if (GL_CheckError("glGenTexures()", renderer) < 0) {
   1.118          return -1;
   1.119      }
   1.120      texture->driverdata = data;
   1.121 +    renderdata->glActiveTexture(GL_TEXTURE0);
   1.122      renderdata->glBindTexture(data->texture_type, data->texture);
   1.123      renderdata->glTexParameteri(data->texture_type, GL_TEXTURE_MIN_FILTER, scaleMode);
   1.124      renderdata->glTexParameteri(data->texture_type, GL_TEXTURE_MAG_FILTER, scaleMode);
   1.125 @@ -575,6 +632,53 @@
   1.126  }
   1.127  
   1.128  static int
   1.129 +GLES2_UpdateTextureYUV(SDL_Renderer * renderer, SDL_Texture * texture,
   1.130 +                    const SDL_Rect * rect,
   1.131 +                    const Uint8 *Yplane, int Ypitch,
   1.132 +                    const Uint8 *Uplane, int Upitch,
   1.133 +                    const Uint8 *Vplane, int Vpitch)
   1.134 +{
   1.135 +    GLES2_DriverContext *data = (GLES2_DriverContext *)renderer->driverdata;
   1.136 +    GLES2_TextureData *tdata = (GLES2_TextureData *)texture->driverdata;
   1.137 +
   1.138 +    data->glBindTexture(tdata->texture_type, tdata->texture_v);
   1.139 +    data->glTexSubImage2D(tdata->texture_type,
   1.140 +                    0,
   1.141 +                    rect->x,
   1.142 +                    rect->y,
   1.143 +                    rect->w / 2,
   1.144 +                    rect->h / 2,
   1.145 +                    tdata->pixel_format,
   1.146 +                    tdata->pixel_type,
   1.147 +                    Vplane);
   1.148 +
   1.149 +    data->glBindTexture(tdata->texture_type, tdata->texture_u);
   1.150 +    data->glTexSubImage2D(tdata->texture_type,
   1.151 +                    0,
   1.152 +                    rect->x,
   1.153 +                    rect->y,
   1.154 +                    rect->w / 2,
   1.155 +                    rect->h / 2,
   1.156 +                    tdata->pixel_format,
   1.157 +                    tdata->pixel_type,
   1.158 +                    Uplane);
   1.159 +
   1.160 +    data->glBindTexture(tdata->texture_type, tdata->texture);
   1.161 +    data->glTexSubImage2D(tdata->texture_type,
   1.162 +                    0,
   1.163 +                    rect->x,
   1.164 +                    rect->y,
   1.165 +                    rect->w,
   1.166 +                    rect->h,
   1.167 +                    tdata->pixel_format,
   1.168 +                    tdata->pixel_type,
   1.169 +                    Yplane);
   1.170 +
   1.171 +
   1.172 +    return GL_CheckError("glTexSubImage2D()", renderer);
   1.173 +}
   1.174 +
   1.175 +static int
   1.176  GLES2_LockTexture(SDL_Renderer *renderer, SDL_Texture *texture, const SDL_Rect *rect,
   1.177                    void **pixels, int *pitch)
   1.178  {
   1.179 @@ -638,6 +742,12 @@
   1.180      if (tdata)
   1.181      {
   1.182          data->glDeleteTextures(1, &tdata->texture);
   1.183 +        if (tdata->texture_v) {
   1.184 +            data->glDeleteTextures(1, &tdata->texture_v);
   1.185 +        }
   1.186 +        if (tdata->texture_u) {
   1.187 +            data->glDeleteTextures(1, &tdata->texture_u);
   1.188 +        }
   1.189          SDL_free(tdata->pixel_data);
   1.190          SDL_free(tdata);
   1.191          texture->driverdata = NULL;
   1.192 @@ -723,6 +833,10 @@
   1.193      /* Predetermine locations of uniform variables */
   1.194      entry->uniform_locations[GLES2_UNIFORM_PROJECTION] =
   1.195          data->glGetUniformLocation(entry->id, "u_projection");
   1.196 +    entry->uniform_locations[GLES2_UNIFORM_TEXTURE_V] =
   1.197 +        data->glGetUniformLocation(entry->id, "u_texture_v");
   1.198 +    entry->uniform_locations[GLES2_UNIFORM_TEXTURE_U] =
   1.199 +        data->glGetUniformLocation(entry->id, "u_texture_u");
   1.200      entry->uniform_locations[GLES2_UNIFORM_TEXTURE] =
   1.201          data->glGetUniformLocation(entry->id, "u_texture");
   1.202      entry->uniform_locations[GLES2_UNIFORM_MODULATION] =
   1.203 @@ -734,8 +848,10 @@
   1.204      entry->color_r = entry->color_g = entry->color_b = entry->color_a = 255;
   1.205  
   1.206      data->glUseProgram(entry->id);
   1.207 +    data->glUniform1i(entry->uniform_locations[GLES2_UNIFORM_TEXTURE_V], 2);  /* always texture unit 2. */
   1.208 +    data->glUniform1i(entry->uniform_locations[GLES2_UNIFORM_TEXTURE_U], 1);  /* always texture unit 1. */
   1.209 +    data->glUniform1i(entry->uniform_locations[GLES2_UNIFORM_TEXTURE], 0);  /* always texture unit 0. */
   1.210      data->glUniformMatrix4fv(entry->uniform_locations[GLES2_UNIFORM_PROJECTION], 1, GL_FALSE, (GLfloat *)entry->projection);
   1.211 -    data->glUniform1i(entry->uniform_locations[GLES2_UNIFORM_TEXTURE], 0);  /* always texture unit 0. */
   1.212      data->glUniform4f(entry->uniform_locations[GLES2_UNIFORM_MODULATION], 1.0f, 1.0f, 1.0f, 1.0f);
   1.213      data->glUniform4f(entry->uniform_locations[GLES2_UNIFORM_COLOR], 1.0f, 1.0f, 1.0f, 1.0f);
   1.214  
   1.215 @@ -927,6 +1043,9 @@
   1.216      case GLES2_IMAGESOURCE_TEXTURE_BGR:
   1.217          ftype = GLES2_SHADER_FRAGMENT_TEXTURE_BGR_SRC;
   1.218          break;
   1.219 +    case GLES2_IMAGESOURCE_TEXTURE_YUV:
   1.220 +        ftype = GLES2_SHADER_FRAGMENT_TEXTURE_YUV_SRC;
   1.221 +        break;
   1.222      default:
   1.223          goto fault;
   1.224      }
   1.225 @@ -1358,6 +1477,11 @@
   1.226              case SDL_PIXELFORMAT_RGB888:
   1.227                  sourceType = GLES2_IMAGESOURCE_TEXTURE_RGB;
   1.228                  break;
   1.229 +            // TODO: new shader to change yv planes YV12 format
   1.230 +            case SDL_PIXELFORMAT_IYUV:
   1.231 +            case SDL_PIXELFORMAT_YV12:
   1.232 +                sourceType = GLES2_IMAGESOURCE_TEXTURE_YUV;
   1.233 +                break;
   1.234              default:
   1.235                  return -1;
   1.236          }
   1.237 @@ -1368,6 +1492,15 @@
   1.238      }
   1.239  
   1.240      /* Select the target texture */
   1.241 +    if (tdata->yuv) {
   1.242 +        data->glActiveTexture(GL_TEXTURE2);
   1.243 +        data->glBindTexture(tdata->texture_type, tdata->texture_v);
   1.244 +
   1.245 +        data->glActiveTexture(GL_TEXTURE1);
   1.246 +        data->glBindTexture(tdata->texture_type, tdata->texture_u);
   1.247 +
   1.248 +        data->glActiveTexture(GL_TEXTURE0);
   1.249 +    }
   1.250      data->glBindTexture(tdata->texture_type, tdata->texture);
   1.251  
   1.252      /* Configure color modulation */
   1.253 @@ -1869,6 +2002,7 @@
   1.254      renderer->WindowEvent         = &GLES2_WindowEvent;
   1.255      renderer->CreateTexture       = &GLES2_CreateTexture;
   1.256      renderer->UpdateTexture       = &GLES2_UpdateTexture;
   1.257 +    renderer->UpdateTextureYUV    = &GLES2_UpdateTextureYUV;
   1.258      renderer->LockTexture         = &GLES2_LockTexture;
   1.259      renderer->UnlockTexture       = &GLES2_UnlockTexture;
   1.260      renderer->SetRenderTarget     = &GLES2_SetRenderTarget;
   1.261 @@ -1887,6 +2021,9 @@
   1.262      renderer->GL_BindTexture      = &GLES2_BindTexture;
   1.263      renderer->GL_UnbindTexture    = &GLES2_UnbindTexture;
   1.264  
   1.265 +    renderer->info.texture_formats[renderer->info.num_texture_formats++] = SDL_PIXELFORMAT_YV12;
   1.266 +    renderer->info.texture_formats[renderer->info.num_texture_formats++] = SDL_PIXELFORMAT_IYUV;
   1.267 +
   1.268      GLES2_ResetState(renderer);
   1.269  
   1.270      return renderer;
     2.1 --- a/src/render/opengles2/SDL_shaders_gles2.c	Fri Jun 06 18:33:17 2014 -0300
     2.2 +++ b/src/render/opengles2/SDL_shaders_gles2.c	Sat Jun 07 11:36:08 2014 -0700
     2.3 @@ -126,6 +126,30 @@
     2.4      } \
     2.5  ";
     2.6  
     2.7 +/* YUV to ABGR conversion */
     2.8 +static const Uint8 GLES2_FragmentSrc_TextureYUVSrc_[] = " \
     2.9 +    precision mediump float; \
    2.10 +    uniform sampler2D u_texture; \
    2.11 +    uniform sampler2D u_texture_u; \
    2.12 +    uniform sampler2D u_texture_v; \
    2.13 +    uniform vec4 u_modulation; \
    2.14 +    varying vec2 v_texCoord; \
    2.15 +    \
    2.16 +    void main() \
    2.17 +    { \
    2.18 +        mediump vec3 yuv; \
    2.19 +        lowp vec3 rgb; \
    2.20 +        yuv.x = texture2D(u_texture,   v_texCoord).r; \
    2.21 +        yuv.y = texture2D(u_texture_u, v_texCoord).r - 0.5; \
    2.22 +        yuv.z = texture2D(u_texture_v, v_texCoord).r - 0.5; \
    2.23 +        rgb = mat3( 1,        1,       1, \
    2.24 +                    0,       -0.39465, 2.03211, \
    2.25 +                    1.13983, -0.58060, 0) * yuv; \
    2.26 +        gl_FragColor = vec4(rgb, 1); \
    2.27 +        gl_FragColor *= u_modulation; \
    2.28 +    } \
    2.29 +";
    2.30 +
    2.31  static const GLES2_ShaderInstance GLES2_VertexSrc_Default = {
    2.32      GL_VERTEX_SHADER,
    2.33      GLES2_SOURCE_SHADER,
    2.34 @@ -168,6 +192,14 @@
    2.35      GLES2_FragmentSrc_TextureBGRSrc_
    2.36  };
    2.37  
    2.38 +static const GLES2_ShaderInstance GLES2_FragmentSrc_TextureYUVSrc = {
    2.39 +    GL_FRAGMENT_SHADER,
    2.40 +    GLES2_SOURCE_SHADER,
    2.41 +    sizeof(GLES2_FragmentSrc_TextureYUVSrc_),
    2.42 +    GLES2_FragmentSrc_TextureYUVSrc_
    2.43 +};
    2.44 +
    2.45 +
    2.46  /*************************************************************************************************
    2.47   * Vertex/fragment shader binaries (NVIDIA Tegra 1/2)                                            *
    2.48   *************************************************************************************************/
    2.49 @@ -692,6 +724,14 @@
    2.50      }
    2.51  };
    2.52  
    2.53 +static GLES2_Shader GLES2_FragmentShader_TextureYUVSrc = {
    2.54 +    1,
    2.55 +    {
    2.56 +        &GLES2_FragmentSrc_TextureYUVSrc
    2.57 +    }
    2.58 +};
    2.59 +
    2.60 +
    2.61  /*************************************************************************************************
    2.62   * Shader selector                                                                               *
    2.63   *************************************************************************************************/
    2.64 @@ -774,6 +814,11 @@
    2.65          default:
    2.66              return NULL;
    2.67      }
    2.68 +    
    2.69 +    case GLES2_SHADER_FRAGMENT_TEXTURE_YUV_SRC:
    2.70 +    {
    2.71 +        return &GLES2_FragmentShader_TextureYUVSrc;
    2.72 +    }
    2.73  
    2.74      default:
    2.75          return NULL;
     3.1 --- a/src/render/opengles2/SDL_shaders_gles2.h	Fri Jun 06 18:33:17 2014 -0300
     3.2 +++ b/src/render/opengles2/SDL_shaders_gles2.h	Sat Jun 07 11:36:08 2014 -0700
     3.3 @@ -46,7 +46,8 @@
     3.4      GLES2_SHADER_FRAGMENT_TEXTURE_ABGR_SRC,
     3.5      GLES2_SHADER_FRAGMENT_TEXTURE_ARGB_SRC,
     3.6      GLES2_SHADER_FRAGMENT_TEXTURE_BGR_SRC,
     3.7 -    GLES2_SHADER_FRAGMENT_TEXTURE_RGB_SRC
     3.8 +    GLES2_SHADER_FRAGMENT_TEXTURE_RGB_SRC,
     3.9 +    GLES2_SHADER_FRAGMENT_TEXTURE_YUV_SRC
    3.10  } GLES2_ShaderType;
    3.11  
    3.12  #define GLES2_SOURCE_SHADER (GLenum)-1