Factorize rle uncompression code from raw, vaapi and opengl2 drivers.

Fix highlight area and buffer overflows.

3 files changed, 8 insertions, 137 deletions

diff --git a/src/video_out/video_out_opengl2.c b/src/video_out/video_out_opengl2.c
index 4de8ccfec..c0f0b064a 100644
--- a/src/video_out/video_out_opengl2.c
+++ b/src/video_out/video_out_opengl2.c
@@ -540,52 +540,9 @@ static int opengl2_process_ovl( opengl2_driver_t *this_gen, vo_overlay_t *overla
     ovl->vid_scale = 1;
   else
     ovl->vid_scale = 0;
-  ovl->type = GL_RGBA;
-
-  int num_rle = overlay->num_rle;
-  rle_elem_t *rle = overlay->rle;
-  uint8_t *rgba = ovl->ovl_rgba;
-  clut_t *low_colors = (clut_t*)overlay->color;
-  clut_t *hili_colors = (clut_t*)overlay->hili_color;
-  uint8_t *low_trans = overlay->trans;
-  uint8_t *hili_trans = overlay->hili_trans;
-  clut_t *colors;
-  uint8_t *trans;
-  uint8_t alpha;
-  int rlelen = 0;
-  uint8_t clr = 0;
-  int i, pos=0, x, y;
-
-  while ( num_rle>0 ) {
-    x = pos%ovl->ovl_w;
-    y = pos/ovl->ovl_w;
-    if ( (x>=overlay->hili_left && x<=overlay->hili_right) && (y>=overlay->hili_top && y<=overlay->hili_bottom) ) {
-    colors = hili_colors;
-    trans = hili_trans;
-    }
-    else {
-    colors = low_colors;
-    trans = low_trans;
-    }
-    rlelen = rle->len;
-    clr = rle->color;
-    alpha = trans[clr];
-    for ( i=0; i<rlelen; ++i ) {
-      if ( alpha == 0 ) {
-        rgba[0] = rgba[1] = rgba[2] = rgba[3] = 0;
-      }
-      else {
-        rgba[0] = colors[clr].y;
-        rgba[1] = colors[clr].cr;
-        rgba[2] = colors[clr].cb;
-        rgba[3] = alpha*255/15;
-      }
-      rgba+= 4;
-    ++pos;
-    }
-    ++rle;
-    --num_rle;
-  }
+  ovl->type = GL_BGRA;
+
+  _x_overlay_to_argb32(overlay, (uint32_t*)ovl->ovl_rgba, overlay->width, "BGRA");
   return 1;
 }
 
diff --git a/src/video_out/video_out_raw.c b/src/video_out/video_out_raw.c
index 702c7dc2e..a19285707 100644
--- a/src/video_out/video_out_raw.c
+++ b/src/video_out/video_out_raw.c
@@ -115,50 +115,7 @@ static int raw_process_ovl( raw_driver_t *this_gen, vo_overlay_t *overlay )
   ovl->ovl_x = overlay->x;
   ovl->ovl_y = overlay->y;
 
-  int num_rle = overlay->num_rle;
-  rle_elem_t *rle = overlay->rle;
-  uint8_t *rgba = ovl->ovl_rgba;
-  clut_t *low_colors = (clut_t*)overlay->color;
-  clut_t *hili_colors = (clut_t*)overlay->hili_color;
-  uint8_t *low_trans = overlay->trans;
-  uint8_t *hili_trans = overlay->hili_trans;
-  clut_t *colors;
-  uint8_t *trans;
-  uint8_t alpha;
-  int rlelen = 0;
-  uint8_t clr = 0;
-  int i, pos=0, x, y;
-
-  while ( num_rle>0 ) {
-    x = pos%ovl->ovl_w;
-    y = pos/ovl->ovl_w;
-    if ( (x>=overlay->hili_left && x<=overlay->hili_right) && (y>=overlay->hili_top && y<=overlay->hili_bottom) ) {
-	colors = hili_colors;
-	trans = hili_trans;
-    }
-    else {
-	colors = low_colors;
-	trans = low_trans;
-    }
-    rlelen = rle->len;
-    clr = rle->color;
-    alpha = trans[clr];
-    for ( i=0; i<rlelen; ++i ) {
-      if ( alpha == 0 ) {
-        rgba[0] = rgba[1] = rgba[2] = rgba[3] = 0;
-      }
-      else {
-        rgba[0] = colors[clr].y;
-        rgba[1] = colors[clr].cr;
-        rgba[2] = colors[clr].cb;
-        rgba[3] = alpha*255/15;
-      }
-      rgba+= 4;
-	++pos;
-    }
-    ++rle;
-    --num_rle;
-  }
+  _x_overlay_to_argb32(overlay, (uint32_t*)ovl->ovl_rgba, overlay->width, "RGBA");
   return 1;
 }
 
diff --git a/src/video_out/video_out_vaapi.c b/src/video_out/video_out_vaapi.c
index af759c2ff..823d0fc5a 100644
--- a/src/video_out/video_out_vaapi.c
+++ b/src/video_out/video_out_vaapi.c
@@ -2717,7 +2717,6 @@ static void vaapi_overlay_end (vo_driver_t *this_gen, vo_frame_t *frame_gen) {
   for (i = 0; i < novls; ++i) {
     vo_overlay_t *ovl = this->overlays[i];
     uint32_t *bitmap = NULL;
-    uint32_t *rgba = NULL;
 
     if (ovl->rle) {
       if(ovl->width<=0 || ovl->height<=0)
@@ -2726,52 +2725,10 @@ static void vaapi_overlay_end (vo_driver_t *this_gen, vo_frame_t *frame_gen) {
       if (!ovl->rgb_clut || !ovl->hili_rgb_clut)
         _x_overlay_clut_yuv2rgb (ovl);
 
-      bitmap = rgba = calloc(ovl->width * ovl->height * 4, sizeof(uint32_t));
-
-      int num_rle = ovl->num_rle;
-      rle_elem_t *rle = ovl->rle;
-      uint32_t red, green, blue, alpha;
-      clut_t *low_colors = (clut_t*)ovl->color;
-      clut_t *hili_colors = (clut_t*)ovl->hili_color;
-      uint8_t *low_trans = ovl->trans;
-      uint8_t *hili_trans = ovl->hili_trans;
-      clut_t *colors;
-      uint8_t *trans;
-      int rlelen = 0;
-      uint8_t clr = 0;
-      int i, pos=0, x, y;
-
-      while (num_rle > 0) {
-        x = pos % ovl->width;
-        y = pos / ovl->width;
-
-        if ( (x>=ovl->hili_left && x<=ovl->hili_right) && (y>=ovl->hili_top && y<=ovl->hili_bottom) ) {
-          colors = hili_colors;
-          trans = hili_trans;
-        }
-        else {
-          colors = low_colors;
-          trans = low_trans;
-        }
-        rlelen = rle->len;
-        clr = rle->color;
-        for ( i=0; i<rlelen; ++i ) {
-          if ( trans[clr] == 0 ) {
-            alpha = red = green = blue = 0;
-          }
-          else {
-            red = colors[clr].y; // red
-            green = colors[clr].cr; // green
-            blue = colors[clr].cb; // blue
-            alpha = trans[clr]*255/15;
-          }
-          *rgba = (alpha<<24) | (red<<16) | (green<<8) | blue;
-          rgba++;
-          ++pos;
-        }
-        ++rle;
-        --num_rle;
-      }
+      bitmap = malloc(ovl->width * ovl->height * sizeof(uint32_t));
+
+      _x_overlay_to_argb32(ovl, bitmap, ovl->width, "BGRA");
+
       lprintf("width %d height %d pos %d %d\n", ovl->width, ovl->height, pos, ovl->width * ovl->height);
     } else {
       pthread_mutex_lock(&ovl->argb_layer->mutex);