Factorize rle uncompression code from raw, vaapi and opengl2 drivers.

Fix highlight area and buffer overflows.

4 files changed, 130 insertions, 137 deletions

diff --git a/src/video_out/video_out_opengl2.c b/src/video_out/video_out_opengl2.c
index 4de8ccfec..c0f0b064a 100644
--- a/src/video_out/video_out_opengl2.c
+++ b/src/video_out/video_out_opengl2.c
@@ -540,52 +540,9 @@ static int opengl2_process_ovl( opengl2_driver_t *this_gen, vo_overlay_t *overla
     ovl->vid_scale = 1;
   else
     ovl->vid_scale = 0;
-  ovl->type = GL_RGBA;
-
-  int num_rle = overlay->num_rle;
-  rle_elem_t *rle = overlay->rle;
-  uint8_t *rgba = ovl->ovl_rgba;
-  clut_t *low_colors = (clut_t*)overlay->color;
-  clut_t *hili_colors = (clut_t*)overlay->hili_color;
-  uint8_t *low_trans = overlay->trans;
-  uint8_t *hili_trans = overlay->hili_trans;
-  clut_t *colors;
-  uint8_t *trans;
-  uint8_t alpha;
-  int rlelen = 0;
-  uint8_t clr = 0;
-  int i, pos=0, x, y;
-
-  while ( num_rle>0 ) {
-    x = pos%ovl->ovl_w;
-    y = pos/ovl->ovl_w;
-    if ( (x>=overlay->hili_left && x<=overlay->hili_right) && (y>=overlay->hili_top && y<=overlay->hili_bottom) ) {
-    colors = hili_colors;
-    trans = hili_trans;
-    }
-    else {
-    colors = low_colors;
-    trans = low_trans;
-    }
-    rlelen = rle->len;
-    clr = rle->color;
-    alpha = trans[clr];
-    for ( i=0; i<rlelen; ++i ) {
-      if ( alpha == 0 ) {
-        rgba[0] = rgba[1] = rgba[2] = rgba[3] = 0;
-      }
-      else {
-        rgba[0] = colors[clr].y;
-        rgba[1] = colors[clr].cr;
-        rgba[2] = colors[clr].cb;
-        rgba[3] = alpha*255/15;
-      }
-      rgba+= 4;
-    ++pos;
-    }
-    ++rle;
-    --num_rle;
-  }
+  ovl->type = GL_BGRA;
+
+  _x_overlay_to_argb32(overlay, (uint32_t*)ovl->ovl_rgba, overlay->width, "BGRA");
   return 1;
 }
 
diff --git a/src/video_out/video_out_raw.c b/src/video_out/video_out_raw.c
index 702c7dc2e..a19285707 100644
--- a/src/video_out/video_out_raw.c
+++ b/src/video_out/video_out_raw.c
@@ -115,50 +115,7 @@ static int raw_process_ovl( raw_driver_t *this_gen, vo_overlay_t *overlay )
   ovl->ovl_x = overlay->x;
   ovl->ovl_y = overlay->y;
 
-  int num_rle = overlay->num_rle;
-  rle_elem_t *rle = overlay->rle;
-  uint8_t *rgba = ovl->ovl_rgba;
-  clut_t *low_colors = (clut_t*)overlay->color;
-  clut_t *hili_colors = (clut_t*)overlay->hili_color;
-  uint8_t *low_trans = overlay->trans;
-  uint8_t *hili_trans = overlay->hili_trans;
-  clut_t *colors;
-  uint8_t *trans;
-  uint8_t alpha;
-  int rlelen = 0;
-  uint8_t clr = 0;
-  int i, pos=0, x, y;
-
-  while ( num_rle>0 ) {
-    x = pos%ovl->ovl_w;
-    y = pos/ovl->ovl_w;
-    if ( (x>=overlay->hili_left && x<=overlay->hili_right) && (y>=overlay->hili_top && y<=overlay->hili_bottom) ) {
-	colors = hili_colors;
-	trans = hili_trans;
-    }
-    else {
-	colors = low_colors;
-	trans = low_trans;
-    }
-    rlelen = rle->len;
-    clr = rle->color;
-    alpha = trans[clr];
-    for ( i=0; i<rlelen; ++i ) {
-      if ( alpha == 0 ) {
-        rgba[0] = rgba[1] = rgba[2] = rgba[3] = 0;
-      }
-      else {
-        rgba[0] = colors[clr].y;
-        rgba[1] = colors[clr].cr;
-        rgba[2] = colors[clr].cb;
-        rgba[3] = alpha*255/15;
-      }
-      rgba+= 4;
-	++pos;
-    }
-    ++rle;
-    --num_rle;
-  }
+  _x_overlay_to_argb32(overlay, (uint32_t*)ovl->ovl_rgba, overlay->width, "RGBA");
   return 1;
 }
 
diff --git a/src/video_out/video_out_vaapi.c b/src/video_out/video_out_vaapi.c
index af759c2ff..823d0fc5a 100644
--- a/src/video_out/video_out_vaapi.c
+++ b/src/video_out/video_out_vaapi.c
@@ -2717,7 +2717,6 @@ static void vaapi_overlay_end (vo_driver_t *this_gen, vo_frame_t *frame_gen) {
   for (i = 0; i < novls; ++i) {
     vo_overlay_t *ovl = this->overlays[i];
     uint32_t *bitmap = NULL;
-    uint32_t *rgba = NULL;
 
     if (ovl->rle) {
       if(ovl->width<=0 || ovl->height<=0)
@@ -2726,52 +2725,10 @@ static void vaapi_overlay_end (vo_driver_t *this_gen, vo_frame_t *frame_gen) {
       if (!ovl->rgb_clut || !ovl->hili_rgb_clut)
         _x_overlay_clut_yuv2rgb (ovl);
 
-      bitmap = rgba = calloc(ovl->width * ovl->height * 4, sizeof(uint32_t));
-
-      int num_rle = ovl->num_rle;
-      rle_elem_t *rle = ovl->rle;
-      uint32_t red, green, blue, alpha;
-      clut_t *low_colors = (clut_t*)ovl->color;
-      clut_t *hili_colors = (clut_t*)ovl->hili_color;
-      uint8_t *low_trans = ovl->trans;
-      uint8_t *hili_trans = ovl->hili_trans;
-      clut_t *colors;
-      uint8_t *trans;
-      int rlelen = 0;
-      uint8_t clr = 0;
-      int i, pos=0, x, y;
-
-      while (num_rle > 0) {
-        x = pos % ovl->width;
-        y = pos / ovl->width;
-
-        if ( (x>=ovl->hili_left && x<=ovl->hili_right) && (y>=ovl->hili_top && y<=ovl->hili_bottom) ) {
-          colors = hili_colors;
-          trans = hili_trans;
-        }
-        else {
-          colors = low_colors;
-          trans = low_trans;
-        }
-        rlelen = rle->len;
-        clr = rle->color;
-        for ( i=0; i<rlelen; ++i ) {
-          if ( trans[clr] == 0 ) {
-            alpha = red = green = blue = 0;
-          }
-          else {
-            red = colors[clr].y; // red
-            green = colors[clr].cr; // green
-            blue = colors[clr].cb; // blue
-            alpha = trans[clr]*255/15;
-          }
-          *rgba = (alpha<<24) | (red<<16) | (green<<8) | blue;
-          rgba++;
-          ++pos;
-        }
-        ++rle;
-        --num_rle;
-      }
+      bitmap = malloc(ovl->width * ovl->height * sizeof(uint32_t));
+
+      _x_overlay_to_argb32(ovl, bitmap, ovl->width, "BGRA");
+
       lprintf("width %d height %d pos %d %d\n", ovl->width, ovl->height, pos, ovl->width * ovl->height);
     } else {
       pthread_mutex_lock(&ovl->argb_layer->mutex);
diff --git a/src/xine-engine/video_overlay.c b/src/xine-engine/video_overlay.c
index c45396232..e044d9d90 100644
--- a/src/xine-engine/video_overlay.c
+++ b/src/xine-engine/video_overlay.c
@@ -507,6 +507,128 @@ void _x_overlay_clut_yuv2rgb(vo_overlay_t *overlay)
   }
 }
 
+static void clut_to_argb(const uint32_t *color, const uint8_t *trans, int num_items, uint32_t *argb, const char *format)
+{
+  int i;
+  union {
+    uint32_t u32;
+    clut_t   c;
+  } tmp ;
+
+  if (!strcmp(format, "BGRA")) {
+    for (i = 0; i < num_items; i++) {
+      tmp.u32 = color[i];
+      uint8_t *rgba = (uint8_t*)(argb + i);
+      rgba[0] = tmp.c.cb;
+      rgba[1] = tmp.c.cr;
+      rgba[2] = tmp.c.y;
+      rgba[3] = trans[i] * 255 / 15;
+    }
+  }
+  else if (!strcmp(format, "RGBA")) {
+    for (i = 0; i < num_items; i++) {
+      tmp.u32 = color[i];
+      uint8_t *rgba = (uint8_t*)(argb + i);
+      rgba[0] = tmp.c.y;
+      rgba[1] = tmp.c.cr;
+      rgba[2] = tmp.c.cb;
+      rgba[3] = trans[i] * 255 / 15;
+    }
+  }
+  else {
+    fprintf(stderr, "clut_to_argb: unknown format %s\n", format);
+  }
+}
+
+#define LUT_SIZE (sizeof(overlay->color)/sizeof(overlay->color[0]))
+#define NEXT_BITE                                                       \
+  do {                                                                  \
+    if (rle_len < 1) {                                                  \
+      rle++;                                                            \
+      if (rle >= rle_end) {                                             \
+        /* fill with transparent */                                     \
+        memset(rgba, 0, (overlay->width - x) * sizeof(uint32_t));       \
+        rgba += stride;                                                 \
+        for (; y < overlay->height; y++, rgba += stride) {              \
+          memset(rgba, 0, stride * sizeof(uint32_t));                   \
+        }                                                               \
+        return;                                                         \
+      }                                                                 \
+      rle_len = rle->len;                                               \
+    }                                                                   \
+  } while (0)
+#define LIMIT_WIDTH                             \
+  do {                                          \
+    x_limit = x + rle_len;                      \
+    if (x_limit > overlay->width) {             \
+      rle_len = x_limit - overlay->width;       \
+      x_limit = overlay->width;                 \
+    } else {                                    \
+      rle_len = 0;                              \
+    }                                           \
+  } while (0)
+#define BLEND_LINE                              \
+  do {                                          \
+    for (x = 0; x < overlay->width; ) {         \
+      NEXT_BITE;                                \
+      LIMIT_WIDTH;                              \
+                                                \
+      while (x < x_limit) {                     \
+        rgba[x++] = colors[rle->color];         \
+      }                                         \
+    }                                           \
+    rgba += stride;                             \
+  } while (0)
+
+void _x_overlay_to_argb32(const vo_overlay_t *overlay, uint32_t *rgba, int stride, const char *format)
+{
+  const rle_elem_t *rle_end = overlay->rle + overlay->num_rle;
+  const rle_elem_t *rle = overlay->rle;
+  int x, y, x_limit;
+  int rle_len = rle->len;
+  int no_hili = overlay->hili_bottom < 0 || overlay->hili_bottom < overlay->hili_top ||
+                overlay->hili_right  < 0 || overlay->hili_right  < overlay->hili_left;
+
+  if (overlay->num_rle < 1)
+    return;
+
+  if (no_hili) {
+    uint32_t colors[LUT_SIZE];
+    clut_to_argb(overlay->color, overlay->trans, LUT_SIZE, colors, format);
+
+    for (y = 0; y < overlay->height; y++) {
+      BLEND_LINE;
+    }
+
+  } else {
+    uint32_t colors[LUT_SIZE * 2];
+    clut_to_argb(overlay->color,      overlay->trans,      LUT_SIZE, colors,            format);
+    clut_to_argb(overlay->hili_color, overlay->hili_trans, LUT_SIZE, colors + LUT_SIZE, format);
+
+    for (y = 0; y < overlay->height; y++) {
+      int hili_y = (y >= overlay->hili_top && y <= overlay->hili_bottom);
+      if (!hili_y) {
+        BLEND_LINE;
+      } else {
+        for (x = 0; x < overlay->width; ) {
+          NEXT_BITE;
+          LIMIT_WIDTH;
+
+          while (x < x_limit) {
+            int hili = (x >= overlay->hili_left && x <= overlay->hili_right);
+            rgba[x++] = colors[rle->color + hili * LUT_SIZE];
+          }
+        }
+        rgba += stride;
+      }
+    }
+  }
+}
+#undef LUT_SIZE
+#undef NEXT_BITE
+#undef LIMIT_WIDTH
+#undef BLEND_LINE
+
 /* This is called from video_out.c
  * must call output->overlay_blend for each active overlay.
  */