libv4l: dont try to allocate large buffers on the stack

From: Hans de Goede <hdegoede@redhat.com>

When conversion requires multiple passes don't alloc the needed temporary
buffer on the stack, as some apps (ekiga) use so much stack themselves
this causes us to run out of stack space

Priority: normal

Signed-off-by: Hans de Goede <hdegoede@redhat.com>

4 files changed, 62 insertions, 9 deletions

diff --git a/v4l2-apps/libv4l/ChangeLog b/v4l2-apps/libv4l/ChangeLog
index fa18321f4..f092cc418 100644
--- a/v4l2-apps/libv4l/ChangeLog
+++ b/v4l2-apps/libv4l/ChangeLog
@@ -1,3 +1,9 @@
+libv4l-0.5.3
+------------
+* When conversion requires multiple passes don't alloc the needed temporary
+  buffer on the stack, as some apps (ekiga) use so much stack themselves
+  this causes us to run out of stack space
+
 libv4l-0.5.2
 ------------
 * Add Philips SPC210NC to list of cams with upside down sensor, reported by
diff --git a/v4l2-apps/libv4l/Makefile b/v4l2-apps/libv4l/Makefile
index bfc31a185..8de10a4cb 100644
--- a/v4l2-apps/libv4l/Makefile
+++ b/v4l2-apps/libv4l/Makefile
@@ -1,5 +1,5 @@
 LIB_RELEASE=0
-V4L2_LIB_VERSION=$(LIB_RELEASE).5.2
+V4L2_LIB_VERSION=$(LIB_RELEASE).5.3
 
 all clean install:
 	$(MAKE) -C libv4lconvert V4L2_LIB_VERSION=$(V4L2_LIB_VERSION) $@
diff --git a/v4l2-apps/libv4l/libv4lconvert/libv4lconvert-priv.h b/v4l2-apps/libv4l/libv4lconvert/libv4lconvert-priv.h
index 99ffdec20..8473ba68f 100644
--- a/v4l2-apps/libv4l/libv4lconvert/libv4lconvert-priv.h
+++ b/v4l2-apps/libv4l/libv4lconvert/libv4lconvert-priv.h
@@ -86,6 +86,12 @@ struct v4lconvert_data {
   struct jdec_private *jdec;
   struct v4l2_frmsizeenum framesizes[V4LCONVERT_MAX_FRAMESIZES];
   unsigned int no_framesizes;
+  int convert_buf_size;
+  int rotate_buf_size;
+  int convert_pixfmt_buf_size;
+  unsigned char *convert_buf;
+  unsigned char *rotate_buf;
+  unsigned char *convert_pixfmt_buf;
 };
 
 struct v4lconvert_flags_info {
diff --git a/v4l2-apps/libv4l/libv4lconvert/libv4lconvert.c b/v4l2-apps/libv4l/libv4lconvert/libv4lconvert.c
index 4725e638f..ee6ef33a0 100644
--- a/v4l2-apps/libv4l/libv4lconvert/libv4lconvert.c
+++ b/v4l2-apps/libv4l/libv4lconvert/libv4lconvert.c
@@ -94,7 +94,6 @@ struct v4lconvert_data *v4lconvert_create(int fd)
     return NULL;
 
   data->fd = fd;
-  data->jdec = NULL;
 
   /* Check supported formats */
   for (i = 0; ; i++) {
@@ -135,6 +134,9 @@ void v4lconvert_destroy(struct v4lconvert_data *data)
     tinyjpeg_set_components(data->jdec, comps, 3);
     tinyjpeg_free(data->jdec);
   }
+  free(data->convert_buf);
+  free(data->rotate_buf);
+  free(data->convert_pixfmt_buf);
   free(data);
 }
 
@@ -333,6 +335,23 @@ int v4lconvert_needs_conversion(struct v4lconvert_data *data,
     return 1; /* Needs flip and thus conversion */
 }
 
+static unsigned char *v4lconvert_alloc_buffer(struct v4lconvert_data *data,
+  int needed, unsigned char **buf, int *buf_size)
+{
+  if (*buf_size < needed) {
+    free(*buf);
+    *buf = malloc(needed);
+    if (*buf == NULL) {
+      *buf_size = 0;
+      V4LCONVERT_ERR("could not allocate memory\n");
+      errno = ENOMEM;
+      return NULL;
+    }
+    *buf_size = needed;
+  }
+  return *buf;
+}
+
 static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
   unsigned int src_pix_fmt, unsigned int dest_pix_fmt,
   unsigned int width, unsigned int height,
@@ -444,8 +463,15 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
     case V4L2_PIX_FMT_SPCA505:
     case V4L2_PIX_FMT_SPCA508:
     {
-      unsigned char tmpbuf[width * height * 3 / 2];
-      unsigned char *d = (dest_pix_fmt != V4L2_PIX_FMT_YUV420) ? tmpbuf : dest;
+      unsigned char *d;
+
+      if (dest_pix_fmt != V4L2_PIX_FMT_YUV420) {
+	d = v4lconvert_alloc_buffer(data, width * height * 3 / 2,
+	      &data->convert_pixfmt_buf, &data->convert_pixfmt_buf_size);
+	if (!d)
+	  return -1;
+      } else
+	d = dest;
 
       switch (src_pix_fmt) {
 	case V4L2_PIX_FMT_SPCA501:
@@ -461,10 +487,12 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
 
       switch (dest_pix_fmt) {
       case V4L2_PIX_FMT_RGB24:
-	v4lconvert_yuv420_to_rgb24(tmpbuf, dest, width, height);
+	v4lconvert_yuv420_to_rgb24(data->convert_pixfmt_buf, dest, width,
+				   height);
 	break;
       case V4L2_PIX_FMT_BGR24:
-	v4lconvert_yuv420_to_bgr24(tmpbuf, dest, width, height);
+	v4lconvert_yuv420_to_bgr24(data->convert_pixfmt_buf, dest, width,
+				   height);
 	break;
       }
       break;
@@ -475,9 +503,14 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
     case V4L2_PIX_FMT_SN9C10X:
     case V4L2_PIX_FMT_PAC207:
     {
-      unsigned char tmpbuf[width * height];
+      unsigned char *tmpbuf;
       unsigned int bayer_fmt = 0;
 
+      tmpbuf = v4lconvert_alloc_buffer(data, width * height,
+	    &data->convert_pixfmt_buf, &data->convert_pixfmt_buf_size);
+      if (!tmpbuf)
+	return -1;
+
       switch (src_pix_fmt) {
 	case V4L2_PIX_FMT_SPCA561:
 	  v4lconvert_decode_spca561(src, tmpbuf, width, height);
@@ -642,12 +675,20 @@ int v4lconvert_convert(struct v4lconvert_data *data,
 
   /* convert_pixfmt -> rotate -> crop, all steps are optional */
   if (convert && (rotate || crop)) {
-    convert_dest = alloca(temp_needed);
+    convert_dest = v4lconvert_alloc_buffer(data, temp_needed,
+		     &data->convert_buf, &data->convert_buf_size);
+    if (!convert_dest)
+      return -1;
+
     rotate_src = crop_src = convert_dest;
   }
 
   if (rotate && crop) {
-    rotate_dest = alloca(temp_needed);
+    rotate_dest = v4lconvert_alloc_buffer(data, temp_needed,
+		    &data->rotate_buf, &data->rotate_buf_size);
+    if (!rotate_dest)
+      return -1;
+
     crop_src = rotate_dest;
   }