Fix for CVE-2008-5236.

Multiple heap-based buffer overflows in xine-lib 1.1.12, and other
1.1.15 and earlier versions, allow remote attackers to execute
arbitrary code via vectors related to (1) a crafted EBML element
length processed by the parse_block_group function in
demux_matroska.c; (2) a certain combination of sps, w, and h values
processed by the real_parse_audio_specific_data and
demux_real_send_chunk functions in demux_real.c; and (3) an
unspecified combination of three values processed by the open_ra_file
function in demux_realaudio.c.  NOTE: vector 2 reportedly exists
because of an incomplete fix in 1.1.15.

3 files changed, 26 insertions, 8 deletions

diff --git a/src/demuxers/demux_matroska.c b/src/demuxers/demux_matroska.c
index a8af6c44f..6bd700f9a 100644
--- a/src/demuxers/demux_matroska.c
+++ b/src/demuxers/demux_matroska.c
@@ -1179,7 +1179,12 @@ static int parse_track_entry(demux_matroska_t *this, matroska_track_t *track) {
       break;
         
       case MATROSKA_ID_TR_CODECPRIVATE: {
-        char *codec_private = malloc (elem.len);
+        char *codec_private;
+	if (elem.len >= 0x80000000)
+	  return 0;
+        codec_private = malloc (elem.len);
+	if (! codec_private)
+	  return 0;
         lprintf("CodecPrivate\n");
         if (!ebml_read_binary(ebml, &elem, codec_private)) {
 	  free(codec_private);
diff --git a/src/demuxers/demux_real.c b/src/demuxers/demux_real.c
index 32b516537..965470125 100644
--- a/src/demuxers/demux_real.c
+++ b/src/demuxers/demux_real.c
@@ -359,9 +359,14 @@ static void real_parse_audio_specific_data (demux_real_t *this,
    *   stream->frame_size      = stream->w / stream->sps * stream->h * stream->sps;
    * but it looks pointless? the compiler will probably optimise it away, I suppose?
    */
-  stream->frame_size = stream->w * stream->h;
+  if (stream->w < 32768 && stream->h < 32768) {
+    stream->frame_size = stream->w * stream->h;
+    stream->frame_buffer = calloc(stream->frame_size, 1);
+  } else {
+    stream->frame_size = 0;
+    stream->frame_buffer = NULL;
+  }
 
-  stream->frame_buffer = calloc(stream->frame_size, 1);
   stream->frame_num_bytes = 0;
   stream->sub_packet_cnt = 0;
 
diff --git a/src/demuxers/demux_realaudio.c b/src/demuxers/demux_realaudio.c
index 1c39c6208..44449667c 100644
--- a/src/demuxers/demux_realaudio.c
+++ b/src/demuxers/demux_realaudio.c
@@ -202,11 +202,19 @@ static int open_ra_file(demux_ra_t *this) {
     this->h           = _X_BE_16 (this->header+40);
     this->cfs         = _X_BE_32 (this->header+24);
 
-    this->frame_len = this->w * this->h;
-    this->frame_size = this->frame_len * sps;
-
-    this->frame_buffer = calloc(this->frame_size, 1);
-    _x_assert(this->frame_buffer != NULL);
+    if (this->w < 0x8000 && this->h < 0x8000) {
+      uint64_t fs;
+      this->frame_len = this->w * this->h;
+      fs = (uint64_t) this->frame_len * sps;
+      if (fs < 0x80000000) {
+        this->frame_size = fs;
+        this->frame_buffer = calloc(this->frame_size, 1);
+      }
+    }
+    if (! this->frame_buffer) {
+      xprintf(this->stream->xine, XINE_VERBOSITY_DEBUG, "demux_realaudio: malloc failed\n");
+      return 0;
+    }
 
     if (this->audio_type == BUF_AUDIO_28_8 || this->audio_type == BUF_AUDIO_SIPRO)
       this->block_align = this->cfs;