From 7da363097cc2ce5c13d4799c7c0523882976b611 Mon Sep 17 00:00:00 2001 From: Christophe Thommeret Date: Fri, 21 May 2010 11:39:52 +0200 Subject: Bitstream buffers overflow prevention in vdpau decoders. --- src/video_dec/libvdpau/bits_reader.h | 54 ++++++++++++++++++++++++++++++++--- src/video_dec/libvdpau/vdpau_mpeg12.c | 18 ++++++------ src/video_dec/libvdpau/vdpau_vc1.c | 34 +++++++++++----------- 3 files changed, 76 insertions(+), 30 deletions(-) diff --git a/src/video_dec/libvdpau/bits_reader.h b/src/video_dec/libvdpau/bits_reader.h index 9563c9d3b..db7cdfc7e 100644 --- a/src/video_dec/libvdpau/bits_reader.h +++ b/src/video_dec/libvdpau/bits_reader.h @@ -3,16 +3,22 @@ typedef struct { - uint8_t *buffer; - int offbits; + uint8_t *buffer, *start; + int offbits, length, oflow; } bits_reader_t; -static void bits_reader_set( bits_reader_t *br, uint8_t *buf ) + + +static void bits_reader_set( bits_reader_t *br, uint8_t *buf, int len ) { - br->buffer = buf; + br->buffer = br->start = buf; br->offbits = 0; + br->length = len; + br->oflow = 0; } + + static uint32_t read_bits( bits_reader_t *br, int nbits ) { int i, nbytes; @@ -23,6 +29,10 @@ static uint32_t read_bits( bits_reader_t *br, int nbits ) nbytes = (br->offbits + nbits)/8; if ( ((br->offbits + nbits) %8 ) > 0 ) nbytes++; + if ( (buf + nbytes) > (br->start + br->length) ) { + br->oflow = 1; + return 0; + } for ( i=0; ioffbits; @@ -34,3 +44,39 @@ static uint32_t read_bits( bits_reader_t *br, int nbits ) return ret; } + + + +static void skip_bits( bits_reader_t *br, int nbits ) +{ + br->offbits += nbits; + br->buffer += br->offbits / 8; + br->offbits %= 8; + if ( br->buffer > (br->start + br->length) ) { + br->oflow = 1; + } +} + + + +static uint32_t get_bits( bits_reader_t *br, int nbits ) +{ + int i, nbytes; + uint32_t ret = 0; + uint8_t *buf; + + buf = br->buffer; + nbytes = (br->offbits + nbits)/8; + if ( ((br->offbits + nbits) %8 ) > 0 ) + nbytes++; + if ( (buf + nbytes) > (br->start + br->length) ) { + br->oflow = 1; + return 0; + } + for ( i=0; ioffbits; + ret = ((ret<>i)>>((nbytes*8)-nbits-br->offbits); + + return ret; +} diff --git a/src/video_dec/libvdpau/vdpau_mpeg12.c b/src/video_dec/libvdpau/vdpau_mpeg12.c index 1ae7a06aa..f55b756b1 100644 --- a/src/video_dec/libvdpau/vdpau_mpeg12.c +++ b/src/video_dec/libvdpau/vdpau_mpeg12.c @@ -263,7 +263,7 @@ static void sequence_header( vdpau_mpeg12_decoder_t *this_gen, uint8_t *buf, int sequence->cur_pts = 0; } - bits_reader_set( &sequence->br, buf ); + bits_reader_set( &sequence->br, buf, len ); sequence->coded_width = read_bits( &sequence->br, 12 ); lprintf( "coded_width: %d\n", sequence->coded_width ); sequence->coded_height = read_bits( &sequence->br, 12 ); @@ -370,14 +370,14 @@ static void picture_header( sequence_t *sequence, uint8_t *buf, int len ) infos = &sequence->picture.vdp_infos2; } - bits_reader_set( &sequence->br, buf ); + bits_reader_set( &sequence->br, buf, len ); int tmp = read_bits( &sequence->br, 10 ); lprintf( "temporal_reference: %d\n", tmp ); infos->picture_coding_type = read_bits( &sequence->br, 3 ); lprintf( "picture_coding_type: %d\n", infos->picture_coding_type ); infos->forward_reference = VDP_INVALID_HANDLE; infos->backward_reference = VDP_INVALID_HANDLE; - read_bits( &sequence->br, 16 ); + skip_bits( &sequence->br, 16 ); if ( infos->picture_coding_type > I_FRAME ) { infos->full_pel_forward_vector = read_bits( &sequence->br, 1 ); infos->f_code[0][0] = infos->f_code[0][1] = read_bits( &sequence->br, 3 ); @@ -400,15 +400,15 @@ static void picture_header( sequence_t *sequence, uint8_t *buf, int len ) static void sequence_extension( sequence_t *sequence, uint8_t *buf, int len ) { - bits_reader_set( &sequence->br, buf ); + bits_reader_set( &sequence->br, buf, len ); int tmp = read_bits( &sequence->br, 4 ); lprintf( "extension_start_code_identifier: %d\n", tmp ); - read_bits( &sequence->br, 1 ); + skip_bits( &sequence->br, 1 ); switch ( read_bits( &sequence->br, 3 ) ) { case 5: sequence->profile = VDP_DECODER_PROFILE_MPEG2_SIMPLE; break; default: sequence->profile = VDP_DECODER_PROFILE_MPEG2_MAIN; } - read_bits( &sequence->br, 4 ); + skip_bits( &sequence->br, 4 ); tmp = read_bits( &sequence->br, 1 ); lprintf( "progressive_sequence: %d\n", tmp ); if ( read_bits( &sequence->br, 2 ) == 2 ) @@ -442,7 +442,7 @@ static void picture_coding_extension( sequence_t *sequence, uint8_t *buf, int le if ( infos->picture_structure && infos->picture_structure!=PICTURE_FRAME ) infos = &sequence->picture.vdp_infos2; - bits_reader_set( &sequence->br, buf ); + bits_reader_set( &sequence->br, buf, len ); int tmp = read_bits( &sequence->br, 4 ); lprintf( "extension_start_code_identifier: %d\n", tmp ); infos->f_code[0][0] = read_bits( &sequence->br, 4 ); @@ -484,8 +484,8 @@ static void quant_matrix_extension( sequence_t *sequence, uint8_t *buf, int len { int i, j; - bits_reader_set( &sequence->br, buf ); - read_bits( &sequence->br, 4 ); + bits_reader_set( &sequence->br, buf, len ); + skip_bits( &sequence->br, 4 ); i = read_bits( &sequence->br, 1 ); lprintf( "load_intra_quantizer_matrix: %d\n", i ); if ( i ) { diff --git a/src/video_dec/libvdpau/vdpau_vc1.c b/src/video_dec/libvdpau/vdpau_vc1.c index 201d67c2a..ea6844d69 100644 --- a/src/video_dec/libvdpau/vdpau_vc1.c +++ b/src/video_dec/libvdpau/vdpau_vc1.c @@ -254,8 +254,8 @@ static void sequence_header_advanced( vdpau_vc1_decoder_t *this_gen, uint8_t *bu sequence->profile = VDP_DECODER_PROFILE_VC1_ADVANCED; lprintf("VDP_DECODER_PROFILE_VC1_ADVANCED\n"); - bits_reader_set( &sequence->br, buf ); - read_bits( &sequence->br, 15 ); + bits_reader_set( &sequence->br, buf, len ); + skip_bits( &sequence->br, 15 ); sequence->picture.vdp_infos.postprocflag = read_bits( &sequence->br, 1 ); sequence->coded_width = read_bits( &sequence->br, 12 )<<1; sequence->coded_height = (read_bits( &sequence->br, 12 )+1)<<1; @@ -263,7 +263,7 @@ static void sequence_header_advanced( vdpau_vc1_decoder_t *this_gen, uint8_t *bu sequence->picture.vdp_infos.interlace = read_bits( &sequence->br, 1 ); sequence->picture.vdp_infos.tfcntrflag = read_bits( &sequence->br, 1 ); sequence->picture.vdp_infos.finterpflag = read_bits( &sequence->br, 1 ); - read_bits( &sequence->br, 1 ); + skip_bits( &sequence->br, 1 ); sequence->picture.vdp_infos.psf = read_bits( &sequence->br, 1 ); sequence->picture.vdp_infos.maxbframes = 7; if ( read_bits( &sequence->br, 1 ) ) { @@ -312,7 +312,7 @@ static void sequence_header_advanced( vdpau_vc1_decoder_t *this_gen, uint8_t *bu if ( read_bits( &sequence->br, 1 ) ) { int col = read_bits( &sequence->br, 8 ); lprintf("color_standard = %d\n", col); - read_bits( &sequence->br, 16 ); + skip_bits( &sequence->br, 16 ); } } sequence->picture.hrd_param_flag = read_bits( &sequence->br, 1 ); @@ -332,7 +332,7 @@ static void sequence_header( vdpau_vc1_decoder_t *this_gen, uint8_t *buf, int le if ( len < 4 ) return; - bits_reader_set( &sequence->br, buf ); + bits_reader_set( &sequence->br, buf, len ); switch ( read_bits( &sequence->br, 2 ) ) { case 0: sequence->profile = VDP_DECODER_PROFILE_VC1_SIMPLE; lprintf("VDP_DECODER_PROFILE_VC1_SIMPLE\n"); break; case 1: sequence->profile = VDP_DECODER_PROFILE_VC1_MAIN; lprintf("VDP_DECODER_PROFILE_VC1_MAIN\n"); break; @@ -340,16 +340,16 @@ static void sequence_header( vdpau_vc1_decoder_t *this_gen, uint8_t *buf, int le case 3: return sequence_header_advanced( this_gen, buf, len ); break; default: return; /* illegal value, broken header? */ } - read_bits( &sequence->br, 10 ); + skip_bits( &sequence->br, 10 ); sequence->picture.vdp_infos.loopfilter = read_bits( &sequence->br, 1 ); - read_bits( &sequence->br, 1 ); + skip_bits( &sequence->br, 1 ); sequence->picture.vdp_infos.multires = read_bits( &sequence->br, 1 ); - read_bits( &sequence->br, 1 ); + skip_bits( &sequence->br, 1 ); sequence->picture.vdp_infos.fastuvmc = read_bits( &sequence->br, 1 ); sequence->picture.vdp_infos.extended_mv = read_bits( &sequence->br, 1 ); sequence->picture.vdp_infos.dquant = read_bits( &sequence->br, 2 ); sequence->picture.vdp_infos.vstransform = read_bits( &sequence->br, 1 ); - read_bits( &sequence->br, 1 ); + skip_bits( &sequence->br, 1 ); sequence->picture.vdp_infos.overlap = read_bits( &sequence->br, 1 ); sequence->picture.vdp_infos.syncmarker = read_bits( &sequence->br, 1 ); sequence->picture.vdp_infos.rangered = read_bits( &sequence->br, 1 ); @@ -367,8 +367,8 @@ static void entry_point( vdpau_vc1_decoder_t *this_gen, uint8_t *buf, int len ) lprintf( "entry_point\n" ); sequence_t *sequence = (sequence_t*)&this_gen->sequence; - bits_reader_set( &sequence->br, buf ); - read_bits( &sequence->br, 2 ); + bits_reader_set( &sequence->br, buf, len ); + skip_bits( &sequence->br, 2 ); sequence->picture.vdp_infos.panscan_flag = read_bits( &sequence->br, 1 ); sequence->picture.vdp_infos.refdist_flag = read_bits( &sequence->br, 1 ); sequence->picture.vdp_infos.loopfilter = read_bits( &sequence->br, 1 ); @@ -382,7 +382,7 @@ static void entry_point( vdpau_vc1_decoder_t *this_gen, uint8_t *buf, int len ) if ( sequence->picture.hrd_param_flag ) { int i; for ( i=0; ipicture.hrd_num_leaky_buckets; ++i ) - read_bits( &sequence->br, 8 ); + skip_bits( &sequence->br, 8 ); } if ( read_bits( &sequence->br, 1 ) ) { @@ -414,11 +414,11 @@ static void picture_header( vdpau_vc1_decoder_t *this_gen, uint8_t *buf, int len lprintf("picture_header\n"); - bits_reader_set( &sequence->br, buf ); - read_bits( &sequence->br, 2 ); + bits_reader_set( &sequence->br, buf, len ); + skip_bits( &sequence->br, 2 ); if ( info->finterpflag ) - read_bits( &sequence->br, 1 ); + skip_bits( &sequence->br, 1 ); if ( info->rangered ) { /*info->rangered &= ~2; info->rangered |= get_bits( buf,off++,1 ) << 1;*/ @@ -460,7 +460,7 @@ static void picture_header_advanced( vdpau_vc1_decoder_t *this_gen, uint8_t *buf lprintf("picture_header_advanced\n"); - bits_reader_set( &sequence->br, buf ); + bits_reader_set( &sequence->br, buf, len ); if ( info->interlace ) { lprintf("frame->interlace=1\n"); @@ -517,7 +517,7 @@ static void picture_header_advanced( vdpau_vc1_decoder_t *this_gen, uint8_t *buf } if ( info->tfcntrflag ) { lprintf("tfcntrflag=1\n"); - read_bits( &sequence->br, 8 ); + skip_bits( &sequence->br, 8 ); } if ( info->pulldown && info->interlace ) { pic->top_field_first = read_bits( &sequence->br, 1 ); -- cgit v1.2.3