mpegvideo: Fix exported qp table offest

Found-by: wm4 Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
danimo · Mar 15, 2013 · 6430894 · 6430894
1 parent 70c0ae9
commit 6430894
Show file tree

Hide file tree

Showing 6 changed files with 22 additions and 8 deletions.
diff --git a/libavcodec/h263dec.c b/libavcodec/h263dec.c
@@ -750,12 +750,12 @@ uint64_t time= rdtsc();
         if ((ret = av_frame_ref(pict, &s->current_picture_ptr->f)) < 0)
             return ret;
         ff_print_debug_info(s, s->current_picture_ptr, pict);
-        av_frame_set_qp_table(pict, av_buffer_ref(s->current_picture_ptr->qscale_table_buf), s->mb_stride, FF_QSCALE_TYPE_MPEG1);
+        ff_mpv_export_qp_table(s, pict, s->current_picture_ptr, FF_QSCALE_TYPE_MPEG1);
     } else if (s->last_picture_ptr != NULL) {
         if ((ret = av_frame_ref(pict, &s->last_picture_ptr->f)) < 0)
             return ret;
         ff_print_debug_info(s, s->last_picture_ptr, pict);
-        av_frame_set_qp_table(pict, av_buffer_ref(s->last_picture_ptr->qscale_table_buf), s->mb_stride, FF_QSCALE_TYPE_MPEG1);
+        ff_mpv_export_qp_table(s, pict, s->last_picture_ptr, FF_QSCALE_TYPE_MPEG1);
     }
 
     if(s->last_picture_ptr || s->low_delay){

diff --git a/libavcodec/mpeg12.c b/libavcodec/mpeg12.c
@@ -1990,7 +1990,7 @@ static int slice_end(AVCodecContext *avctx, AVFrame *pict)
             if (ret < 0)
                 return ret;
             ff_print_debug_info(s, s->current_picture_ptr, pict);
-            av_frame_set_qp_table(pict, av_buffer_ref(s->current_picture_ptr->qscale_table_buf), s->mb_stride, FF_QSCALE_TYPE_MPEG2);
+            ff_mpv_export_qp_table(s, pict, s->current_picture_ptr, FF_QSCALE_TYPE_MPEG2);
         } else {
             if (avctx->active_thread_type & FF_THREAD_FRAME)
                 s->picture_number++;
@@ -2001,7 +2001,7 @@ static int slice_end(AVCodecContext *avctx, AVFrame *pict)
                 if (ret < 0)
                     return ret;
                 ff_print_debug_info(s, s->last_picture_ptr, pict);
-                av_frame_set_qp_table(pict, av_buffer_ref(s->last_picture_ptr->qscale_table_buf), s->mb_stride, FF_QSCALE_TYPE_MPEG2);
+                ff_mpv_export_qp_table(s, pict, s->last_picture_ptr, FF_QSCALE_TYPE_MPEG2);
             }
         }
 

diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
@@ -2196,6 +2196,18 @@ void ff_print_debug_info(MpegEncContext *s, Picture *p, AVFrame *pict)
                          s->mb_width, s->mb_height, s->mb_stride, s->quarter_sample);
 }
 
+int ff_mpv_export_qp_table(MpegEncContext *s, AVFrame *f, Picture *p, int qp_type)
+{
+    AVBufferRef *ref = av_buffer_ref(p->qscale_table_buf);
+    int offset = 2*s->mb_stride + 1;
+    if(!ref)
+        return AVERROR(ENOMEM);
+    av_assert0(ref->size >= offset + s->mb_stride * ((f->height+15)/16));
+    ref->size -= offset;
+    ref->data += offset;
+    return av_frame_set_qp_table(f, ref, s->mb_stride, qp_type);
+}
+
 static inline int hpel_motion_lowres(MpegEncContext *s,
                                      uint8_t *dest, uint8_t *src,
                                      int field_based, int field_select,

diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h
@@ -813,6 +813,8 @@ void ff_print_debug_info2(AVCodecContext *avctx, Picture *p, AVFrame *pict, uint
                          int *low_delay,
                          int mb_width, int mb_height, int mb_stride, int quarter_sample);
 
+int ff_mpv_export_qp_table(MpegEncContext *s, AVFrame *f, Picture *p, int qp_type);
+
 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix);
 void ff_release_unused_pictures(MpegEncContext *s, int remove_current);
 int ff_find_unused_picture(MpegEncContext *s, int shared);

diff --git a/libavcodec/rv10.c b/libavcodec/rv10.c
@@ -742,12 +742,12 @@ static int rv10_decode_frame(AVCodecContext *avctx,
             if ((ret = av_frame_ref(pict, &s->current_picture_ptr->f)) < 0)
                 return ret;
             ff_print_debug_info(s, s->current_picture_ptr, pict);
-            av_frame_set_qp_table(pict, av_buffer_ref(s->current_picture_ptr->qscale_table_buf), s->mb_stride, FF_QSCALE_TYPE_MPEG1);
+            ff_mpv_export_qp_table(s, pict, s->current_picture_ptr, FF_QSCALE_TYPE_MPEG1);
         } else if (s->last_picture_ptr != NULL) {
             if ((ret = av_frame_ref(pict, &s->last_picture_ptr->f)) < 0)
                 return ret;
             ff_print_debug_info(s, s->last_picture_ptr, pict);
-            av_frame_set_qp_table(pict, av_buffer_ref(s->last_picture_ptr->qscale_table_buf), s->mb_stride, FF_QSCALE_TYPE_MPEG1);
+            ff_mpv_export_qp_table(s, pict,s->last_picture_ptr, FF_QSCALE_TYPE_MPEG1);
         }
 
         if(s->last_picture_ptr || s->low_delay){

diff --git a/libavcodec/rv34.c b/libavcodec/rv34.c
@@ -1584,13 +1584,13 @@ static int finish_frame(AVCodecContext *avctx, AVFrame *pict)
         if ((ret = av_frame_ref(pict, &s->current_picture_ptr->f)) < 0)
             return ret;
         ff_print_debug_info(s, s->current_picture_ptr, pict);
-        av_frame_set_qp_table(pict, av_buffer_ref(s->current_picture_ptr->qscale_table_buf), s->mb_stride, FF_QSCALE_TYPE_MPEG1);
+        ff_mpv_export_qp_table(s, pict, s->current_picture_ptr, FF_QSCALE_TYPE_MPEG1);
         got_picture = 1;
     } else if (s->last_picture_ptr != NULL) {
         if ((ret = av_frame_ref(pict, &s->last_picture_ptr->f)) < 0)
             return ret;
         ff_print_debug_info(s, s->last_picture_ptr, pict);
-        av_frame_set_qp_table(pict, av_buffer_ref(s->last_picture_ptr->qscale_table_buf), s->mb_stride, FF_QSCALE_TYPE_MPEG1);
+        ff_mpv_export_qp_table(s, pict, s->last_picture_ptr, FF_QSCALE_TYPE_MPEG1);
         got_picture = 1;
     }