diff mbox

[FFmpeg-devel] lavc/vp8: Fix HWAccel VP8 decoder can't support resolution change.

Message ID 9f434fa3-2bf3-e375-3713-90369800836c@gmail.com
State Superseded
Headers show

Commit Message

Jun Zhao Nov. 28, 2017, 5:20 a.m. UTC 
From 94d511d93fdb83103fdafbd9ca0d02abfbd2e305 Mon Sep 17 00:00:00 2001
From: Jun Zhao <jun.zhao@intel.com>
Date: Tue, 28 Nov 2017 21:05:18 +0800
Subject: [PATCH] lavc/vp8: Fix HWAccel VP8 decoder can't support resolution
 change.

Use the following command to reproduce this issue:
make fate-vp8-size-change HWACCEL="vaapi -vaapi_device \
/dev/dri/renderD128 -hwaccel_output_format yuv420p"
SAMPLES=../fate-suite/.

At the same time, reconstruct the public logic as a function.

Signed-off-by: Yun Zhou <yunx.z.zhou@intel.com>
Signed-off-by: Jun Zhao <jun.zhao@intel.com>
---
 libavcodec/vp8.c | 46 ++++++++++++++++++++++++++++++++++------------
 1 file changed, 34 insertions(+), 12 deletions(-)

Comments

Carl Eugen Hoyos Nov. 28, 2017, 1:23 p.m. UTC | #1 
2017-11-28 6:20 GMT+01:00 Jun Zhao <mypopydev@gmail.com>:

Could be split.

Carl Eugen
Michael Niedermayer Nov. 28, 2017, 11:09 p.m. UTC | #2 
On Tue, Nov 28, 2017 at 01:20:59PM +0800, Jun Zhao wrote:
> 

>  vp8.c |   46 ++++++++++++++++++++++++++++++++++------------
>  1 file changed, 34 insertions(+), 12 deletions(-)
> d548af3b82a9d77f67fe9621fc7aeee0392fcc89  0001-lavc-vp8-Fix-HWAccel-VP8-decoder-can-t-support-resol.patch
> From 94d511d93fdb83103fdafbd9ca0d02abfbd2e305 Mon Sep 17 00:00:00 2001
> From: Jun Zhao <jun.zhao@intel.com>
> Date: Tue, 28 Nov 2017 21:05:18 +0800
> Subject: [PATCH] lavc/vp8: Fix HWAccel VP8 decoder can't support resolution
>  change.
> 
> Use the following command to reproduce this issue:
> make fate-vp8-size-change HWACCEL="vaapi -vaapi_device \
> /dev/dri/renderD128 -hwaccel_output_format yuv420p"
> SAMPLES=../fate-suite/.
> 
> At the same time, reconstruct the public logic as a function.
> 
> Signed-off-by: Yun Zhou <yunx.z.zhou@intel.com>
> Signed-off-by: Jun Zhao <jun.zhao@intel.com>
> ---
>  libavcodec/vp8.c | 46 ++++++++++++++++++++++++++++++++++------------
>  1 file changed, 34 insertions(+), 12 deletions(-)

Causes fate-webp-rgba-lossy-q80 to segfault

Program received signal SIGSEGV, Segmentation fault.
0x0000000000ceee65 in vp8_lossy_decode_alpha (avctx=0x21c7da0, p=0x21c8980, data_start=0x21caa67 "\027 \020HR\331\037r\215\210\b\a\261`r\a8\177\235\020\"\372\037\064\001VP8 R", data_size=25) at libavcodec/webp.c:1312
1312                    *pp = *ap;
(gdb) bt
Python Exception <type 'exceptions.ImportError'> No module named gdb.frames:
#0  0x0000000000ceee65 in vp8_lossy_decode_alpha (avctx=0x21c7da0, p=0x21c8980, data_start=0x21caa67 "\027 \020HR\331\037r\215\210\b\a\261`r\a8\177\235\020\"\372\037\064\001VP8 R", data_size=25) at libavcodec/webp.c:1312
#1  0x0000000000cef050 in vp8_lossy_decode_frame (avctx=0x21c7da0, p=0x21c8980, got_frame=0x7fffffffd648, data_start=0x21caa88 "\320\001", data_size=82) at libavcodec/webp.c:1362
#2  0x0000000000cef357 in webp_decode_frame (avctx=0x21c7da0, data=0x21c8980, got_frame=0x7fffffffd648, avpkt=0x21c8ca0) at libavcodec/webp.c:1421
#3  0x0000000000875781 in decode_simple_internal (avctx=0x21c7da0, frame=0x21c8980) at libavcodec/decode.c:398
#4  0x0000000000876408 in decode_simple_receive_frame (avctx=0x21c7da0, frame=0x21c8980) at libavcodec/decode.c:594
#5  0x00000000008764d3 in decode_receive_frame_internal (avctx=0x21c7da0, frame=0x21c8980) at libavcodec/decode.c:612
#6  0x000000000087674b in avcodec_send_packet (avctx=0x21c7da0, avpkt=0x7fffffffd7f0) at libavcodec/decode.c:674
#7  0x00000000007be3e1 in try_decode_frame (s=0x21c6420, st=0x21c7480, avpkt=0x7fffffffd9a0, options=0x21c6e20) at libavformat/utils.c:3007
#8  0x00000000007c174a in avformat_find_stream_info (ic=0x21c6420, options=0x21c6e20) at libavformat/utils.c:3832
#9  0x00000000004154a5 in open_input_file (o=0x7fffffffdcf0, filename=0x7fffffffe688 "/home/michael/fatesamples/fate/fate-suite//webp/rgba_q80.webp") at fftools/ffmpeg_opt.c:1091
#10 0x000000000041f061 in open_files (l=0x21c5d78, inout=0x11fa077 "input", open_file=0x4149d3 <open_input_file>) at fftools/ffmpeg_opt.c:3296
#11 0x000000000041f1f3 in ffmpeg_parse_options (argc=17, argv=0x7fffffffe308) at fftools/ffmpeg_opt.c:3336
#12 0x000000000043d5d2 in main (argc=17, argv=0x7fffffffe308) at fftools/ffmpeg.c:4837

[...]
Jun Zhao Nov. 29, 2017, 12:54 a.m. UTC | #3 
On 2017/11/29 7:09, Michael Niedermayer wrote:
> On Tue, Nov 28, 2017 at 01:20:59PM +0800, Jun Zhao wrote:
>>  vp8.c |   46 ++++++++++++++++++++++++++++++++++------------
>>  1 file changed, 34 insertions(+), 12 deletions(-)
>> d548af3b82a9d77f67fe9621fc7aeee0392fcc89  0001-lavc-vp8-Fix-HWAccel-VP8-decoder-can-t-support-resol.patch
>> From 94d511d93fdb83103fdafbd9ca0d02abfbd2e305 Mon Sep 17 00:00:00 2001
>> From: Jun Zhao <jun.zhao@intel.com>
>> Date: Tue, 28 Nov 2017 21:05:18 +0800
>> Subject: [PATCH] lavc/vp8: Fix HWAccel VP8 decoder can't support resolution
>>  change.
>>
>> Use the following command to reproduce this issue:
>> make fate-vp8-size-change HWACCEL="vaapi -vaapi_device \
>> /dev/dri/renderD128 -hwaccel_output_format yuv420p"
>> SAMPLES=../fate-suite/.
>>
>> At the same time, reconstruct the public logic as a function.
>>
>> Signed-off-by: Yun Zhou <yunx.z.zhou@intel.com>
>> Signed-off-by: Jun Zhao <jun.zhao@intel.com>
>> ---
>>  libavcodec/vp8.c | 46 ++++++++++++++++++++++++++++++++++------------
>>  1 file changed, 34 insertions(+), 12 deletions(-)
> Causes fate-webp-rgba-lossy-q80 to segfault
>
> Program received signal SIGSEGV, Segmentation fault.
> 0x0000000000ceee65 in vp8_lossy_decode_alpha (avctx=0x21c7da0, p=0x21c8980, data_start=0x21caa67 "\027 \020HR\331\037r\215\210\b\a\261`r\a8\177\235\020\"\372\037\064\001VP8 R", data_size=25) at libavcodec/webp.c:1312
> 1312                    *pp = *ap;
> (gdb) bt
> Python Exception <type 'exceptions.ImportError'> No module named gdb.frames:
> #0  0x0000000000ceee65 in vp8_lossy_decode_alpha (avctx=0x21c7da0, p=0x21c8980, data_start=0x21caa67 "\027 \020HR\331\037r\215\210\b\a\261`r\a8\177\235\020\"\372\037\064\001VP8 R", data_size=25) at libavcodec/webp.c:1312
> #1  0x0000000000cef050 in vp8_lossy_decode_frame (avctx=0x21c7da0, p=0x21c8980, got_frame=0x7fffffffd648, data_start=0x21caa88 "\320\001", data_size=82) at libavcodec/webp.c:1362
> #2  0x0000000000cef357 in webp_decode_frame (avctx=0x21c7da0, data=0x21c8980, got_frame=0x7fffffffd648, avpkt=0x21c8ca0) at libavcodec/webp.c:1421
> #3  0x0000000000875781 in decode_simple_internal (avctx=0x21c7da0, frame=0x21c8980) at libavcodec/decode.c:398
> #4  0x0000000000876408 in decode_simple_receive_frame (avctx=0x21c7da0, frame=0x21c8980) at libavcodec/decode.c:594
> #5  0x00000000008764d3 in decode_receive_frame_internal (avctx=0x21c7da0, frame=0x21c8980) at libavcodec/decode.c:612
> #6  0x000000000087674b in avcodec_send_packet (avctx=0x21c7da0, avpkt=0x7fffffffd7f0) at libavcodec/decode.c:674
> #7  0x00000000007be3e1 in try_decode_frame (s=0x21c6420, st=0x21c7480, avpkt=0x7fffffffd9a0, options=0x21c6e20) at libavformat/utils.c:3007
> #8  0x00000000007c174a in avformat_find_stream_info (ic=0x21c6420, options=0x21c6e20) at libavformat/utils.c:3832
> #9  0x00000000004154a5 in open_input_file (o=0x7fffffffdcf0, filename=0x7fffffffe688 "/home/michael/fatesamples/fate/fate-suite//webp/rgba_q80.webp") at fftools/ffmpeg_opt.c:1091
> #10 0x000000000041f061 in open_files (l=0x21c5d78, inout=0x11fa077 "input", open_file=0x4149d3 <open_input_file>) at fftools/ffmpeg_opt.c:3296
> #11 0x000000000041f1f3 in ffmpeg_parse_options (argc=17, argv=0x7fffffffe308) at fftools/ffmpeg_opt.c:3336
> #12 0x000000000043d5d2 in main (argc=17, argv=0x7fffffffe308) at fftools/ffmpeg.c:4837
>
> [...]
Will check the segmentation fault, Tks.
>
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
Jun Zhao Nov. 29, 2017, 3:21 a.m. UTC | #4 
On 2017/11/28 21:23, Carl Eugen Hoyos wrote:
> 2017-11-28 6:20 GMT+01:00 Jun Zhao <mypopydev@gmail.com>:
>
> Could be split.
Do you means split with 2 patches? 1) for issue fix 2) for coding
reconstruct ?
>
> Carl Eugen
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
diff mbox

Patch

diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c
index 471c0bb89e..5bf00a44f9 100644
--- a/libavcodec/vp8.c
+++ b/libavcodec/vp8.c
@@ -167,6 +167,30 @@  static VP8Frame *vp8_find_free_buffer(VP8Context *s)
     return frame;
 }
 
+static enum AVPixelFormat get_pixel_format(VP8Context *s)
+{
+    enum AVPixelFormat fmt;
+    enum AVPixelFormat pix_fmts[] = {
+#if CONFIG_VP8_VAAPI_HWACCEL
+        AV_PIX_FMT_VAAPI,
+#endif
+#if CONFIG_VP8_NVDEC_HWACCEL
+        AV_PIX_FMT_CUDA,
+#endif
+        AV_PIX_FMT_YUV420P,
+        AV_PIX_FMT_NONE,
+    };
+
+    fmt = ff_get_format(s->avctx, pix_fmts);
+    if (fmt < 0) {
+        fmt = AV_PIX_FMT_NONE;
+        av_log(s->avctx, AV_LOG_ERROR,
+               "Can not support the format. \n");
+    }
+
+    return fmt;
+}
+
 static av_always_inline
 int update_dimensions(VP8Context *s, int width, int height, int is_vp7)
 {
@@ -182,6 +206,15 @@  int update_dimensions(VP8Context *s, int width, int height, int is_vp7)
             return ret;
     }
 
+    if (!is_vp7) {
+        s->pix_fmt = get_pixel_format(s);
+        if (s->pix_fmt < 0) {
+            ret = AVERROR(EINVAL);
+            return ret;
+        }
+        avctx->pix_fmt = s->pix_fmt;
+    }
+
     s->mb_width  = (s->avctx->coded_width  + 15) / 16;
     s->mb_height = (s->avctx->coded_height + 15) / 16;
 
@@ -2598,18 +2631,7 @@  int vp78_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     if (s->actually_webp) {
         // avctx->pix_fmt already set in caller.
     } else if (!is_vp7 && s->pix_fmt == AV_PIX_FMT_NONE) {
-        enum AVPixelFormat pix_fmts[] = {
-#if CONFIG_VP8_VAAPI_HWACCEL
-            AV_PIX_FMT_VAAPI,
-#endif
-#if CONFIG_VP8_NVDEC_HWACCEL
-            AV_PIX_FMT_CUDA,
-#endif
-            AV_PIX_FMT_YUV420P,
-            AV_PIX_FMT_NONE,
-        };
-
-        s->pix_fmt = ff_get_format(s->avctx, pix_fmts);
+        s->pix_fmt = get_pixel_format(s);
         if (s->pix_fmt < 0) {
             ret = AVERROR(EINVAL);
             goto err;