diff mbox series

[FFmpeg-devel,v4,6/9] avcodec/av1dec: add display frame for film grain usage

Message ID 20210709033229.19125-6-fei.w.wang@intel.com
State New
Headers show
Series [FFmpeg-devel,v4,1/9] cbs_av1: fix incorrect data type
Related show

Checks

Context Check Description
andriy/x86_make success Make finished
andriy/x86_make_fate success Make fate finished
andriy/PPC64_make success Make finished
andriy/PPC64_make_fate success Make fate finished

Commit Message

Wang, Fei W July 9, 2021, 3:32 a.m. UTC
Make it flexible if decode frame(without apply film grain) and display
frame(applied film grain) both needed when decode film grain clips.

Signed-off-by: Fei Wang <fei.w.wang@intel.com>
---
 libavcodec/av1dec.c | 43 ++++++++++++++++++++++++++++++++++++++++++-
 libavcodec/av1dec.h |  3 +++
 2 files changed, 45 insertions(+), 1 deletion(-)

Comments

James Almer July 9, 2021, 12:52 p.m. UTC | #1
On 7/9/2021 12:32 AM, Fei Wang wrote:
> Make it flexible if decode frame(without apply film grain) and display
> frame(applied film grain) both needed when decode film grain clips.

Why do you need to do this for Vaapi? Nvdec doesn't require it, it just 
ensures to reference one of the extra eight surfaces it has already 
reserved in its DPB when starting a frame with apply_grain == 1.

You should be able to achieve the same for Vaapi. It's not ok to have 
hwaccel-specific code in the base decoder, same way it isn't ok to have 
arch specific code, and why we don't allow it.

> 
> Signed-off-by: Fei Wang <fei.w.wang@intel.com>
> ---
>   libavcodec/av1dec.c | 43 ++++++++++++++++++++++++++++++++++++++++++-
>   libavcodec/av1dec.h |  3 +++
>   2 files changed, 45 insertions(+), 1 deletion(-)
> 
> diff --git a/libavcodec/av1dec.c b/libavcodec/av1dec.c
> index d808a9d4c6..6bba2cb7a2 100644
> --- a/libavcodec/av1dec.c
> +++ b/libavcodec/av1dec.c
> @@ -570,6 +570,8 @@ static int get_pixel_format(AVCodecContext *avctx)
>   static void av1_frame_unref(AVCodecContext *avctx, AV1Frame *f)
>   {
>       ff_thread_release_buffer(avctx, &f->tf);
> +    if (f->tf_display.f->buf[0])
> +        ff_thread_release_buffer(avctx, &f->tf_display);
>       av_buffer_unref(&f->hwaccel_priv_buf);
>       f->hwaccel_picture_private = NULL;
>       av_buffer_unref(&f->header_ref);
> @@ -589,6 +591,12 @@ static int av1_frame_ref(AVCodecContext *avctx, AV1Frame *dst, const AV1Frame *s
>       if (ret < 0)
>           return ret;
>   
> +    if (src->tf_display.f->buf[0]) {
> +        ret = ff_thread_ref_frame(&dst->tf_display, &src->tf_display);
> +        if (ret < 0)
> +            return ret;
> +    }
> +
>       dst->header_ref = av_buffer_ref(src->header_ref);
>       if (!dst->header_ref)
>           goto fail;
> @@ -635,9 +643,11 @@ static av_cold int av1_decode_free(AVCodecContext *avctx)
>       for (int i = 0; i < FF_ARRAY_ELEMS(s->ref); i++) {
>           av1_frame_unref(avctx, &s->ref[i]);
>           av_frame_free(&s->ref[i].tf.f);
> +        av_frame_free(&s->ref[i].tf_display.f);
>       }
>       av1_frame_unref(avctx, &s->cur_frame);
>       av_frame_free(&s->cur_frame.tf.f);
> +    av_frame_free(&s->cur_frame.tf_display.f);
>   
>       av_buffer_unref(&s->seq_ref);
>       av_buffer_unref(&s->header_ref);
> @@ -739,6 +749,13 @@ static av_cold int av1_decode_init(AVCodecContext *avctx)
>                      "Failed to allocate reference frame buffer %d.\n", i);
>               return AVERROR(ENOMEM);
>           }
> +
> +        s->ref[i].tf_display.f = av_frame_alloc();
> +        if (!s->ref[i].tf_display.f) {
> +            av_log(avctx, AV_LOG_ERROR,
> +                   "Failed to allocate display frame buffer %d.\n", i);
> +            return AVERROR(ENOMEM);
> +        }
>       }
>   
>       s->cur_frame.tf.f = av_frame_alloc();
> @@ -748,6 +765,13 @@ static av_cold int av1_decode_init(AVCodecContext *avctx)
>           return AVERROR(ENOMEM);
>       }
>   
> +    s->cur_frame.tf_display.f = av_frame_alloc();
> +    if (!s->cur_frame.tf_display.f) {
> +        av_log(avctx, AV_LOG_ERROR,
> +               "Failed to allocate current display frame buffer.\n");
> +        return AVERROR(ENOMEM);
> +    }
> +
>       ret = ff_cbs_init(&s->cbc, AV_CODEC_ID_AV1, avctx);
>       if (ret < 0)
>           return ret;
> @@ -820,6 +844,16 @@ static int av1_frame_alloc(AVCodecContext *avctx, AV1Frame *f)
>           break;
>       }
>   
> +    if (header->film_grain.apply_grain &&
> +        (avctx->pix_fmt == AV_PIX_FMT_VAAPI) &&

Case in point.

> +        !(avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN)) {
> +        if ((ret = ff_thread_get_buffer(avctx, &f->tf_display, AV_GET_BUFFER_FLAG_REF)) < 0)
> +            goto fail;
> +
> +        f->tf_display.f->key_frame = frame->key_frame;
> +        f->tf_display.f->pict_type = frame->pict_type;
> +    }
> +
>       if (avctx->hwaccel) {
>           const AVHWAccel *hwaccel = avctx->hwaccel;
>           if (hwaccel->frame_priv_data_size) {
> @@ -903,9 +937,16 @@ static int set_output_frame(AVCodecContext *avctx, AVFrame *frame,
>                               const AVPacket *pkt, int *got_frame)
>   {
>       AV1DecContext *s = avctx->priv_data;
> -    const AVFrame *srcframe = s->cur_frame.tf.f;
> +    const AVFrame *srcframe;
>       int ret;
>   
> +    /* Use tf_display as output when it is available. */
> +    if (s->cur_frame.tf_display.f->buf[0]) {
> +        srcframe = s->cur_frame.tf_display.f;
> +    } else {
> +        srcframe= s->cur_frame.tf.f;
> +    }
> +
>       // TODO: all layers
>       if (s->operating_point_idc &&
>           av_log2(s->operating_point_idc >> 8) > s->cur_frame.spatial_id)
> diff --git a/libavcodec/av1dec.h b/libavcodec/av1dec.h
> index 4e140588b9..5af3dfc867 100644
> --- a/libavcodec/av1dec.h
> +++ b/libavcodec/av1dec.h
> @@ -33,6 +33,9 @@
>   typedef struct AV1Frame {
>       ThreadFrame tf;
>   
> +    /** current tf_display is only used for VA-API to apply film grain */
> +    ThreadFrame tf_display;
> +
>       AVBufferRef *hwaccel_priv_buf;
>       void *hwaccel_picture_private;
>   
>
diff mbox series

Patch

diff --git a/libavcodec/av1dec.c b/libavcodec/av1dec.c
index d808a9d4c6..6bba2cb7a2 100644
--- a/libavcodec/av1dec.c
+++ b/libavcodec/av1dec.c
@@ -570,6 +570,8 @@  static int get_pixel_format(AVCodecContext *avctx)
 static void av1_frame_unref(AVCodecContext *avctx, AV1Frame *f)
 {
     ff_thread_release_buffer(avctx, &f->tf);
+    if (f->tf_display.f->buf[0])
+        ff_thread_release_buffer(avctx, &f->tf_display);
     av_buffer_unref(&f->hwaccel_priv_buf);
     f->hwaccel_picture_private = NULL;
     av_buffer_unref(&f->header_ref);
@@ -589,6 +591,12 @@  static int av1_frame_ref(AVCodecContext *avctx, AV1Frame *dst, const AV1Frame *s
     if (ret < 0)
         return ret;
 
+    if (src->tf_display.f->buf[0]) {
+        ret = ff_thread_ref_frame(&dst->tf_display, &src->tf_display);
+        if (ret < 0)
+            return ret;
+    }
+
     dst->header_ref = av_buffer_ref(src->header_ref);
     if (!dst->header_ref)
         goto fail;
@@ -635,9 +643,11 @@  static av_cold int av1_decode_free(AVCodecContext *avctx)
     for (int i = 0; i < FF_ARRAY_ELEMS(s->ref); i++) {
         av1_frame_unref(avctx, &s->ref[i]);
         av_frame_free(&s->ref[i].tf.f);
+        av_frame_free(&s->ref[i].tf_display.f);
     }
     av1_frame_unref(avctx, &s->cur_frame);
     av_frame_free(&s->cur_frame.tf.f);
+    av_frame_free(&s->cur_frame.tf_display.f);
 
     av_buffer_unref(&s->seq_ref);
     av_buffer_unref(&s->header_ref);
@@ -739,6 +749,13 @@  static av_cold int av1_decode_init(AVCodecContext *avctx)
                    "Failed to allocate reference frame buffer %d.\n", i);
             return AVERROR(ENOMEM);
         }
+
+        s->ref[i].tf_display.f = av_frame_alloc();
+        if (!s->ref[i].tf_display.f) {
+            av_log(avctx, AV_LOG_ERROR,
+                   "Failed to allocate display frame buffer %d.\n", i);
+            return AVERROR(ENOMEM);
+        }
     }
 
     s->cur_frame.tf.f = av_frame_alloc();
@@ -748,6 +765,13 @@  static av_cold int av1_decode_init(AVCodecContext *avctx)
         return AVERROR(ENOMEM);
     }
 
+    s->cur_frame.tf_display.f = av_frame_alloc();
+    if (!s->cur_frame.tf_display.f) {
+        av_log(avctx, AV_LOG_ERROR,
+               "Failed to allocate current display frame buffer.\n");
+        return AVERROR(ENOMEM);
+    }
+
     ret = ff_cbs_init(&s->cbc, AV_CODEC_ID_AV1, avctx);
     if (ret < 0)
         return ret;
@@ -820,6 +844,16 @@  static int av1_frame_alloc(AVCodecContext *avctx, AV1Frame *f)
         break;
     }
 
+    if (header->film_grain.apply_grain &&
+        (avctx->pix_fmt == AV_PIX_FMT_VAAPI) &&
+        !(avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN)) {
+        if ((ret = ff_thread_get_buffer(avctx, &f->tf_display, AV_GET_BUFFER_FLAG_REF)) < 0)
+            goto fail;
+
+        f->tf_display.f->key_frame = frame->key_frame;
+        f->tf_display.f->pict_type = frame->pict_type;
+    }
+
     if (avctx->hwaccel) {
         const AVHWAccel *hwaccel = avctx->hwaccel;
         if (hwaccel->frame_priv_data_size) {
@@ -903,9 +937,16 @@  static int set_output_frame(AVCodecContext *avctx, AVFrame *frame,
                             const AVPacket *pkt, int *got_frame)
 {
     AV1DecContext *s = avctx->priv_data;
-    const AVFrame *srcframe = s->cur_frame.tf.f;
+    const AVFrame *srcframe;
     int ret;
 
+    /* Use tf_display as output when it is available. */
+    if (s->cur_frame.tf_display.f->buf[0]) {
+        srcframe = s->cur_frame.tf_display.f;
+    } else {
+        srcframe= s->cur_frame.tf.f;
+    }
+
     // TODO: all layers
     if (s->operating_point_idc &&
         av_log2(s->operating_point_idc >> 8) > s->cur_frame.spatial_id)
diff --git a/libavcodec/av1dec.h b/libavcodec/av1dec.h
index 4e140588b9..5af3dfc867 100644
--- a/libavcodec/av1dec.h
+++ b/libavcodec/av1dec.h
@@ -33,6 +33,9 @@ 
 typedef struct AV1Frame {
     ThreadFrame tf;
 
+    /** current tf_display is only used for VA-API to apply film grain */
+    ThreadFrame tf_display;
+
     AVBufferRef *hwaccel_priv_buf;
     void *hwaccel_picture_private;