[FFmpeg-devel,v3] avcodec/libvpxenc: add a way to set VP9E_SET_SVC_REF_FRAME_CONFIG

In order to fine-control referencing schemes in VP9 encoding, there
is a need to use VP9E_SET_SVC_REF_FRAME_CONFIG method. This commit
provides a way to use the API through frame metadata.
---
 doc/encoders.texi      | 32 +++++++++++++++++
 libavcodec/libvpxenc.c | 81 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+)

Hi James

On Tue, Dec 15, 2020 at 9:18 AM Wonkap Jang <wonkap@google.com> wrote:

> In order to fine-control referencing schemes in VP9 encoding, there
> is a need to use VP9E_SET_SVC_REF_FRAME_CONFIG method. This commit
> provides a way to use the API through frame metadata.
> ---
>  doc/encoders.texi      | 32 +++++++++++++++++
>  libavcodec/libvpxenc.c | 81 ++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 113 insertions(+)
>
> diff --git a/doc/encoders.texi b/doc/encoders.texi
> index 0b1c69e982..2a10bc0608 100644
> --- a/doc/encoders.texi
> +++ b/doc/encoders.texi
> @@ -2129,6 +2129,38 @@ midpoint is passed in rather than calculated for a
> specific clip or chunk.
>  The valid range is [0, 10000]. 0 (default) uses standard VBR.
>  @item enable-tpl @var{boolean}
>  Enable temporal dependency model.
> +@item ref-frame-config
> +Using per-frame metadata, set members of the structure
> @code{vpx_svc_ref_frame_config_t} in @code{vpx/vp8cx.h} to fine-control
> referencing schemes and frame buffer management.
> +@*Use a :-separated list of key=value pairs.
> +For example,
> +@example
> +av_dict_set(&av_frame->metadata, "ref-frame-config", \
>
> +"rfc_update_buffer_slot=7:rfc_lst_fb_idx=0:rfc_gld_fb_idx=1:rfc_alt_fb_idx=2:rfc_reference_last=0:rfc_reference_golden=0:rfc_reference_alt_ref=0");}
> +@end example
> +@table @option
> +@item rfc_update_buffer_slot
> +Indicates the buffer slot number to update
> +@item rfc_update_last
> +Indicates whether to update the LAST frame
> +@item rfc_update_golden
> +Indicates whether to update GOLDEN frame
> +@item rfc_update_alt_ref
> +Indicates whether to update ALT_REF frame
> +@item rfc_lst_fb_idx
> +LAST frame buffer index
> +@item rfc_gld_fb_idx
> +GOLDEN frame buffer index
> +@item rfc_alt_fb_idx
> +ALT_REF frame buffer index
> +@item rfc_reference_last
> +Indicates whetehr to reference LAST frame
> +@item rfc_reference_golden
> +Indicates whether to reference GOLDEN frame
> +@item rfc_reference_alt_ref
> +Indicates whether to reference ALT_REF frame
> +@item rfc_reference_duration
> +Indicates frame duration
> +@end table
>  @end table
>
>  @end table
> diff --git a/libavcodec/libvpxenc.c b/libavcodec/libvpxenc.c
> index a7c76eb835..4943a9e32c 100644
> --- a/libavcodec/libvpxenc.c
> +++ b/libavcodec/libvpxenc.c
> @@ -125,6 +125,10 @@ typedef struct VPxEncoderContext {
>       * encounter a frame with ROI side data.
>       */
>      int roi_warned;
> +#if CONFIG_LIBVPX_VP9_ENCODER && defined
> (VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT)
> +    vpx_svc_ref_frame_config_t ref_frame_config;
> +    AVDictionary *vpx_ref_frame_config;
> +#endif
>  } VPxContext;
>
>  /** String mappings for enum vp8e_enc_control_id */
> @@ -152,6 +156,7 @@ static const char *const ctlidstr[] = {
>      [VP9E_SET_SVC_LAYER_ID]            = "VP9E_SET_SVC_LAYER_ID",
>  #if VPX_ENCODER_ABI_VERSION >= 12
>      [VP9E_SET_SVC_PARAMETERS]          = "VP9E_SET_SVC_PARAMETERS",
> +    [VP9E_SET_SVC_REF_FRAME_CONFIG]    = "VP9E_SET_SVC_REF_FRAME_CONFIG",
>  #endif
>      [VP9E_SET_SVC]                     = "VP9E_SET_SVC",
>  #if VPX_ENCODER_ABI_VERSION >= 11
> @@ -394,6 +399,21 @@ static void vp8_ts_parse_int_array(int *dest, char
> *value, size_t value_len, int
>      }
>  }
>
> +#if CONFIG_LIBVPX_VP9_ENCODER && defined
> (VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT)
> +static void vp8_ts_parse_int64_array(int64_t *dest, char *value, size_t
> value_len, int max_entries)
> +{
> +    int dest_idx = 0;
> +    char *saveptr = NULL;
> +    char *token = av_strtok(value, ",", &saveptr);
> +
> +    while (token && dest_idx < max_entries) {
> +        dest[dest_idx++] = strtoul(token, NULL, 10);
> +        token = av_strtok(NULL, ",", &saveptr);
> +    }
> +}
> +
> +#endif
> +
>  static void set_temporal_layer_pattern(int layering_mode,
> vpx_codec_enc_cfg_t *cfg,
>                                         int *layer_flags, int
> *flag_periodicity)
>  {
> @@ -541,6 +561,49 @@ static int vpx_ts_param_parse(VPxContext *ctx, struct
> vpx_codec_enc_cfg *enccfg,
>      return 0;
>  }
>
> +#if CONFIG_LIBVPX_VP9_ENCODER && defined
> (VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT)
> +static int vpx_ref_frame_config_parse(VPxContext *ctx, const struct
> vpx_codec_enc_cfg *enccfg,
> +                              char *key, char *value, enum AVCodecID
> codec_id)
> +{
> +    size_t value_len = strlen(value);
> +    int ss_number_layers = enccfg->ss_number_layers;
> +    vpx_svc_ref_frame_config_t *ref_frame_config = &ctx->ref_frame_config;
> +
> +    if (!value_len)
> +        return -1;
> +
> +    if (codec_id != AV_CODEC_ID_VP9)
> +        return -1;
> +
> +    if (!strcmp(key, "rfc_update_buffer_slot")) {
> +        vp8_ts_parse_int_array(ref_frame_config->update_buffer_slot,
> value, value_len, ss_number_layers);
> +    } else if (!strcmp(key, "rfc_update_last")) {
> +        vp8_ts_parse_int_array(ref_frame_config->update_last, value,
> value_len, ss_number_layers);
> +    } else if (!strcmp(key, "rfc_update_golden")) {
> +        vp8_ts_parse_int_array(ref_frame_config->update_golden, value,
> value_len, ss_number_layers);
> +    } else if (!strcmp(key, "rfc_update_alt_ref")) {
> +        vp8_ts_parse_int_array(ref_frame_config->update_alt_ref, value,
> value_len, ss_number_layers);
> +    } else if (!strcmp(key, "rfc_lst_fb_idx")) {
> +        vp8_ts_parse_int_array(ref_frame_config->lst_fb_idx, value,
> value_len, ss_number_layers);
> +    } else if (!strcmp(key, "rfc_gld_fb_idx")) {
> +        vp8_ts_parse_int_array(ref_frame_config->gld_fb_idx, value,
> value_len, ss_number_layers);
> +    } else if (!strcmp(key, "rfc_alt_fb_idx")) {
> +        vp8_ts_parse_int_array(ref_frame_config->alt_fb_idx, value,
> value_len, ss_number_layers);
> +    } else if (!strcmp(key, "rfc_reference_last")) {
> +        vp8_ts_parse_int_array(ref_frame_config->reference_last, value,
> value_len, ss_number_layers);
> +    } else if (!strcmp(key, "rfc_reference_golden")) {
> +        vp8_ts_parse_int_array(ref_frame_config->reference_golden, value,
> value_len, ss_number_layers);
> +    } else if (!strcmp(key, "rfc_reference_alt_ref")) {
> +        vp8_ts_parse_int_array(ref_frame_config->reference_alt_ref,
> value, value_len, ss_number_layers);
> +    } else if (!strcmp(key, "rfc_reference_duration")) {
> +        vp8_ts_parse_int64_array(ref_frame_config->duration, value,
> value_len, ss_number_layers);
> +    }
> +
> +    return 0;
> +}
> +#endif
> +
> +
>  #if CONFIG_LIBVPX_VP9_ENCODER
>  static int set_pix_fmt(AVCodecContext *avctx, vpx_codec_caps_t codec_caps,
>                         struct vpx_codec_enc_cfg *enccfg,
> vpx_codec_flags_t *flags,
> @@ -1528,6 +1591,24 @@ static int vpx_encode(AVCodecContext *avctx,
> AVPacket *pkt,
>  #endif
>                  layer_id_valid = 1;
>              }
> +#if CONFIG_LIBVPX_VP9_ENCODER && defined
> (VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT)
> +            en = av_dict_get(frame->metadata, "ref-frame-config", NULL,
> 0);
> +
> +            if (en) {
> +                AVDictionaryEntry* en2 = NULL;
> +                av_dict_parse_string(&ctx->vpx_ref_frame_config,
> en->value, "=", ":", 0);
> +
> +                while ((en2 = av_dict_get(ctx->vpx_ref_frame_config, "",
> en2, AV_DICT_IGNORE_SUFFIX))) {
> +                    if (vpx_ref_frame_config_parse(ctx, enccfg, en2->key,
> en2->value, avctx->codec_id) < 0)
> +                        av_log(avctx, AV_LOG_WARNING,
> +                               "Error parsing option '%s = %s'.\n",
> +                               en2->key, en2->value);
> +                }
> +
> +                codecctl_intp(avctx, VP9E_SET_SVC_REF_FRAME_CONFIG, (int
> *)&ctx->ref_frame_config);
> +            }
> +#endif
> +
>          }
>
>          if (sd) {
> --
> 2.29.2.684.gfbc64c5ab5-goog
>
> In reply to your comment about using ABI_VERSION for guarding the changes,
the reason I used the unrelated   "VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT"
is because the ABI_VERION did not seem to have changed between the last
commit that you suggested (v1.7.0-936-g04b3d49ba) to use and the
lateset libvpx version. Can you confirm this?

Thank you,

Wonkap

On Tue, Dec 15, 2020 at 9:24 AM Wonkap Jang <wonkap@google.com> wrote:
>
> Hi James
>
> On Tue, Dec 15, 2020 at 9:18 AM Wonkap Jang <wonkap@google.com> wrote:
>>
>> In order to fine-control referencing schemes in VP9 encoding, there
>> is a need to use VP9E_SET_SVC_REF_FRAME_CONFIG method. This commit
>> provides a way to use the API through frame metadata.
>> ---
>>  doc/encoders.texi      | 32 +++++++++++++++++
>>  libavcodec/libvpxenc.c | 81 ++++++++++++++++++++++++++++++++++++++++++
>>  2 files changed, 113 insertions(+)
>>
> [...]
> In reply to your comment about using ABI_VERSION for guarding the changes, the reason I used the unrelated   "VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT" is because the ABI_VERION did not seem to have changed between the last commit that you suggested (v1.7.0-936-g04b3d49ba) to use and the lateset libvpx version. Can you confirm this?
>

I think I may have mentioned, this doesn't have to be precise. You can
just make 1.8.0 the baseline.
Also it looks like you might have some syntax errors in encoders.texi.
If you have the prerequisites, make doc can test the file locally.
https://patchwork.ffmpeg.org/project/ffmpeg/patch/20201215171825.2598712-1-wonkap@google.com/

On Thu, Dec 17, 2020 at 10:48 AM James Zern <jzern@google.com> wrote:

> On Tue, Dec 15, 2020 at 9:24 AM Wonkap Jang <wonkap@google.com> wrote:
> >
> > Hi James
> >
> > On Tue, Dec 15, 2020 at 9:18 AM Wonkap Jang <wonkap@google.com> wrote:
> >>
> >> In order to fine-control referencing schemes in VP9 encoding, there
> >> is a need to use VP9E_SET_SVC_REF_FRAME_CONFIG method. This commit
> >> provides a way to use the API through frame metadata.
> >> ---
> >>  doc/encoders.texi      | 32 +++++++++++++++++
> >>  libavcodec/libvpxenc.c | 81 ++++++++++++++++++++++++++++++++++++++++++
> >>  2 files changed, 113 insertions(+)
> >>
> > [...]
> > In reply to your comment about using ABI_VERSION for guarding the
> changes, the reason I used the unrelated
>  "VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT" is because the ABI_VERION did
> not seem to have changed between the last commit that you suggested
> (v1.7.0-936-g04b3d49ba) to use and the lateset libvpx version. Can you
> confirm this?
> >
>
> I think I may have mentioned, this doesn't have to be precise. You can
> just make 1.8.0 the baseline.
> Also it looks like you might have some syntax errors in encoders.texi.
> If you have the prerequisites, make doc can test the file locally.
>
> https://patchwork.ffmpeg.org/project/ffmpeg/patch/20201215171825.2598712-1-wonkap@google.com/


Hi James,

Maybe my local setup has outdated codebase for the doc changes...weird
because it is building without any problems... I even checked it
with "texi2html" tool.
I'll try to fix the doc compile issue. But, I don't quite get what you mean
by  "just make 1.8.0 the baseline".
What I was trying to get to was the fact that you wanted me to use
VPX_ENCODER_ABI_VERSION instead of VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT.
But, what I see is that VPX_ENCODER_ABI_VERSION has not changed even in the
latest codebase. Are you suggesting something else by saying "just make
1.8.0 the baseline"?
Is there something I am missing? :)

Let me know,

Thanks!!

Wonkap

On Thu, Dec 17, 2020 at 2:43 PM Wonkap Jang <wonkap@google.com> wrote:
>
>
>
> On Thu, Dec 17, 2020 at 10:48 AM James Zern <jzern@google.com> wrote:
>>
>> On Tue, Dec 15, 2020 at 9:24 AM Wonkap Jang <wonkap@google.com> wrote:
>> >
>> > Hi James
>> >
>> > On Tue, Dec 15, 2020 at 9:18 AM Wonkap Jang <wonkap@google.com> wrote:
>> >>
>> >> In order to fine-control referencing schemes in VP9 encoding, there
>> >> is a need to use VP9E_SET_SVC_REF_FRAME_CONFIG method. This commit
>> >> provides a way to use the API through frame metadata.
>> >> ---
>> >>  doc/encoders.texi      | 32 +++++++++++++++++
>> >>  libavcodec/libvpxenc.c | 81 ++++++++++++++++++++++++++++++++++++++++++
>> >>  2 files changed, 113 insertions(+)
>> >>
>> > [...]
>> > In reply to your comment about using ABI_VERSION for guarding the changes, the reason I used the unrelated   "VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT" is because the ABI_VERION did not seem to have changed between the last commit that you suggested (v1.7.0-936-g04b3d49ba) to use and the lateset libvpx version. Can you confirm this?
>> >
>>
>> I think I may have mentioned, this doesn't have to be precise. You can
>> just make 1.8.0 the baseline.
>> Also it looks like you might have some syntax errors in encoders.texi.
>> If you have the prerequisites, make doc can test the file locally.
>> https://patchwork.ffmpeg.org/project/ffmpeg/patch/20201215171825.2598712-1-wonkap@google.com/
>
>
> Hi James,
>
> Maybe my local setup has outdated codebase for the doc changes...weird because it is building without any problems... I even checked it with "texi2html" tool.
> I'll try to fix the doc compile issue. But, I don't quite get what you mean by  "just make 1.8.0 the baseline".
> What I was trying to get to was the fact that you wanted me to use VPX_ENCODER_ABI_VERSION instead of VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT.
> But, what I see is that VPX_ENCODER_ABI_VERSION has not changed even in the latest codebase. Are you suggesting something else by saying "just make 1.8.0 the baseline"?
> Is there something I am missing? :)
>

It's possible I misread your earlier replies. I see what you mean, the
abi version hasn't changed at all since then, though it should have.
It will change in the next release, but this check is ok. We have some
similar variants already.

> Let me know,
>
> Thanks!!
>
> Wonkap

On Mon, Dec 21, 2020 at 1:40 PM James Zern <jzern@google.com> wrote:

> On Thu, Dec 17, 2020 at 2:43 PM Wonkap Jang <wonkap@google.com> wrote:
> >
> >
> >
> > On Thu, Dec 17, 2020 at 10:48 AM James Zern <jzern@google.com> wrote:
> >>
> >> On Tue, Dec 15, 2020 at 9:24 AM Wonkap Jang <wonkap@google.com> wrote:
> >> >
> >> > Hi James
> >> >
> >> > On Tue, Dec 15, 2020 at 9:18 AM Wonkap Jang <wonkap@google.com>
> wrote:
> >> >>
> >> >> In order to fine-control referencing schemes in VP9 encoding, there
> >> >> is a need to use VP9E_SET_SVC_REF_FRAME_CONFIG method. This commit
> >> >> provides a way to use the API through frame metadata.
> >> >> ---
> >> >>  doc/encoders.texi      | 32 +++++++++++++++++
> >> >>  libavcodec/libvpxenc.c | 81
> ++++++++++++++++++++++++++++++++++++++++++
> >> >>  2 files changed, 113 insertions(+)
> >> >>
> >> > [...]
> >> > In reply to your comment about using ABI_VERSION for guarding the
> changes, the reason I used the unrelated
>  "VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT" is because the ABI_VERION did
> not seem to have changed between the last commit that you suggested
> (v1.7.0-936-g04b3d49ba) to use and the lateset libvpx version. Can you
> confirm this?
> >> >
> >>
> >> I think I may have mentioned, this doesn't have to be precise. You can
> >> just make 1.8.0 the baseline.
> >> Also it looks like you might have some syntax errors in encoders.texi.
> >> If you have the prerequisites, make doc can test the file locally.
> >>
> https://patchwork.ffmpeg.org/project/ffmpeg/patch/20201215171825.2598712-1-wonkap@google.com/
> >
> >
> > Hi James,
> >
> > Maybe my local setup has outdated codebase for the doc changes...weird
> because it is building without any problems... I even checked it with
> "texi2html" tool.
> > I'll try to fix the doc compile issue. But, I don't quite get what you
> mean by  "just make 1.8.0 the baseline".
> > What I was trying to get to was the fact that you wanted me to use
> VPX_ENCODER_ABI_VERSION instead of VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT.
> > But, what I see is that VPX_ENCODER_ABI_VERSION has not changed even in
> the latest codebase. Are you suggesting something else by saying "just make
> 1.8.0 the baseline"?
> > Is there something I am missing? :)
> >
>
> It's possible I misread your earlier replies. I see what you mean, the
> abi version hasn't changed at all since then, though it should have.
> It will change in the next release, but this check is ok. We have some
> similar variants already.
>
> > Let me know,
> >
> > Thanks!!
> >
> > Wonkap
>

OK. then, I'll send out new changes. with updated code base.

Thanks,

Wonkap