[FFmpeg-devel,RFC,v2,1/3] fftools: Add thumbnail output to vaapi_h264 decoder

Submitted by Zachary Zhou on April 8, 2019, 8:53 a.m.

Details

Message ID 20190408085334.23947-1-zachary.zhou@intel.com
State New
Headers show

Commit Message

Zachary Zhou April 8, 2019, 8:53 a.m.
This is sample code for reference

HW support for decode+scaling in a single HW command (VDBOX+SFC).
The primary target usage is video analytics, but can be used playback,
transcoding, etc.

For VAAPI -
https://github.com/intel/libva
basically, it allows multiple outputs (in different resolutions) using the decode context in a single call (you can search for “additional_outputs” in va.h).

VAAPI sample code -
https://github.com/intel/libva-utils/commit/957a269f02b00760b7e807643c821ee26abc529b
---
 fftools/ffmpeg_opt.c | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

Comments

mypopy@gmail.com April 8, 2019, 9:18 a.m.
On Mon, Apr 8, 2019 at 4:54 PM Zachary Zhou <zachary.zhou@intel.com> wrote:
>
> This is sample code for reference
>
> HW support for decode+scaling in a single HW command (VDBOX+SFC).
> The primary target usage is video analytics, but can be used playback,
> transcoding, etc.
>
> For VAAPI -
> https://github.com/intel/libva
> basically, it allows multiple outputs (in different resolutions) using the decode context in a single call (you can search for “additional_outputs” in va.h).
>
> VAAPI sample code -
> https://github.com/intel/libva-utils/commit/957a269f02b00760b7e807643c821ee26abc529b
> ---
>  fftools/ffmpeg_opt.c | 28 ++++++++++++++++++++++++++++
>  1 file changed, 28 insertions(+)
>
> diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c
> index 53d688b764..c0dc376541 100644
> --- a/fftools/ffmpeg_opt.c
> +++ b/fftools/ffmpeg_opt.c
> @@ -111,6 +111,10 @@ int filter_nbthreads = 0;
>  int filter_complex_nbthreads = 0;
>  int vstats_version = 2;
>
> +int thumbnail_flags = 0;
> +int thumbnail_width = 0;
> +int thumbnail_height = 0;
> +char *thumbnail_format;
>
>  static int intra_only         = 0;
>  static int file_overwrite     = 0;
> @@ -1100,6 +1104,13 @@ static int open_input_file(OptionsContext *o, const char *filename)
>          av_dict_set(&o->g->format_opts, "scan_all_pmts", "1", AV_DICT_DONT_OVERWRITE);
>          scan_all_pmts_set = 1;
>      }
> +
> +    //thumbnail opt
> +    av_dict_set_int(&o->g->codec_opts, "thumbnail_flags", thumbnail_flags, AV_DICT_DONT_OVERWRITE);
> +    av_dict_set_int(&o->g->codec_opts, "thumbnail_width", thumbnail_width, AV_DICT_DONT_OVERWRITE);
> +    av_dict_set_int(&o->g->codec_opts, "thumbnail_height", thumbnail_height, AV_DICT_DONT_OVERWRITE);
> +    av_dict_set(&o->g->codec_opts, "thumbnail_format", thumbnail_format, AV_DICT_DONT_OVERWRITE);
> +
>      /* open the input file with generic avformat function */
>      err = avformat_open_input(&ic, filename, file_iformat, &o->g->format_opts);
>      if (err < 0) {
> @@ -2898,6 +2909,13 @@ static int opt_vstats_file(void *optctx, const char *opt, const char *arg)
>      return 0;
>  }
>
> +static int opt_thumbnail_format(void *optctx, const char *opt, const char *arg)
> +{
> +    av_free (thumbnail_format);
> +    thumbnail_format = av_strdup (arg);
> +    return 0;
> +}
> +
>  static int opt_vstats(void *optctx, const char *opt, const char *arg)
>  {
>      char filename[40];
> @@ -3746,5 +3764,15 @@ const OptionDef options[] = {
>      { "filter_hw_device", HAS_ARG | OPT_EXPERT, { .func_arg = opt_filter_hw_device },
>          "set hardware device used when filtering", "device" },
>
> +    //thumbnail opt
> +    { "thumbnail_flags",    OPT_VIDEO | HAS_ARG | OPT_INT | OPT_INPUT, { &thumbnail_flags },
> +      "set thumbnail flags", "thumbnail" },
> +    { "thumbnail_width",    OPT_VIDEO | HAS_ARG | OPT_INT | OPT_INPUT, { &thumbnail_width },
> +      "set thumbnail width", "thumbnail" },
> +    { "thumbnail_height",   OPT_VIDEO | HAS_ARG | OPT_INT | OPT_INPUT, { &thumbnail_height },
> +      "set thumbnail height", "thumbnail" },
> +    { "thumbnail_format",  OPT_VIDEO | HAS_ARG | OPT_EXPERT ,  { .func_arg = opt_thumbnail_format },
> +        "set thumbnail format", "thumbnail" },
> +
>      { NULL, },
>  };
> --
> 2.17.1

Changed the FFmpeg tools for a hardware specific feature is not good
idea, I think.

The other question is, how to enable this function from FFmpeg API
level? (We can not just enable a feature from the FFmpeg command
tools).
Zachary Zhou April 10, 2019, 2:11 a.m.
On 4/8/19 5:18 PM, mypopy@gmail.com wrote:
> On Mon, Apr 8, 2019 at 4:54 PM Zachary Zhou <zachary.zhou@intel.com> wrote:
>> This is sample code for reference
>>
>> HW support for decode+scaling in a single HW command (VDBOX+SFC).
>> The primary target usage is video analytics, but can be used playback,
>> transcoding, etc.
>>
>> For VAAPI -
>> https://github.com/intel/libva
>> basically, it allows multiple outputs (in different resolutions) using the decode context in a single call (you can search for “additional_outputs” in va.h).
>>
>> VAAPI sample code -
>> https://github.com/intel/libva-utils/commit/957a269f02b00760b7e807643c821ee26abc529b
>> ---
>>   fftools/ffmpeg_opt.c | 28 ++++++++++++++++++++++++++++
>>   1 file changed, 28 insertions(+)
>>
>> diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c
>> index 53d688b764..c0dc376541 100644
>> --- a/fftools/ffmpeg_opt.c
>> +++ b/fftools/ffmpeg_opt.c
>> @@ -111,6 +111,10 @@ int filter_nbthreads = 0;
>>   int filter_complex_nbthreads = 0;
>>   int vstats_version = 2;
>>
>> +int thumbnail_flags = 0;
>> +int thumbnail_width = 0;
>> +int thumbnail_height = 0;
>> +char *thumbnail_format;
>>
>>   static int intra_only         = 0;
>>   static int file_overwrite     = 0;
>> @@ -1100,6 +1104,13 @@ static int open_input_file(OptionsContext *o, const char *filename)
>>           av_dict_set(&o->g->format_opts, "scan_all_pmts", "1", AV_DICT_DONT_OVERWRITE);
>>           scan_all_pmts_set = 1;
>>       }
>> +
>> +    //thumbnail opt
>> +    av_dict_set_int(&o->g->codec_opts, "thumbnail_flags", thumbnail_flags, AV_DICT_DONT_OVERWRITE);
>> +    av_dict_set_int(&o->g->codec_opts, "thumbnail_width", thumbnail_width, AV_DICT_DONT_OVERWRITE);
>> +    av_dict_set_int(&o->g->codec_opts, "thumbnail_height", thumbnail_height, AV_DICT_DONT_OVERWRITE);
>> +    av_dict_set(&o->g->codec_opts, "thumbnail_format", thumbnail_format, AV_DICT_DONT_OVERWRITE);
>> +
>>       /* open the input file with generic avformat function */
>>       err = avformat_open_input(&ic, filename, file_iformat, &o->g->format_opts);
>>       if (err < 0) {
>> @@ -2898,6 +2909,13 @@ static int opt_vstats_file(void *optctx, const char *opt, const char *arg)
>>       return 0;
>>   }
>>
>> +static int opt_thumbnail_format(void *optctx, const char *opt, const char *arg)
>> +{
>> +    av_free (thumbnail_format);
>> +    thumbnail_format = av_strdup (arg);
>> +    return 0;
>> +}
>> +
>>   static int opt_vstats(void *optctx, const char *opt, const char *arg)
>>   {
>>       char filename[40];
>> @@ -3746,5 +3764,15 @@ const OptionDef options[] = {
>>       { "filter_hw_device", HAS_ARG | OPT_EXPERT, { .func_arg = opt_filter_hw_device },
>>           "set hardware device used when filtering", "device" },
>>
>> +    //thumbnail opt
>> +    { "thumbnail_flags",    OPT_VIDEO | HAS_ARG | OPT_INT | OPT_INPUT, { &thumbnail_flags },
>> +      "set thumbnail flags", "thumbnail" },
>> +    { "thumbnail_width",    OPT_VIDEO | HAS_ARG | OPT_INT | OPT_INPUT, { &thumbnail_width },
>> +      "set thumbnail width", "thumbnail" },
>> +    { "thumbnail_height",   OPT_VIDEO | HAS_ARG | OPT_INT | OPT_INPUT, { &thumbnail_height },
>> +      "set thumbnail height", "thumbnail" },
>> +    { "thumbnail_format",  OPT_VIDEO | HAS_ARG | OPT_EXPERT ,  { .func_arg = opt_thumbnail_format },
>> +        "set thumbnail format", "thumbnail" },
>> +
>>       { NULL, },
>>   };
>> --
>> 2.17.1
> Changed the FFmpeg tools for a hardware specific feature is not good
> idea, I think.
>
> The other question is, how to enable this function from FFmpeg API
> level? (We can not just enable a feature from the FFmpeg command
> tools).
>
Thanks for the comments. The feature mostly changes decoder in ffmpeg.

I have another two patches like below -

https://patchwork.ffmpeg.org/patch/12640/

    https://patchwork.ffmpeg.org/patch/12641/

These two patches show the changes on FFmpeg API level.

-Zach

Patch hide | download patch | download mbox

diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c
index 53d688b764..c0dc376541 100644
--- a/fftools/ffmpeg_opt.c
+++ b/fftools/ffmpeg_opt.c
@@ -111,6 +111,10 @@  int filter_nbthreads = 0;
 int filter_complex_nbthreads = 0;
 int vstats_version = 2;
 
+int thumbnail_flags = 0;
+int thumbnail_width = 0;
+int thumbnail_height = 0;
+char *thumbnail_format;
 
 static int intra_only         = 0;
 static int file_overwrite     = 0;
@@ -1100,6 +1104,13 @@  static int open_input_file(OptionsContext *o, const char *filename)
         av_dict_set(&o->g->format_opts, "scan_all_pmts", "1", AV_DICT_DONT_OVERWRITE);
         scan_all_pmts_set = 1;
     }
+
+    //thumbnail opt
+    av_dict_set_int(&o->g->codec_opts, "thumbnail_flags", thumbnail_flags, AV_DICT_DONT_OVERWRITE);
+    av_dict_set_int(&o->g->codec_opts, "thumbnail_width", thumbnail_width, AV_DICT_DONT_OVERWRITE);
+    av_dict_set_int(&o->g->codec_opts, "thumbnail_height", thumbnail_height, AV_DICT_DONT_OVERWRITE);
+    av_dict_set(&o->g->codec_opts, "thumbnail_format", thumbnail_format, AV_DICT_DONT_OVERWRITE);
+
     /* open the input file with generic avformat function */
     err = avformat_open_input(&ic, filename, file_iformat, &o->g->format_opts);
     if (err < 0) {
@@ -2898,6 +2909,13 @@  static int opt_vstats_file(void *optctx, const char *opt, const char *arg)
     return 0;
 }
 
+static int opt_thumbnail_format(void *optctx, const char *opt, const char *arg)
+{
+    av_free (thumbnail_format);
+    thumbnail_format = av_strdup (arg);
+    return 0;
+}
+
 static int opt_vstats(void *optctx, const char *opt, const char *arg)
 {
     char filename[40];
@@ -3746,5 +3764,15 @@  const OptionDef options[] = {
     { "filter_hw_device", HAS_ARG | OPT_EXPERT, { .func_arg = opt_filter_hw_device },
         "set hardware device used when filtering", "device" },
 
+    //thumbnail opt
+    { "thumbnail_flags",    OPT_VIDEO | HAS_ARG | OPT_INT | OPT_INPUT, { &thumbnail_flags },
+      "set thumbnail flags", "thumbnail" },
+    { "thumbnail_width",    OPT_VIDEO | HAS_ARG | OPT_INT | OPT_INPUT, { &thumbnail_width },
+      "set thumbnail width", "thumbnail" },
+    { "thumbnail_height",   OPT_VIDEO | HAS_ARG | OPT_INT | OPT_INPUT, { &thumbnail_height },
+      "set thumbnail height", "thumbnail" },
+    { "thumbnail_format",  OPT_VIDEO | HAS_ARG | OPT_EXPERT ,  { .func_arg = opt_thumbnail_format },
+        "set thumbnail format", "thumbnail" },
+
     { NULL, },
 };