[FFmpeg-devel,01/24] hwcontext_vaapi: Try to support the VDPAU wrapper

The driver is somewhat bitrotten (not updated for years) but is still
usable for decoding with this change.  To support it, this adds a new
driver quirk to indicate no support at all for surface attributes.

Based on a patch by wm4 <nfxjfg@googlemail.com>.

(cherry picked from commit e791b915c774408fbc0ec9e7270b021899e08ccc)
---
 libavutil/hwcontext_vaapi.c | 79 ++++++++++++++++++++++++++-------------------
 libavutil/hwcontext_vaapi.h |  7 ++++
 2 files changed, 52 insertions(+), 34 deletions(-)

On Mon, 12 Jun 2017 23:40:18 +0100
Mark Thompson <sw@jkqxz.net> wrote:

> The driver is somewhat bitrotten (not updated for years) but is still
> usable for decoding with this change.  To support it, this adds a new
> driver quirk to indicate no support at all for surface attributes.
> 
> Based on a patch by wm4 <nfxjfg@googlemail.com>.
> 
> (cherry picked from commit e791b915c774408fbc0ec9e7270b021899e08ccc)
> ---
>  libavutil/hwcontext_vaapi.c | 79 ++++++++++++++++++++++++++-------------------
>  libavutil/hwcontext_vaapi.h |  7 ++++
>  2 files changed, 52 insertions(+), 34 deletions(-)
> 
> diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
> index 3b50e95615..3970726d30 100644
> --- a/libavutil/hwcontext_vaapi.c
> +++ b/libavutil/hwcontext_vaapi.c
> @@ -155,7 +155,8 @@ static int vaapi_frames_get_constraints(AVHWDeviceContext *hwdev,
>      unsigned int fourcc;
>      int err, i, j, attr_count, pix_fmt_count;
>  
> -    if (config) {
> +    if (config &&
> +        !(hwctx->driver_quirks & AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES)) {
>          attr_count = 0;
>          vas = vaQuerySurfaceAttributes(hwctx->display, config->config_id,
>                                         0, &attr_count);
> @@ -273,6 +274,11 @@ static const struct {
>          "ubit",
>          AV_VAAPI_DRIVER_QUIRK_ATTRIB_MEMTYPE,
>      },
> +    {
> +        "VDPAU wrapper",
> +        "Splitted-Desktop Systems VDPAU backend for VA-API",
> +        AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES,
> +    },
>  };
>  
>  static int vaapi_device_init(AVHWDeviceContext *hwdev)
> @@ -451,43 +457,48 @@ static int vaapi_frames_init(AVHWFramesContext *hwfc)
>      }
>  
>      if (!hwfc->pool) {
> -        int need_memory_type = !(hwctx->driver_quirks & AV_VAAPI_DRIVER_QUIRK_ATTRIB_MEMTYPE);
> -        int need_pixel_format = 1;
> -        for (i = 0; i < avfc->nb_attributes; i++) {
> -            if (ctx->attributes[i].type == VASurfaceAttribMemoryType)
> -                need_memory_type  = 0;
> -            if (ctx->attributes[i].type == VASurfaceAttribPixelFormat)
> -                need_pixel_format = 0;
> -        }
> -        ctx->nb_attributes =
> -            avfc->nb_attributes + need_memory_type + need_pixel_format;
> +        if (!(hwctx->driver_quirks & AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES)) {
> +            int need_memory_type = !(hwctx->driver_quirks & AV_VAAPI_DRIVER_QUIRK_ATTRIB_MEMTYPE);
> +            int need_pixel_format = 1;
> +            for (i = 0; i < avfc->nb_attributes; i++) {
> +                if (ctx->attributes[i].type == VASurfaceAttribMemoryType)
> +                    need_memory_type  = 0;
> +                if (ctx->attributes[i].type == VASurfaceAttribPixelFormat)
> +                    need_pixel_format = 0;
> +            }
> +            ctx->nb_attributes =
> +                avfc->nb_attributes + need_memory_type + need_pixel_format;
>  
> -        ctx->attributes = av_malloc(ctx->nb_attributes *
> +            ctx->attributes = av_malloc(ctx->nb_attributes *
>                                          sizeof(*ctx->attributes));
> -        if (!ctx->attributes) {
> -            err = AVERROR(ENOMEM);
> -            goto fail;
> -        }
> +            if (!ctx->attributes) {
> +                err = AVERROR(ENOMEM);
> +                goto fail;
> +            }
>  
> -        for (i = 0; i < avfc->nb_attributes; i++)
> -            ctx->attributes[i] = avfc->attributes[i];
> -        if (need_memory_type) {
> -            ctx->attributes[i++] = (VASurfaceAttrib) {
> -                .type          = VASurfaceAttribMemoryType,
> -                .flags         = VA_SURFACE_ATTRIB_SETTABLE,
> -                .value.type    = VAGenericValueTypeInteger,
> -                .value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_VA,
> -            };
> -        }
> -        if (need_pixel_format) {
> -            ctx->attributes[i++] = (VASurfaceAttrib) {
> -                .type          = VASurfaceAttribPixelFormat,
> -                .flags         = VA_SURFACE_ATTRIB_SETTABLE,
> -                .value.type    = VAGenericValueTypeInteger,
> -                .value.value.i = fourcc,
> -            };
> +            for (i = 0; i < avfc->nb_attributes; i++)
> +                ctx->attributes[i] = avfc->attributes[i];
> +            if (need_memory_type) {
> +                ctx->attributes[i++] = (VASurfaceAttrib) {
> +                    .type          = VASurfaceAttribMemoryType,
> +                    .flags         = VA_SURFACE_ATTRIB_SETTABLE,
> +                    .value.type    = VAGenericValueTypeInteger,
> +                    .value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_VA,
> +                };
> +            }
> +            if (need_pixel_format) {
> +                ctx->attributes[i++] = (VASurfaceAttrib) {
> +                    .type          = VASurfaceAttribPixelFormat,
> +                    .flags         = VA_SURFACE_ATTRIB_SETTABLE,
> +                    .value.type    = VAGenericValueTypeInteger,
> +                    .value.value.i = fourcc,
> +                };
> +            }
> +            av_assert0(i == ctx->nb_attributes);
> +        } else {
> +            ctx->attributes = NULL;
> +            ctx->nb_attributes = 0;
>          }
> -        av_assert0(i == ctx->nb_attributes);
>  
>          ctx->rt_format = rt_format;
>  
> diff --git a/libavutil/hwcontext_vaapi.h b/libavutil/hwcontext_vaapi.h
> index da1d4fe6c2..0b2e071cb3 100644
> --- a/libavutil/hwcontext_vaapi.h
> +++ b/libavutil/hwcontext_vaapi.h
> @@ -51,6 +51,13 @@ enum {
>       * so the surface allocation code will not try to use it.
>       */
>      AV_VAAPI_DRIVER_QUIRK_ATTRIB_MEMTYPE = (1 << 2),
> +
> +    /**
> +     * The driver does not support surface attributes at all.
> +     * The surface allocation code will never pass them to surface allocation,
> +     * and the results of the vaQuerySurfaceAttributes() call will be faked.
> +     */
> +    AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES = (1 << 3),
>  };
>  
>  /**

Fine, of course only if you want to (consenting adults etc.)