diff mbox series

[FFmpeg-devel,V5,5/5] libavutil/hwcontext_vulkan: specify the modifier to create VKImage

Message ID 20211202014320.1066212-5-wenbin.chen@intel.com
State New
Headers show
Series [FFmpeg-devel,V5,1/5] hwcontext_vaapi: Use PRIME_2 memory type for modifiers. | expand

Checks

Context Check Description
andriy/make_x86 success Make finished
andriy/make_fate_x86 fail Make fate failed
andriy/make_ppc success Make finished
andriy/make_fate_ppc fail Make fate failed

Commit Message

Wenbin Chen Dec. 2, 2021, 1:43 a.m. UTC 
When vulkan image exports to drm, the tilling need to be
VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT. Now add code to create vulkan
image using this format.

Now the following command line works:

ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 -hwaccel_output_format \
vaapi -i input_1080p.264 -vf "hwmap=derive_device=vulkan,format=vulkan, \
scale_vulkan=1920:1080,hwmap=derive_device=vaapi,format=vaapi" -c:v h264_vaapi output.264

Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
---
 libavutil/hwcontext_vulkan.c | 127 +++++++++++++++++++++++++++++++++--
 1 file changed, 121 insertions(+), 6 deletions(-)

Comments

Lynne Dec. 3, 2021, 10:58 p.m. UTC | #1 
2 Dec 2021, 02:43 by wenbin.chen@intel.com:

> When vulkan image exports to drm, the tilling need to be
> VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT. Now add code to create vulkan
> image using this format.
>
> Now the following command line works:
>
> ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 -hwaccel_output_format \
> vaapi -i input_1080p.264 -vf "hwmap=derive_device=vulkan,format=vulkan, \
> scale_vulkan=1920:1080,hwmap=derive_device=vaapi,format=vaapi" -c:v h264_vaapi output.264
>
> Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
> ---
>  libavutil/hwcontext_vulkan.c | 127 +++++++++++++++++++++++++++++++++--
>  1 file changed, 121 insertions(+), 6 deletions(-)
>
> diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
> index f980b72720..8224c0d4e4 100644
> --- a/libavutil/hwcontext_vulkan.c
> +++ b/libavutil/hwcontext_vulkan.c
> @@ -120,6 +120,9 @@ typedef struct VulkanFramesPriv {
>  /* Image transfers */
>  VulkanExecCtx upload_ctx;
>  VulkanExecCtx download_ctx;
> +
> +    /*modifier info*/
> +    VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
>  } VulkanFramesPriv;
>  
>  typedef struct AVVkFrameInternal {
> @@ -242,6 +245,28 @@ const VkFormat *av_vkfmt_from_pixfmt(enum AVPixelFormat p)
>  return NULL;
>  }
>  
> +static void *find_in_structure_list(VkBaseOutStructure *stru_list, VkStructureType sType) {
> +    if (!stru_list)
> +        return NULL;
> +
> +    for(;stru_list;stru_list = stru_list->pNext)
> +        if (stru_list->sType == sType)
> +            return stru_list;
> +
> +    return NULL;
> +}
> +
> +static void append_to_structure_list(VkBaseOutStructure **stru_list, VkBaseOutStructure *added_stru) {
> +    VkBaseOutStructure *p;
> +    if (!*stru_list) {
> +        *stru_list = added_stru;
> +        return;
> +    }
> +    for(p = *stru_list; p->pNext; p = p->pNext);
> +    p->pNext = added_stru;
> +    return;
> +}
>

Could you tidy these 2 functions up? Also, code style issues, as usual.
Take a look at libplacebo's functions or copy them, they're much
neater: https://0x1.st/tx.txt


>  static int pixfmt_is_supported(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p,
>  int linear)
>  {
> @@ -2094,6 +2119,10 @@ static void try_export_flags(AVHWFramesContext *hwfc,
>  AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
>  VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
>  FFVulkanFunctions *vk = &p->vkfn;
> +    const int has_modifiers = hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
> +    int loop_count;
> +    VkImageDrmFormatModifierListCreateInfoEXT *modifier_info = find_in_structure_list(hwctx->create_pnext,
> +                                        VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
>  VkExternalImageFormatProperties eprops = {
>  .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
>  };
> @@ -2101,9 +2130,18 @@ static void try_export_flags(AVHWFramesContext *hwfc,
>  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
>  .pNext = &eprops,
>  };
> +    VkPhysicalDeviceImageDrmFormatModifierInfoEXT phy_dev_mod_info = {
> +        .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
> +        .pNext = NULL,
> +        .pQueueFamilyIndices   = p->qfs,
> +        .queueFamilyIndexCount = p->num_qfs,
> +        .sharingMode           = p->num_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
> +                                                  VK_SHARING_MODE_EXCLUSIVE,
> +    };
>  VkPhysicalDeviceExternalImageFormatInfo enext = {
>  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
>  .handleType = exp,
> +        .pNext = has_modifiers && modifier_info ? &phy_dev_mod_info : NULL,
>  };
>  VkPhysicalDeviceImageFormatInfo2 pinfo = {
>  .sType  = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
> @@ -2115,11 +2153,16 @@ static void try_export_flags(AVHWFramesContext *hwfc,
>  .flags  = VK_IMAGE_CREATE_ALIAS_BIT,
>  };
>  
> -    ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
> -                                                      &pinfo, &props);
> -    if (ret == VK_SUCCESS) {
> -        *iexp |= exp;
> -        *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
> +    loop_count = has_modifiers && modifier_info ? modifier_info->drmFormatModifierCount : 1;
> +    for (int i = 0; i < loop_count; i++) {
> +        if (has_modifiers && modifier_info)
> +            phy_dev_mod_info.drmFormatModifier = modifier_info->pDrmFormatModifiers[i];
> +        ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
> +                                                        &pinfo, &props);
> +        if (ret == VK_SUCCESS) {
> +            *iexp |= exp;
> +            *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
> +        }
>  }
>  }
>  
> @@ -2190,6 +2233,12 @@ static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
>  {
>  VulkanFramesPriv *fp = hwfc->internal->priv;
>  
> +    if (fp->modifier_info) {
> +        if (fp->modifier_info->pDrmFormatModifiers)
> +            av_freep(&fp->modifier_info->pDrmFormatModifiers);
> +        av_freep(&fp->modifier_info);
> +    }
> +
>  free_exec_ctx(hwfc, &fp->conv_ctx);
>  free_exec_ctx(hwfc, &fp->upload_ctx);
>  free_exec_ctx(hwfc, &fp->download_ctx);
> @@ -2203,11 +2252,77 @@ static int vulkan_frames_init(AVHWFramesContext *hwfc)
>  VulkanFramesPriv *fp = hwfc->internal->priv;
>  AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
>  VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
> +    VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
> +    const int has_modifiers = !!(p->extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS);
>  
>  /* Default pool flags */
> -    hwctx->tiling = hwctx->tiling ? hwctx->tiling : p->use_linear_images ?
> +    hwctx->tiling = hwctx->tiling ? hwctx->tiling : has_modifiers ?
> +                    VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT : p->use_linear_images ?
>  VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL;
>  
> +    modifier_info = find_in_structure_list(hwctx->create_pnext,
> +                                    VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
> +    /* get the supported modifier */
> +    if (has_modifiers && !modifier_info) {
> +        const VkFormat *fmt = av_vkfmt_from_pixfmt(hwfc->sw_format);
> +        FFVulkanFunctions *vk = &p->vkfn;
> +        VkDrmFormatModifierPropertiesEXT *mod_props;
> +        uint64_t *modifiers;
> +        int modifier_count = 0;
> +
> +        VkDrmFormatModifierPropertiesListEXT mod_props_list = {
> +            .sType = VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT,
> +            .pNext = NULL,
> +            .drmFormatModifierCount = 0,
> +            .pDrmFormatModifierProperties = NULL,
> +        };
> +        VkFormatProperties2 prop = {
> +            .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
> +            .pNext = &mod_props_list,
> +        };
> +        vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt[0], &prop);
> +        if (!mod_props_list.drmFormatModifierCount) {
> +            av_log(hwfc, AV_LOG_ERROR, "There are not supported modifiers for sw_format\n");
>

Grammar issues: should be something like
"There are no supported modifiers for given sw_format\n".


> +            return AVERROR(EINVAL);
> +        }
> +
> +        modifier_info = av_malloc(sizeof(*modifier_info));
> +        if (!modifier_info)
> +            return AVERROR(ENOMEM);
> +        append_to_structure_list((VkBaseOutStructure **)&hwctx->create_pnext,
> +                                 (VkBaseOutStructure *)modifier_info);
> +        fp->modifier_info = modifier_info;
> +        modifier_info->pNext = NULL;
> +        modifier_info->sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT;
> +
> +        modifiers = av_malloc(mod_props_list.drmFormatModifierCount *
> +                              sizeof(*modifiers));
> +        if (!modifiers)
> +            return AVERROR(ENOMEM);
> +        modifier_info->pDrmFormatModifiers = modifiers;
> +
> +        mod_props = av_malloc(mod_props_list.drmFormatModifierCount *
> +                              sizeof(*mod_props));
> +        if (!mod_props)
> +            return AVERROR(ENOMEM);
> +        mod_props_list.pDrmFormatModifierProperties = mod_props;
> +        vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt[0], &prop);
> +        for (int i = 0; i < mod_props_list.drmFormatModifierCount; i++) {
> +            if (!(mod_props[i].drmFormatModifierTilingFeatures & FF_VK_DEFAULT_USAGE_FLAGS))
>

You should use the usage flags given by the user in AVVulkanFramesContext,
not the default ones (unless none were specified by the user, but in that case,
that field is already set to the default usage flags).


> +                continue;
> +            modifiers[modifier_count++] = mod_props[i].drmFormatModifier;
> +        }
> +        if (!modifier_count) {
> +            av_log(hwfc, AV_LOG_ERROR, "The supported modifiers doesn't support "
> +                                       "default usage\n");
>

Same, should be something like:
"None of the given modifiers supports the usage flags!"


> +            av_freep(&mod_props);
> +            return AVERROR(EINVAL);
> +        }
> +        modifier_info->drmFormatModifierCount = modifier_count;
> +        av_freep(&mod_props);
> +    }
> +
> +
>  if (!hwctx->usage)
>  hwctx->usage = FF_VK_DEFAULT_USAGE_FLAGS;
>  
> -- 
> 2.25.1
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
Wenbin Chen Dec. 6, 2021, 5:48 a.m. UTC | #2 
> 2 Dec 2021, 02:43 by wenbin.chen@intel.com:
> 
> > When vulkan image exports to drm, the tilling need to be
> > VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT. Now add code to
> create vulkan
> > image using this format.
> >
> > Now the following command line works:
> >
> > ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 -
> hwaccel_output_format \
> > vaapi -i input_1080p.264 -vf "hwmap=derive_device=vulkan,format=vulkan,
> \
> > scale_vulkan=1920:1080,hwmap=derive_device=vaapi,format=vaapi" -c:v
> h264_vaapi output.264
> >
> > Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
> > ---
> >  libavutil/hwcontext_vulkan.c | 127
> +++++++++++++++++++++++++++++++++--
> >  1 file changed, 121 insertions(+), 6 deletions(-)
> >
> > diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
> > index f980b72720..8224c0d4e4 100644
> > --- a/libavutil/hwcontext_vulkan.c
> > +++ b/libavutil/hwcontext_vulkan.c
> > @@ -120,6 +120,9 @@ typedef struct VulkanFramesPriv {
> >  /* Image transfers */
> >  VulkanExecCtx upload_ctx;
> >  VulkanExecCtx download_ctx;
> > +
> > +    /*modifier info*/
> > +    VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
> >  } VulkanFramesPriv;
> >
> >  typedef struct AVVkFrameInternal {
> > @@ -242,6 +245,28 @@ const VkFormat *av_vkfmt_from_pixfmt(enum
> AVPixelFormat p)
> >  return NULL;
> >  }
> >
> > +static void *find_in_structure_list(VkBaseOutStructure *stru_list,
> VkStructureType sType) {
> > +    if (!stru_list)
> > +        return NULL;
> > +
> > +    for(;stru_list;stru_list = stru_list->pNext)
> > +        if (stru_list->sType == sType)
> > +            return stru_list;
> > +
> > +    return NULL;
> > +}
> > +
> > +static void append_to_structure_list(VkBaseOutStructure **stru_list,
> VkBaseOutStructure *added_stru) {
> > +    VkBaseOutStructure *p;
> > +    if (!*stru_list) {
> > +        *stru_list = added_stru;
> > +        return;
> > +    }
> > +    for(p = *stru_list; p->pNext; p = p->pNext);
> > +    p->pNext = added_stru;
> > +    return;
> > +}
> >
> 
> Could you tidy these 2 functions up? Also, code style issues, as usual.
> Take a look at libplacebo's functions or copy them, they're much
> neater: https://0x1.st/tx.txt
> 
> 
> >  static int pixfmt_is_supported(AVHWDeviceContext *dev_ctx, enum
> AVPixelFormat p,
> >  int linear)
> >  {
> > @@ -2094,6 +2119,10 @@ static void
> try_export_flags(AVHWFramesContext *hwfc,
> >  AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
> >  VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
> >  FFVulkanFunctions *vk = &p->vkfn;
> > +    const int has_modifiers = hwctx->tiling ==
> VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
> > +    int loop_count;
> > +    VkImageDrmFormatModifierListCreateInfoEXT *modifier_info =
> find_in_structure_list(hwctx->create_pnext,
> > +
> VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INF
> O_EXT);
> >  VkExternalImageFormatProperties eprops = {
> >  .sType =
> VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
> >  };
> > @@ -2101,9 +2130,18 @@ static void
> try_export_flags(AVHWFramesContext *hwfc,
> >  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
> >  .pNext = &eprops,
> >  };
> > +    VkPhysicalDeviceImageDrmFormatModifierInfoEXT phy_dev_mod_info
> = {
> > +        .sType =
> VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER
> _INFO_EXT,
> > +        .pNext = NULL,
> > +        .pQueueFamilyIndices   = p->qfs,
> > +        .queueFamilyIndexCount = p->num_qfs,
> > +        .sharingMode           = p->num_qfs > 1 ?
> VK_SHARING_MODE_CONCURRENT :
> > +                                                  VK_SHARING_MODE_EXCLUSIVE,
> > +    };
> >  VkPhysicalDeviceExternalImageFormatInfo enext = {
> >  .sType =
> VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
> >  .handleType = exp,
> > +        .pNext = has_modifiers && modifier_info ? &phy_dev_mod_info :
> NULL,
> >  };
> >  VkPhysicalDeviceImageFormatInfo2 pinfo = {
> >  .sType  =
> VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
> > @@ -2115,11 +2153,16 @@ static void
> try_export_flags(AVHWFramesContext *hwfc,
> >  .flags  = VK_IMAGE_CREATE_ALIAS_BIT,
> >  };
> >
> > -    ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx-
> >phys_dev,
> > -                                                      &pinfo, &props);
> > -    if (ret == VK_SUCCESS) {
> > -        *iexp |= exp;
> > -        *comp_handle_types |=
> eprops.externalMemoryProperties.compatibleHandleTypes;
> > +    loop_count = has_modifiers && modifier_info ? modifier_info-
> >drmFormatModifierCount : 1;
> > +    for (int i = 0; i < loop_count; i++) {
> > +        if (has_modifiers && modifier_info)
> > +            phy_dev_mod_info.drmFormatModifier = modifier_info-
> >pDrmFormatModifiers[i];
> > +        ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx-
> >phys_dev,
> > +                                                        &pinfo, &props);
> > +        if (ret == VK_SUCCESS) {
> > +            *iexp |= exp;
> > +            *comp_handle_types |=
> eprops.externalMemoryProperties.compatibleHandleTypes;
> > +        }
> >  }
> >  }
> >
> > @@ -2190,6 +2233,12 @@ static void
> vulkan_frames_uninit(AVHWFramesContext *hwfc)
> >  {
> >  VulkanFramesPriv *fp = hwfc->internal->priv;
> >
> > +    if (fp->modifier_info) {
> > +        if (fp->modifier_info->pDrmFormatModifiers)
> > +            av_freep(&fp->modifier_info->pDrmFormatModifiers);
> > +        av_freep(&fp->modifier_info);
> > +    }
> > +
> >  free_exec_ctx(hwfc, &fp->conv_ctx);
> >  free_exec_ctx(hwfc, &fp->upload_ctx);
> >  free_exec_ctx(hwfc, &fp->download_ctx);
> > @@ -2203,11 +2252,77 @@ static int
> vulkan_frames_init(AVHWFramesContext *hwfc)
> >  VulkanFramesPriv *fp = hwfc->internal->priv;
> >  AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
> >  VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
> > +    VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
> > +    const int has_modifiers = !!(p->extensions &
> FF_VK_EXT_DRM_MODIFIER_FLAGS);
> >
> >  /* Default pool flags */
> > -    hwctx->tiling = hwctx->tiling ? hwctx->tiling : p->use_linear_images ?
> > +    hwctx->tiling = hwctx->tiling ? hwctx->tiling : has_modifiers ?
> > +                    VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT : p-
> >use_linear_images ?
> >  VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL;
> >
> > +    modifier_info = find_in_structure_list(hwctx->create_pnext,
> > +
> VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INF
> O_EXT);
> > +    /* get the supported modifier */
> > +    if (has_modifiers && !modifier_info) {
> > +        const VkFormat *fmt = av_vkfmt_from_pixfmt(hwfc->sw_format);
> > +        FFVulkanFunctions *vk = &p->vkfn;
> > +        VkDrmFormatModifierPropertiesEXT *mod_props;
> > +        uint64_t *modifiers;
> > +        int modifier_count = 0;
> > +
> > +        VkDrmFormatModifierPropertiesListEXT mod_props_list = {
> > +            .sType =
> VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT,
> > +            .pNext = NULL,
> > +            .drmFormatModifierCount = 0,
> > +            .pDrmFormatModifierProperties = NULL,
> > +        };
> > +        VkFormatProperties2 prop = {
> > +            .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
> > +            .pNext = &mod_props_list,
> > +        };
> > +        vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev,
> fmt[0], &prop);
> > +        if (!mod_props_list.drmFormatModifierCount) {
> > +            av_log(hwfc, AV_LOG_ERROR, "There are not supported modifiers
> for sw_format\n");
> >
> 
> Grammar issues: should be something like
> "There are no supported modifiers for given sw_format\n".
> 
> 
> > +            return AVERROR(EINVAL);
> > +        }
> > +
> > +        modifier_info = av_malloc(sizeof(*modifier_info));
> > +        if (!modifier_info)
> > +            return AVERROR(ENOMEM);
> > +        append_to_structure_list((VkBaseOutStructure **)&hwctx-
> >create_pnext,
> > +                                 (VkBaseOutStructure *)modifier_info);
> > +        fp->modifier_info = modifier_info;
> > +        modifier_info->pNext = NULL;
> > +        modifier_info->sType =
> VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INF
> O_EXT;
> > +
> > +        modifiers = av_malloc(mod_props_list.drmFormatModifierCount *
> > +                              sizeof(*modifiers));
> > +        if (!modifiers)
> > +            return AVERROR(ENOMEM);
> > +        modifier_info->pDrmFormatModifiers = modifiers;
> > +
> > +        mod_props = av_malloc(mod_props_list.drmFormatModifierCount *
> > +                              sizeof(*mod_props));
> > +        if (!mod_props)
> > +            return AVERROR(ENOMEM);
> > +        mod_props_list.pDrmFormatModifierProperties = mod_props;
> > +        vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev,
> fmt[0], &prop);
> > +        for (int i = 0; i < mod_props_list.drmFormatModifierCount; i++) {
> > +            if (!(mod_props[i].drmFormatModifierTilingFeatures &
> FF_VK_DEFAULT_USAGE_FLAGS))
> >
> 
> You should use the usage flags given by the user in AVVulkanFramesContext,
> not the default ones (unless none were specified by the user, but in that case,
> that field is already set to the default usage flags).
> 
> 
> > +                continue;
> > +            modifiers[modifier_count++] = mod_props[i].drmFormatModifier;
> > +        }
> > +        if (!modifier_count) {
> > +            av_log(hwfc, AV_LOG_ERROR, "The supported modifiers doesn't
> support "
> > +                                       "default usage\n");
> >
> 
> Same, should be something like:
> "None of the given modifiers supports the usage flags!"
> 
> 
> > +            av_freep(&mod_props);
> > +            return AVERROR(EINVAL);
> > +        }
> > +        modifier_info->drmFormatModifierCount = modifier_count;
> > +        av_freep(&mod_props);
> > +    }
> > +
> > +
> >  if (!hwctx->usage)
> >  hwctx->usage = FF_VK_DEFAULT_USAGE_FLAGS;
> >
> > --
> > 2.25.1
> >

Thanks for review
I will fix this problem in next patchset.

> > _______________________________________________
> > ffmpeg-devel mailing list
> > ffmpeg-devel@ffmpeg.org
> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >
> > To unsubscribe, visit link above, or email
> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
> >
> 
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
diff mbox series

Patch

diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
index f980b72720..8224c0d4e4 100644
--- a/libavutil/hwcontext_vulkan.c
+++ b/libavutil/hwcontext_vulkan.c
@@ -120,6 +120,9 @@  typedef struct VulkanFramesPriv {
     /* Image transfers */
     VulkanExecCtx upload_ctx;
     VulkanExecCtx download_ctx;
+
+    /*modifier info*/
+    VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
 } VulkanFramesPriv;
 
 typedef struct AVVkFrameInternal {
@@ -242,6 +245,28 @@  const VkFormat *av_vkfmt_from_pixfmt(enum AVPixelFormat p)
     return NULL;
 }
 
+static void *find_in_structure_list(VkBaseOutStructure *stru_list, VkStructureType sType) {
+    if (!stru_list)
+        return NULL;
+
+    for(;stru_list;stru_list = stru_list->pNext)
+        if (stru_list->sType == sType)
+            return stru_list;
+
+    return NULL;
+}
+
+static void append_to_structure_list(VkBaseOutStructure **stru_list, VkBaseOutStructure *added_stru) {
+    VkBaseOutStructure *p;
+    if (!*stru_list) {
+        *stru_list = added_stru;
+        return;
+    }
+    for(p = *stru_list; p->pNext; p = p->pNext);
+    p->pNext = added_stru;
+    return;
+}
+
 static int pixfmt_is_supported(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p,
                                int linear)
 {
@@ -2094,6 +2119,10 @@  static void try_export_flags(AVHWFramesContext *hwfc,
     AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
     VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
     FFVulkanFunctions *vk = &p->vkfn;
+    const int has_modifiers = hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
+    int loop_count;
+    VkImageDrmFormatModifierListCreateInfoEXT *modifier_info = find_in_structure_list(hwctx->create_pnext,
+                                        VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
     VkExternalImageFormatProperties eprops = {
         .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
     };
@@ -2101,9 +2130,18 @@  static void try_export_flags(AVHWFramesContext *hwfc,
         .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
         .pNext = &eprops,
     };
+    VkPhysicalDeviceImageDrmFormatModifierInfoEXT phy_dev_mod_info = {
+        .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
+        .pNext = NULL,
+        .pQueueFamilyIndices   = p->qfs,
+        .queueFamilyIndexCount = p->num_qfs,
+        .sharingMode           = p->num_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
+                                                  VK_SHARING_MODE_EXCLUSIVE,
+    };
     VkPhysicalDeviceExternalImageFormatInfo enext = {
         .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
         .handleType = exp,
+        .pNext = has_modifiers && modifier_info ? &phy_dev_mod_info : NULL,
     };
     VkPhysicalDeviceImageFormatInfo2 pinfo = {
         .sType  = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
@@ -2115,11 +2153,16 @@  static void try_export_flags(AVHWFramesContext *hwfc,
         .flags  = VK_IMAGE_CREATE_ALIAS_BIT,
     };
 
-    ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
-                                                      &pinfo, &props);
-    if (ret == VK_SUCCESS) {
-        *iexp |= exp;
-        *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
+    loop_count = has_modifiers && modifier_info ? modifier_info->drmFormatModifierCount : 1;
+    for (int i = 0; i < loop_count; i++) {
+        if (has_modifiers && modifier_info)
+            phy_dev_mod_info.drmFormatModifier = modifier_info->pDrmFormatModifiers[i];
+        ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
+                                                        &pinfo, &props);
+        if (ret == VK_SUCCESS) {
+            *iexp |= exp;
+            *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
+        }
     }
 }
 
@@ -2190,6 +2233,12 @@  static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
 {
     VulkanFramesPriv *fp = hwfc->internal->priv;
 
+    if (fp->modifier_info) {
+        if (fp->modifier_info->pDrmFormatModifiers)
+            av_freep(&fp->modifier_info->pDrmFormatModifiers);
+        av_freep(&fp->modifier_info);
+    }
+
     free_exec_ctx(hwfc, &fp->conv_ctx);
     free_exec_ctx(hwfc, &fp->upload_ctx);
     free_exec_ctx(hwfc, &fp->download_ctx);
@@ -2203,11 +2252,77 @@  static int vulkan_frames_init(AVHWFramesContext *hwfc)
     VulkanFramesPriv *fp = hwfc->internal->priv;
     AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
     VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
+    VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
+    const int has_modifiers = !!(p->extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS);
 
     /* Default pool flags */
-    hwctx->tiling = hwctx->tiling ? hwctx->tiling : p->use_linear_images ?
+    hwctx->tiling = hwctx->tiling ? hwctx->tiling : has_modifiers ?
+                    VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT : p->use_linear_images ?
                     VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL;
 
+    modifier_info = find_in_structure_list(hwctx->create_pnext,
+                                    VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
+    /* get the supported modifier */
+    if (has_modifiers && !modifier_info) {
+        const VkFormat *fmt = av_vkfmt_from_pixfmt(hwfc->sw_format);
+        FFVulkanFunctions *vk = &p->vkfn;
+        VkDrmFormatModifierPropertiesEXT *mod_props;
+        uint64_t *modifiers;
+        int modifier_count = 0;
+
+        VkDrmFormatModifierPropertiesListEXT mod_props_list = {
+            .sType = VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT,
+            .pNext = NULL,
+            .drmFormatModifierCount = 0,
+            .pDrmFormatModifierProperties = NULL,
+        };
+        VkFormatProperties2 prop = {
+            .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
+            .pNext = &mod_props_list,
+        };
+        vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt[0], &prop);
+        if (!mod_props_list.drmFormatModifierCount) {
+            av_log(hwfc, AV_LOG_ERROR, "There are not supported modifiers for sw_format\n");
+            return AVERROR(EINVAL);
+        }
+
+        modifier_info = av_malloc(sizeof(*modifier_info));
+        if (!modifier_info)
+            return AVERROR(ENOMEM);
+        append_to_structure_list((VkBaseOutStructure **)&hwctx->create_pnext,
+                                 (VkBaseOutStructure *)modifier_info);
+        fp->modifier_info = modifier_info;
+        modifier_info->pNext = NULL;
+        modifier_info->sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT;
+
+        modifiers = av_malloc(mod_props_list.drmFormatModifierCount *
+                              sizeof(*modifiers));
+        if (!modifiers)
+            return AVERROR(ENOMEM);
+        modifier_info->pDrmFormatModifiers = modifiers;
+
+        mod_props = av_malloc(mod_props_list.drmFormatModifierCount *
+                              sizeof(*mod_props));
+        if (!mod_props)
+            return AVERROR(ENOMEM);
+        mod_props_list.pDrmFormatModifierProperties = mod_props;
+        vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt[0], &prop);
+        for (int i = 0; i < mod_props_list.drmFormatModifierCount; i++) {
+            if (!(mod_props[i].drmFormatModifierTilingFeatures & FF_VK_DEFAULT_USAGE_FLAGS))
+                continue;
+            modifiers[modifier_count++] = mod_props[i].drmFormatModifier;
+        }
+        if (!modifier_count) {
+            av_log(hwfc, AV_LOG_ERROR, "The supported modifiers doesn't support "
+                                       "default usage\n");
+            av_freep(&mod_props);
+            return AVERROR(EINVAL);
+        }
+        modifier_info->drmFormatModifierCount = modifier_count;
+        av_freep(&mod_props);
+    }
+
+
     if (!hwctx->usage)
         hwctx->usage = FF_VK_DEFAULT_USAGE_FLAGS;