diff mbox series

[FFmpeg-devel,V6,5/5] libavutil/hwcontext_vulkan: specify the modifier to create VKImage

Message ID 20211207090553.2138848-5-wenbin.chen@intel.com
State New
Headers show
Series [FFmpeg-devel,V6,1/5] hwcontext_vaapi: Use PRIME_2 memory type for modifiers. | expand

Checks

Context Check Description
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished
andriy/make_ppc success Make finished
andriy/make_fate_ppc success Make fate finished

Commit Message

Wenbin Chen Dec. 7, 2021, 9:05 a.m. UTC 
When vulkan image exports to drm, the tilling need to be
VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT. Now add code to create vulkan
image using this format.

Now the following command line works:

ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 -hwaccel_output_format \
vaapi -i input_1080p.264 -vf "hwmap=derive_device=vulkan,format=vulkan, \
scale_vulkan=1920:1080,hwmap=derive_device=vaapi,format=vaapi" -c:v h264_vaapi output.264

Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
---
 libavutil/hwcontext_vulkan.c | 133 +++++++++++++++++++++++++++++++++--
 1 file changed, 127 insertions(+), 6 deletions(-)

Comments

Lynne Dec. 10, 2021, 4:11 p.m. UTC | #1 
7 Dec 2021, 10:05 by wenbin.chen@intel.com:

> When vulkan image exports to drm, the tilling need to be
> VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT. Now add code to create vulkan
> image using this format.
>
> Now the following command line works:
>
> ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 -hwaccel_output_format \
> vaapi -i input_1080p.264 -vf "hwmap=derive_device=vulkan,format=vulkan, \
> scale_vulkan=1920:1080,hwmap=derive_device=vaapi,format=vaapi" -c:v h264_vaapi output.264
>
> Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
> ---
>  libavutil/hwcontext_vulkan.c | 133 +++++++++++++++++++++++++++++++++--
>  1 file changed, 127 insertions(+), 6 deletions(-)
>

Pushed the patchset, along with the transpose filter.
Tested on all 3 vendors, works fine. AMD requires
dedicated allocation, for which I added an error condition.

The code was quite ugly in quite a lot of places,
so I did touch mostly every line of it in some way.

Please fix your quirky, limited hardware next time.
Wenbin Chen Dec. 13, 2021, 8:54 a.m. UTC | #2 
> 7 Dec 2021, 10:05 by wenbin.chen@intel.com:
> 
> > When vulkan image exports to drm, the tilling need to be
> > VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT. Now add code to
> create vulkan
> > image using this format.
> >
> > Now the following command line works:
> >
> > ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 -
> hwaccel_output_format \
> > vaapi -i input_1080p.264 -vf "hwmap=derive_device=vulkan,format=vulkan,
> \
> > scale_vulkan=1920:1080,hwmap=derive_device=vaapi,format=vaapi" -c:v
> h264_vaapi output.264
> >
> > Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
> > ---
> >  libavutil/hwcontext_vulkan.c | 133
> +++++++++++++++++++++++++++++++++--
> >  1 file changed, 127 insertions(+), 6 deletions(-)
> >
> 
> Pushed the patchset, along with the transpose filter.
> Tested on all 3 vendors, works fine. AMD requires
> dedicated allocation, for which I added an error condition.
> 
> The code was quite ugly in quite a lot of places,
> so I did touch mostly every line of it in some way.
> 
> Please fix your quirky, limited hardware next time.

Thanks for your review and feedback.
This limitation of hardware also bothered me for some time.

Best Regards
Wenbin
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
diff mbox series

Patch

diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
index f980b72720..11c4260e86 100644
--- a/libavutil/hwcontext_vulkan.c
+++ b/libavutil/hwcontext_vulkan.c
@@ -120,6 +120,9 @@  typedef struct VulkanFramesPriv {
     /* Image transfers */
     VulkanExecCtx upload_ctx;
     VulkanExecCtx download_ctx;
+
+    /*modifier info*/
+    VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
 } VulkanFramesPriv;
 
 typedef struct AVVkFrameInternal {
@@ -242,6 +245,31 @@  const VkFormat *av_vkfmt_from_pixfmt(enum AVPixelFormat p)
     return NULL;
 }
 
+static const void *vk_find_struct(const void *chain, VkStructureType stype)
+{
+    const VkBaseInStructure *in = chain;
+    while (in) {
+        if (in->sType == stype)
+            return in;
+
+        in = in->pNext;
+    }
+
+    return NULL;
+}
+
+static void vk_link_struct(void *chain, void *in)
+{
+    VkBaseOutStructure *out = chain;
+    if (!in)
+        return;
+
+    while (out->pNext)
+        out = out->pNext;
+
+    out->pNext = in;
+}
+
 static int pixfmt_is_supported(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p,
                                int linear)
 {
@@ -2094,6 +2122,10 @@  static void try_export_flags(AVHWFramesContext *hwfc,
     AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
     VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
     FFVulkanFunctions *vk = &p->vkfn;
+    const int has_modifiers = hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
+    int loop_count;
+    const VkImageDrmFormatModifierListCreateInfoEXT *modifier_info = vk_find_struct(hwctx->create_pnext,
+                                        VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
     VkExternalImageFormatProperties eprops = {
         .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
     };
@@ -2101,9 +2133,18 @@  static void try_export_flags(AVHWFramesContext *hwfc,
         .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
         .pNext = &eprops,
     };
+    VkPhysicalDeviceImageDrmFormatModifierInfoEXT phy_dev_mod_info = {
+        .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
+        .pNext = NULL,
+        .pQueueFamilyIndices   = p->qfs,
+        .queueFamilyIndexCount = p->num_qfs,
+        .sharingMode           = p->num_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
+                                                  VK_SHARING_MODE_EXCLUSIVE,
+    };
     VkPhysicalDeviceExternalImageFormatInfo enext = {
         .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
         .handleType = exp,
+        .pNext = has_modifiers && modifier_info ? &phy_dev_mod_info : NULL,
     };
     VkPhysicalDeviceImageFormatInfo2 pinfo = {
         .sType  = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
@@ -2115,11 +2156,16 @@  static void try_export_flags(AVHWFramesContext *hwfc,
         .flags  = VK_IMAGE_CREATE_ALIAS_BIT,
     };
 
-    ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
-                                                      &pinfo, &props);
-    if (ret == VK_SUCCESS) {
-        *iexp |= exp;
-        *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
+    loop_count = has_modifiers && modifier_info ? modifier_info->drmFormatModifierCount : 1;
+    for (int i = 0; i < loop_count; i++) {
+        if (has_modifiers && modifier_info)
+            phy_dev_mod_info.drmFormatModifier = modifier_info->pDrmFormatModifiers[i];
+        ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
+                                                        &pinfo, &props);
+        if (ret == VK_SUCCESS) {
+            *iexp |= exp;
+            *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
+        }
     }
 }
 
@@ -2190,6 +2236,12 @@  static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
 {
     VulkanFramesPriv *fp = hwfc->internal->priv;
 
+    if (fp->modifier_info) {
+        if (fp->modifier_info->pDrmFormatModifiers)
+            av_freep(&fp->modifier_info->pDrmFormatModifiers);
+        av_freep(&fp->modifier_info);
+    }
+
     free_exec_ctx(hwfc, &fp->conv_ctx);
     free_exec_ctx(hwfc, &fp->upload_ctx);
     free_exec_ctx(hwfc, &fp->download_ctx);
@@ -2203,9 +2255,12 @@  static int vulkan_frames_init(AVHWFramesContext *hwfc)
     VulkanFramesPriv *fp = hwfc->internal->priv;
     AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
     VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
+    const VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
+    const int has_modifiers = !!(p->extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS);
 
     /* Default pool flags */
-    hwctx->tiling = hwctx->tiling ? hwctx->tiling : p->use_linear_images ?
+    hwctx->tiling = hwctx->tiling ? hwctx->tiling : has_modifiers ?
+                    VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT : p->use_linear_images ?
                     VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL;
 
     if (!hwctx->usage)
@@ -2217,6 +2272,72 @@  static int vulkan_frames_init(AVHWFramesContext *hwfc)
            hwctx->flags |= AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY;
     }
 
+    modifier_info = vk_find_struct(hwctx->create_pnext,
+                                   VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
+    /* get the supported modifier */
+    if (has_modifiers && !modifier_info) {
+        const VkFormat *fmt = av_vkfmt_from_pixfmt(hwfc->sw_format);
+        VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
+        FFVulkanFunctions *vk = &p->vkfn;
+        VkDrmFormatModifierPropertiesEXT *mod_props;
+        uint64_t *modifiers;
+        int modifier_count = 0;
+
+        VkDrmFormatModifierPropertiesListEXT mod_props_list = {
+            .sType = VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT,
+            .pNext = NULL,
+            .drmFormatModifierCount = 0,
+            .pDrmFormatModifierProperties = NULL,
+        };
+        VkFormatProperties2 prop = {
+            .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
+            .pNext = &mod_props_list,
+        };
+        vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt[0], &prop);
+        if (!mod_props_list.drmFormatModifierCount) {
+            av_log(hwfc, AV_LOG_ERROR, "There are no supported modifiers for given sw_format\n");
+            return AVERROR(EINVAL);
+        }
+
+        modifier_info = av_malloc(sizeof(*modifier_info));
+        if (!modifier_info)
+            return AVERROR(ENOMEM);
+
+        if (!hwctx->create_pnext)
+            hwctx->create_pnext = modifier_info;
+        else
+            vk_link_struct(hwctx->create_pnext, (void *)modifier_info);
+        fp->modifier_info = modifier_info;
+        modifier_info->pNext = NULL;
+        modifier_info->sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT;
+
+        modifiers = av_malloc(mod_props_list.drmFormatModifierCount *
+                              sizeof(*modifiers));
+        if (!modifiers)
+            return AVERROR(ENOMEM);
+        modifier_info->pDrmFormatModifiers = modifiers;
+
+        mod_props = av_malloc(mod_props_list.drmFormatModifierCount *
+                              sizeof(*mod_props));
+        if (!mod_props)
+            return AVERROR(ENOMEM);
+        mod_props_list.pDrmFormatModifierProperties = mod_props;
+        vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt[0], &prop);
+        for (int i = 0; i < mod_props_list.drmFormatModifierCount; i++) {
+            if (!(mod_props[i].drmFormatModifierTilingFeatures & hwctx->usage))
+                continue;
+            modifiers[modifier_count++] = mod_props[i].drmFormatModifier;
+        }
+        if (!modifier_count) {
+            av_log(hwfc, AV_LOG_ERROR, "None of the given modifiers supports"
+                                       " the usage flags!\n");
+            av_freep(&mod_props);
+            return AVERROR(EINVAL);
+        }
+        modifier_info->drmFormatModifierCount = modifier_count;
+        av_freep(&mod_props);
+    }
+
     err = create_exec_ctx(hwfc, &fp->conv_ctx,
                           dev_hwctx->queue_family_comp_index,
                           dev_hwctx->nb_comp_queues);