| Message ID | 20211202014320.1066212-3-wenbin.chen@intel.com |
|---|---|
| State | New |
| Headers | show |
| Series | [FFmpeg-devel,V5,1/5] hwcontext_vaapi: Use PRIME_2 memory type for modifiers. | expand |
| Context | Check | Description |
|---|---|---|
| andriy/make_x86 | success | Make finished |
| andriy/make_fate_x86 | fail | Make fate failed |
| andriy/make_ppc | success | Make finished |
| andriy/make_fate_ppc | fail | Make fate failed |
> The vaapi can import external frame, but the planes of the external > frames should be in the same drm object. A new option "contiguous_planes" > is added to device. This flag tells device to allocate places in one > memory. When device is derived from vaapi this flag will be enabled. > A new flag frame_flag is also added to AVVulkanFramesContext. User > can use this flag to force enable or disable this behaviour. > A new variable "offset "is added to AVVKFrame. It describe describe the > offset from the memory currently bound to the VkImage. > > Signed-off-by: Wenbin Chen <wenbin.chen@intel.com> > --- > libavutil/hwcontext_vulkan.c | 68 > +++++++++++++++++++++++++++++++++++- > libavutil/hwcontext_vulkan.h | 24 +++++++++++++ > 2 files changed, 91 insertions(+), 1 deletion(-) > > diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c > index a0437c9661..eef9009ae1 100644 > --- a/libavutil/hwcontext_vulkan.c > +++ b/libavutil/hwcontext_vulkan.c > @@ -103,8 +103,14 @@ typedef struct VulkanDevicePriv { > /* Settings */ > int use_linear_images; > > + /* allocate planes in a contiguous memory */ > + int contiguous_planes; > + > /* Nvidia */ > int dev_is_nvidia; > + > + /* Intel */ > + int dev_is_intel; > } VulkanDevicePriv; > > typedef struct VulkanFramesPriv { > @@ -153,6 +159,8 @@ typedef struct AVVkFrameInternal { > av_free((void *)props); \ > } > > +#define VKF_FLAG(x, f) (((x) & (~AV_VK_FRAME_FLAG_NONE)) & (f)) > + > static const struct { > enum AVPixelFormat pixfmt; > const VkFormat vkfmts[4]; > @@ -1374,6 +1382,13 @@ static int > vulkan_device_create_internal(AVHWDeviceContext *ctx, > if (opt_d) > p->use_linear_images = strtol(opt_d->value, NULL, 10); > > + opt_d = av_dict_get(opts, "contiguous_planes", NULL, 0); > + if (opt_d) > + p->contiguous_planes = strtol(opt_d->value, NULL, 10); > + else > + p->contiguous_planes = -1; > + > + > hwctx->enabled_dev_extensions = dev_info.ppEnabledExtensionNames; > hwctx->nb_enabled_dev_extensions = dev_info.enabledExtensionCount; > > @@ -1425,6 +1440,8 @@ static int vulkan_device_init(AVHWDeviceContext > *ctx) > > p->dev_is_nvidia = (p->props.properties.vendorID == 0x10de); > > + p->dev_is_intel = (p->props.properties.vendorID == 0x8086); > + > vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, > &queue_num, NULL); > if (!queue_num) { > av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n"); > @@ -1742,8 +1759,12 @@ static int alloc_bind_mem(AVHWFramesContext > *hwfc, AVVkFrame *f, > AVHWDeviceContext *ctx = hwfc->device_ctx; > VulkanDevicePriv *p = ctx->internal->priv; > FFVulkanFunctions *vk = &p->vkfn; > + AVVulkanFramesContext *hwfctx = hwfc->hwctx; > const int planes = av_pix_fmt_count_planes(hwfc->sw_format); > VkBindImageMemoryInfo bind_info[AV_NUM_DATA_POINTERS] = { { 0 } }; > + VkMemoryRequirements memory_requirements = { 0 }; > + int mem_size = 0; > + int mem_size_list[AV_NUM_DATA_POINTERS] = { 0 }; > > AVVulkanDeviceContext *hwctx = ctx->hwctx; > > @@ -1771,6 +1792,19 @@ static int alloc_bind_mem(AVHWFramesContext > *hwfc, AVVkFrame *f, > req.memoryRequirements.size = > FFALIGN(req.memoryRequirements.size, > p- > >props.properties.limits.minMemoryMapAlignment); > > + if (VKF_FLAG(hwfctx->flags, > AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY)) { > + if (memory_requirements.size == 0) { > + memory_requirements = req.memoryRequirements; > + } else if (memory_requirements.memoryTypeBits != > req.memoryRequirements.memoryTypeBits) { > + av_log(hwfc, AV_LOG_ERROR, "the param for each planes are not > the same\n"); > + return AVERROR(EINVAL); > + } > + > + mem_size_list[i] = req.memoryRequirements.size; > + mem_size += mem_size_list[i]; > + continue; > + } > + > /* In case the implementation prefers/requires dedicated allocation */ > use_ded_mem = ded_req.prefersDedicatedAllocation | > ded_req.requiresDedicatedAllocation; > @@ -1792,6 +1826,29 @@ static int alloc_bind_mem(AVHWFramesContext > *hwfc, AVVkFrame *f, > bind_info[i].memory = f->mem[i]; > } > > + if (VKF_FLAG(hwfctx->flags, > AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY)) { > + memory_requirements.size = mem_size; > + > + /* Allocate memory */ > + if ((err = alloc_mem(ctx, &memory_requirements, > + f->tiling == VK_IMAGE_TILING_LINEAR ? > + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT : > + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, > + (void *)(((uint8_t *)alloc_pnext)), > + &f->flags, &f->mem[0]))) > + return err; > + > + f->size[0] = memory_requirements.size; > + > + for (int i = 0; i < planes; i++) { > + bind_info[i].sType = > VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO; > + bind_info[i].image = f->img[i]; > + bind_info[i].memory = f->mem[0]; > + bind_info[i].memoryOffset = i == 0 ? 0 : mem_size_list[i-1]; > + f->offset[i] = bind_info[i].memoryOffset; > + } > + } > + > /* Bind the allocated memory to the images */ > ret = vk->BindImageMemory2(hwctx->act_dev, planes, bind_info); > if (ret != VK_SUCCESS) { > @@ -2154,6 +2211,12 @@ static int > vulkan_frames_init(AVHWFramesContext *hwfc) > if (!hwctx->usage) > hwctx->usage = FF_VK_DEFAULT_USAGE_FLAGS; > > + if (!(hwctx->flags & AV_VK_FRAME_FLAG_NONE)) { > + if (p->contiguous_planes == 1 || > + ((p->contiguous_planes == -1) && p->dev_is_intel)) > + hwctx->flags |= AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY; > + } > + > err = create_exec_ctx(hwfc, &fp->conv_ctx, > dev_hwctx->queue_family_comp_index, > dev_hwctx->nb_comp_queues); > @@ -3074,6 +3137,7 @@ static int > vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst, > FFVulkanFunctions *vk = &p->vkfn; > VulkanFramesPriv *fp = hwfc->internal->priv; > AVVulkanDeviceContext *hwctx = hwfc->device_ctx->hwctx; > + AVVulkanFramesContext *hwfctx = hwfc->hwctx; > const int planes = av_pix_fmt_count_planes(hwfc->sw_format); > VkImageDrmFormatModifierPropertiesEXT drm_mod = { > .sType = > VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT, > @@ -3142,7 +3206,9 @@ static int > vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst, > continue; > > vk->GetImageSubresourceLayout(hwctx->act_dev, f->img[i], &sub, > &layout); > - drm_desc->layers[i].planes[0].offset = layout.offset; > + drm_desc->layers[i].planes[0].offset = layout.offset + > + VKF_FLAG(hwfctx->flags, > AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY) ? > + f->offset[i] : 0; > drm_desc->layers[i].planes[0].pitch = layout.rowPitch; > } > > diff --git a/libavutil/hwcontext_vulkan.h b/libavutil/hwcontext_vulkan.h > index fdf2a60156..acb4073e3d 100644 > --- a/libavutil/hwcontext_vulkan.h > +++ b/libavutil/hwcontext_vulkan.h > @@ -35,6 +35,17 @@ > * with the data pointer set to an AVVkFrame. > */ > > +/** > + * Defines the behaviour of frame allocation > + * AV_VK_FRAME_FLAG_NONE: planes will be allocated in separte memory > + * AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY: planes will be allocated in > a > + * contiguous memory. > + */ > +typedef enum { > + AV_VK_FRAME_FLAG_NONE = (1ULL << 0), > + AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY = (1ULL << 1) | 1ULL > +} AVVkFrameFlags; > + > /** > * Main Vulkan context, allocated as AVHWDeviceContext.hwctx. > * All of these can be set before init to change what the context uses > @@ -165,6 +176,14 @@ typedef struct AVVulkanFramesContext { > * extensions are present in enabled_dev_extensions. > */ > void *alloc_pnext[AV_NUM_DATA_POINTERS]; > + > + /** > + * Is a combination of AVVkFrameFlags. Defines the behaviour of frame > + * allocation. > + * If no flag is set, then the flags are automatically determined > + * based on the device. > + */ > + int flags; > } AVVulkanFramesContext; > > /* > @@ -230,6 +249,11 @@ typedef struct AVVkFrame { > * Internal data. > */ > struct AVVkFrameInternal *internal; > + > + /** > + * Describe the offset from the memory currently bound to the VkImage. > + */ > + size_t offset[AV_NUM_DATA_POINTERS]; > } AVVkFrame; > > /** > -- > 2.25.1 Hi Lynn: (I move my comment from patch V4 to here, in case you miss it) I made a mistake. I should use "layout.offset + f->offset[i]". I resubmit the patchset. Let's discuss under this patch. There are two offsets. The first offset relates to the base address of the memory. This offset is the memoryOffset we pass to vulkan when we bind memory. The second offset relates to the base address of the image. This offset is set by vulkan when we create vkImage. The offset we get from vkGetImageSubresourceLayout is the second offset. The offset drm_object need is the first offset. If one image is bind to one memory, these two offsets should be the same, because the start of the image is the start of the memory. If we bind multiple image to one memory, these two offsets are different. I don't find an API to get the second kind of offset, so I create a new variable to store them. In practice, these two offsets are different. When I allocate planes to one memory. The offsets I get from vkGetImageSubresourceLayout are all 0. However, I need to set it to 2088960 to make command works. 2088960 is the memoryOffset value I pass to vulkan when I bind memory. Regards Wenbin > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
2 Dec 2021, 02:53 by wenbin.chen@intel.com: >> The vaapi can import external frame, but the planes of the external >> frames should be in the same drm object. A new option "contiguous_planes" >> is added to device. This flag tells device to allocate places in one >> memory. When device is derived from vaapi this flag will be enabled. >> A new flag frame_flag is also added to AVVulkanFramesContext. User >> can use this flag to force enable or disable this behaviour. >> A new variable "offset "is added to AVVKFrame. It describe describe the >> offset from the memory currently bound to the VkImage. >> >> Signed-off-by: Wenbin Chen <wenbin.chen@intel.com> >> --- >> libavutil/hwcontext_vulkan.c | 68 >> +++++++++++++++++++++++++++++++++++- >> libavutil/hwcontext_vulkan.h | 24 +++++++++++++ >> 2 files changed, 91 insertions(+), 1 deletion(-) >> >> diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c >> index a0437c9661..eef9009ae1 100644 >> --- a/libavutil/hwcontext_vulkan.c >> +++ b/libavutil/hwcontext_vulkan.c >> @@ -103,8 +103,14 @@ typedef struct VulkanDevicePriv { >> /* Settings */ >> int use_linear_images; >> >> + /* allocate planes in a contiguous memory */ >> + int contiguous_planes; >> + >> /* Nvidia */ >> int dev_is_nvidia; >> + >> + /* Intel */ >> + int dev_is_intel; >> } VulkanDevicePriv; >> >> typedef struct VulkanFramesPriv { >> @@ -153,6 +159,8 @@ typedef struct AVVkFrameInternal { >> av_free((void *)props); \ >> } >> >> +#define VKF_FLAG(x, f) (((x) & (~AV_VK_FRAME_FLAG_NONE)) & (f)) >> + >> static const struct { >> enum AVPixelFormat pixfmt; >> const VkFormat vkfmts[4]; >> @@ -1374,6 +1382,13 @@ static int >> vulkan_device_create_internal(AVHWDeviceContext *ctx, >> if (opt_d) >> p->use_linear_images = strtol(opt_d->value, NULL, 10); >> >> + opt_d = av_dict_get(opts, "contiguous_planes", NULL, 0); >> + if (opt_d) >> + p->contiguous_planes = strtol(opt_d->value, NULL, 10); >> + else >> + p->contiguous_planes = -1; >> + >> + >> hwctx->enabled_dev_extensions = dev_info.ppEnabledExtensionNames; >> hwctx->nb_enabled_dev_extensions = dev_info.enabledExtensionCount; >> >> @@ -1425,6 +1440,8 @@ static int vulkan_device_init(AVHWDeviceContext >> *ctx) >> >> p->dev_is_nvidia = (p->props.properties.vendorID == 0x10de); >> >> + p->dev_is_intel = (p->props.properties.vendorID == 0x8086); >> + >> vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, >> &queue_num, NULL); >> if (!queue_num) { >> av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n"); >> @@ -1742,8 +1759,12 @@ static int alloc_bind_mem(AVHWFramesContext >> *hwfc, AVVkFrame *f, >> AVHWDeviceContext *ctx = hwfc->device_ctx; >> VulkanDevicePriv *p = ctx->internal->priv; >> FFVulkanFunctions *vk = &p->vkfn; >> + AVVulkanFramesContext *hwfctx = hwfc->hwctx; >> const int planes = av_pix_fmt_count_planes(hwfc->sw_format); >> VkBindImageMemoryInfo bind_info[AV_NUM_DATA_POINTERS] = { { 0 } }; >> + VkMemoryRequirements memory_requirements = { 0 }; >> + int mem_size = 0; >> + int mem_size_list[AV_NUM_DATA_POINTERS] = { 0 }; >> >> AVVulkanDeviceContext *hwctx = ctx->hwctx; >> >> @@ -1771,6 +1792,19 @@ static int alloc_bind_mem(AVHWFramesContext >> *hwfc, AVVkFrame *f, >> req.memoryRequirements.size = >> FFALIGN(req.memoryRequirements.size, >> p- >> >props.properties.limits.minMemoryMapAlignment); >> >> + if (VKF_FLAG(hwfctx->flags, >> AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY)) { >> + if (memory_requirements.size == 0) { >> + memory_requirements = req.memoryRequirements; >> + } else if (memory_requirements.memoryTypeBits != >> req.memoryRequirements.memoryTypeBits) { >> + av_log(hwfc, AV_LOG_ERROR, "the param for each planes are not >> the same\n"); >> + return AVERROR(EINVAL); >> + } >> + >> + mem_size_list[i] = req.memoryRequirements.size; >> + mem_size += mem_size_list[i]; >> + continue; >> + } >> + >> /* In case the implementation prefers/requires dedicated allocation */ >> use_ded_mem = ded_req.prefersDedicatedAllocation | >> ded_req.requiresDedicatedAllocation; >> @@ -1792,6 +1826,29 @@ static int alloc_bind_mem(AVHWFramesContext >> *hwfc, AVVkFrame *f, >> bind_info[i].memory = f->mem[i]; >> } >> >> + if (VKF_FLAG(hwfctx->flags, >> AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY)) { >> + memory_requirements.size = mem_size; >> + >> + /* Allocate memory */ >> + if ((err = alloc_mem(ctx, &memory_requirements, >> + f->tiling == VK_IMAGE_TILING_LINEAR ? >> + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT : >> + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, >> + (void *)(((uint8_t *)alloc_pnext)), >> + &f->flags, &f->mem[0]))) >> + return err; >> + >> + f->size[0] = memory_requirements.size; >> + >> + for (int i = 0; i < planes; i++) { >> + bind_info[i].sType = >> VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO; >> + bind_info[i].image = f->img[i]; >> + bind_info[i].memory = f->mem[0]; >> + bind_info[i].memoryOffset = i == 0 ? 0 : mem_size_list[i-1]; >> + f->offset[i] = bind_info[i].memoryOffset; >> + } >> + } >> + >> /* Bind the allocated memory to the images */ >> ret = vk->BindImageMemory2(hwctx->act_dev, planes, bind_info); >> if (ret != VK_SUCCESS) { >> @@ -2154,6 +2211,12 @@ static int >> vulkan_frames_init(AVHWFramesContext *hwfc) >> if (!hwctx->usage) >> hwctx->usage = FF_VK_DEFAULT_USAGE_FLAGS; >> >> + if (!(hwctx->flags & AV_VK_FRAME_FLAG_NONE)) { >> + if (p->contiguous_planes == 1 || >> + ((p->contiguous_planes == -1) && p->dev_is_intel)) >> + hwctx->flags |= AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY; >> + } >> + >> err = create_exec_ctx(hwfc, &fp->conv_ctx, >> dev_hwctx->queue_family_comp_index, >> dev_hwctx->nb_comp_queues); >> @@ -3074,6 +3137,7 @@ static int >> vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst, >> FFVulkanFunctions *vk = &p->vkfn; >> VulkanFramesPriv *fp = hwfc->internal->priv; >> AVVulkanDeviceContext *hwctx = hwfc->device_ctx->hwctx; >> + AVVulkanFramesContext *hwfctx = hwfc->hwctx; >> const int planes = av_pix_fmt_count_planes(hwfc->sw_format); >> VkImageDrmFormatModifierPropertiesEXT drm_mod = { >> .sType = >> VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT, >> @@ -3142,7 +3206,9 @@ static int >> vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst, >> continue; >> >> vk->GetImageSubresourceLayout(hwctx->act_dev, f->img[i], &sub, >> &layout); >> - drm_desc->layers[i].planes[0].offset = layout.offset; >> + drm_desc->layers[i].planes[0].offset = layout.offset + >> + VKF_FLAG(hwfctx->flags, >> AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY) ? >> + f->offset[i] : 0; >> drm_desc->layers[i].planes[0].pitch = layout.rowPitch; >> } >> >> diff --git a/libavutil/hwcontext_vulkan.h b/libavutil/hwcontext_vulkan.h >> index fdf2a60156..acb4073e3d 100644 >> --- a/libavutil/hwcontext_vulkan.h >> +++ b/libavutil/hwcontext_vulkan.h >> @@ -35,6 +35,17 @@ >> * with the data pointer set to an AVVkFrame. >> */ >> >> +/** >> + * Defines the behaviour of frame allocation >> + * AV_VK_FRAME_FLAG_NONE: planes will be allocated in separte memory >> + * AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY: planes will be allocated in >> a >> + * contiguous memory. >> + */ >> +typedef enum { >> + AV_VK_FRAME_FLAG_NONE = (1ULL << 0), >> + AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY = (1ULL << 1) | 1ULL >> +} AVVkFrameFlags; >> + >> /** >> * Main Vulkan context, allocated as AVHWDeviceContext.hwctx. >> * All of these can be set before init to change what the context uses >> @@ -165,6 +176,14 @@ typedef struct AVVulkanFramesContext { >> * extensions are present in enabled_dev_extensions. >> */ >> void *alloc_pnext[AV_NUM_DATA_POINTERS]; >> + >> + /** >> + * Is a combination of AVVkFrameFlags. Defines the behaviour of frame >> + * allocation. >> + * If no flag is set, then the flags are automatically determined >> + * based on the device. >> + */ >> + int flags; >> } AVVulkanFramesContext; >> >> /* >> @@ -230,6 +249,11 @@ typedef struct AVVkFrame { >> * Internal data. >> */ >> struct AVVkFrameInternal *internal; >> + >> + /** >> + * Describe the offset from the memory currently bound to the VkImage. >> + */ >> + size_t offset[AV_NUM_DATA_POINTERS]; >> } AVVkFrame; >> >> /** >> -- >> 2.25.1 >> > > Hi Lynn: > (I move my comment from patch V4 to here, in case you miss it) > I made a mistake. I should use "layout.offset + f->offset[i]". I resubmit > the patchset. Let's discuss under this patch. > > There are two offsets. > The first offset relates to the base address of the memory. This offset is the > memoryOffset we pass to vulkan when we bind memory. > The second offset relates to the base address of the image. This offset is set by > vulkan when we create vkImage. > > The offset we get from vkGetImageSubresourceLayout is the second offset. > The offset drm_object need is the first offset. > If one image is bind to one memory, these two offsets should be the same, because the > start of the image is the start of the memory. If we bind multiple image to one memory, > these two offsets are different. > I don't find an API to get the second kind of offset, so I create a new variable to store them. > > In practice, these two offsets are different. When I allocate planes to one memory. The > offsets I get from vkGetImageSubresourceLayout are all 0. However, I need to set it to 2088960 > to make command works. 2088960 is the memoryOffset value I pass to vulkan when I bind > memory. > My question still stands: why can't the driver output the binding offset for DRM-tiled images? The spec explicitly allows for it, and RADV already outputs non-zero offset via vkGetImageSubresourceLayout. I'd rather not commit any driver-specific quirks because 2 internal teams don't communicate with one another.
> 2 Dec 2021, 02:53 by wenbin.chen@intel.com: > > >> The vaapi can import external frame, but the planes of the external > >> frames should be in the same drm object. A new option > "contiguous_planes" > >> is added to device. This flag tells device to allocate places in one > >> memory. When device is derived from vaapi this flag will be enabled. > >> A new flag frame_flag is also added to AVVulkanFramesContext. User > >> can use this flag to force enable or disable this behaviour. > >> A new variable "offset "is added to AVVKFrame. It describe describe the > >> offset from the memory currently bound to the VkImage. > >> > >> Signed-off-by: Wenbin Chen <wenbin.chen@intel.com> > >> --- > >> libavutil/hwcontext_vulkan.c | 68 > >> +++++++++++++++++++++++++++++++++++- > >> libavutil/hwcontext_vulkan.h | 24 +++++++++++++ > >> 2 files changed, 91 insertions(+), 1 deletion(-) > >> > >> diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c > >> index a0437c9661..eef9009ae1 100644 > >> --- a/libavutil/hwcontext_vulkan.c > >> +++ b/libavutil/hwcontext_vulkan.c > >> @@ -103,8 +103,14 @@ typedef struct VulkanDevicePriv { > >> /* Settings */ > >> int use_linear_images; > >> > >> + /* allocate planes in a contiguous memory */ > >> + int contiguous_planes; > >> + > >> /* Nvidia */ > >> int dev_is_nvidia; > >> + > >> + /* Intel */ > >> + int dev_is_intel; > >> } VulkanDevicePriv; > >> > >> typedef struct VulkanFramesPriv { > >> @@ -153,6 +159,8 @@ typedef struct AVVkFrameInternal { > >> av_free((void *)props); \ > >> } > >> > >> +#define VKF_FLAG(x, f) (((x) & (~AV_VK_FRAME_FLAG_NONE)) & (f)) > >> + > >> static const struct { > >> enum AVPixelFormat pixfmt; > >> const VkFormat vkfmts[4]; > >> @@ -1374,6 +1382,13 @@ static int > >> vulkan_device_create_internal(AVHWDeviceContext *ctx, > >> if (opt_d) > >> p->use_linear_images = strtol(opt_d->value, NULL, 10); > >> > >> + opt_d = av_dict_get(opts, "contiguous_planes", NULL, 0); > >> + if (opt_d) > >> + p->contiguous_planes = strtol(opt_d->value, NULL, 10); > >> + else > >> + p->contiguous_planes = -1; > >> + > >> + > >> hwctx->enabled_dev_extensions = dev_info.ppEnabledExtensionNames; > >> hwctx->nb_enabled_dev_extensions = dev_info.enabledExtensionCount; > >> > >> @@ -1425,6 +1440,8 @@ static int > vulkan_device_init(AVHWDeviceContext > >> *ctx) > >> > >> p->dev_is_nvidia = (p->props.properties.vendorID == 0x10de); > >> > >> + p->dev_is_intel = (p->props.properties.vendorID == 0x8086); > >> + > >> vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, > >> &queue_num, NULL); > >> if (!queue_num) { > >> av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n"); > >> @@ -1742,8 +1759,12 @@ static int > alloc_bind_mem(AVHWFramesContext > >> *hwfc, AVVkFrame *f, > >> AVHWDeviceContext *ctx = hwfc->device_ctx; > >> VulkanDevicePriv *p = ctx->internal->priv; > >> FFVulkanFunctions *vk = &p->vkfn; > >> + AVVulkanFramesContext *hwfctx = hwfc->hwctx; > >> const int planes = av_pix_fmt_count_planes(hwfc->sw_format); > >> VkBindImageMemoryInfo bind_info[AV_NUM_DATA_POINTERS] = { { 0 } }; > >> + VkMemoryRequirements memory_requirements = { 0 }; > >> + int mem_size = 0; > >> + int mem_size_list[AV_NUM_DATA_POINTERS] = { 0 }; > >> > >> AVVulkanDeviceContext *hwctx = ctx->hwctx; > >> > >> @@ -1771,6 +1792,19 @@ static int > alloc_bind_mem(AVHWFramesContext > >> *hwfc, AVVkFrame *f, > >> req.memoryRequirements.size = > >> FFALIGN(req.memoryRequirements.size, > >> p- > >> >props.properties.limits.minMemoryMapAlignment); > >> > >> + if (VKF_FLAG(hwfctx->flags, > >> AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY)) { > >> + if (memory_requirements.size == 0) { > >> + memory_requirements = req.memoryRequirements; > >> + } else if (memory_requirements.memoryTypeBits != > >> req.memoryRequirements.memoryTypeBits) { > >> + av_log(hwfc, AV_LOG_ERROR, "the param for each planes are > not > >> the same\n"); > >> + return AVERROR(EINVAL); > >> + } > >> + > >> + mem_size_list[i] = req.memoryRequirements.size; > >> + mem_size += mem_size_list[i]; > >> + continue; > >> + } > >> + > >> /* In case the implementation prefers/requires dedicated allocation */ > >> use_ded_mem = ded_req.prefersDedicatedAllocation | > >> ded_req.requiresDedicatedAllocation; > >> @@ -1792,6 +1826,29 @@ static int > alloc_bind_mem(AVHWFramesContext > >> *hwfc, AVVkFrame *f, > >> bind_info[i].memory = f->mem[i]; > >> } > >> > >> + if (VKF_FLAG(hwfctx->flags, > >> AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY)) { > >> + memory_requirements.size = mem_size; > >> + > >> + /* Allocate memory */ > >> + if ((err = alloc_mem(ctx, &memory_requirements, > >> + f->tiling == VK_IMAGE_TILING_LINEAR ? > >> + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT : > >> + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, > >> + (void *)(((uint8_t *)alloc_pnext)), > >> + &f->flags, &f->mem[0]))) > >> + return err; > >> + > >> + f->size[0] = memory_requirements.size; > >> + > >> + for (int i = 0; i < planes; i++) { > >> + bind_info[i].sType = > >> VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO; > >> + bind_info[i].image = f->img[i]; > >> + bind_info[i].memory = f->mem[0]; > >> + bind_info[i].memoryOffset = i == 0 ? 0 : mem_size_list[i-1]; > >> + f->offset[i] = bind_info[i].memoryOffset; > >> + } > >> + } > >> + > >> /* Bind the allocated memory to the images */ > >> ret = vk->BindImageMemory2(hwctx->act_dev, planes, bind_info); > >> if (ret != VK_SUCCESS) { > >> @@ -2154,6 +2211,12 @@ static int > >> vulkan_frames_init(AVHWFramesContext *hwfc) > >> if (!hwctx->usage) > >> hwctx->usage = FF_VK_DEFAULT_USAGE_FLAGS; > >> > >> + if (!(hwctx->flags & AV_VK_FRAME_FLAG_NONE)) { > >> + if (p->contiguous_planes == 1 || > >> + ((p->contiguous_planes == -1) && p->dev_is_intel)) > >> + hwctx->flags |= AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY; > >> + } > >> + > >> err = create_exec_ctx(hwfc, &fp->conv_ctx, > >> dev_hwctx->queue_family_comp_index, > >> dev_hwctx->nb_comp_queues); > >> @@ -3074,6 +3137,7 @@ static int > >> vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst, > >> FFVulkanFunctions *vk = &p->vkfn; > >> VulkanFramesPriv *fp = hwfc->internal->priv; > >> AVVulkanDeviceContext *hwctx = hwfc->device_ctx->hwctx; > >> + AVVulkanFramesContext *hwfctx = hwfc->hwctx; > >> const int planes = av_pix_fmt_count_planes(hwfc->sw_format); > >> VkImageDrmFormatModifierPropertiesEXT drm_mod = { > >> .sType = > >> > VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT, > >> @@ -3142,7 +3206,9 @@ static int > >> vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst, > >> continue; > >> > >> vk->GetImageSubresourceLayout(hwctx->act_dev, f->img[i], &sub, > >> &layout); > >> - drm_desc->layers[i].planes[0].offset = layout.offset; > >> + drm_desc->layers[i].planes[0].offset = layout.offset + > >> + VKF_FLAG(hwfctx->flags, > >> AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY) ? > >> + f->offset[i] : 0; > >> drm_desc->layers[i].planes[0].pitch = layout.rowPitch; > >> } > >> > >> diff --git a/libavutil/hwcontext_vulkan.h b/libavutil/hwcontext_vulkan.h > >> index fdf2a60156..acb4073e3d 100644 > >> --- a/libavutil/hwcontext_vulkan.h > >> +++ b/libavutil/hwcontext_vulkan.h > >> @@ -35,6 +35,17 @@ > >> * with the data pointer set to an AVVkFrame. > >> */ > >> > >> +/** > >> + * Defines the behaviour of frame allocation > >> + * AV_VK_FRAME_FLAG_NONE: planes will be allocated in separte > memory > >> + * AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY: planes will be > allocated in > >> a > >> + * contiguous memory. > >> + */ > >> +typedef enum { > >> + AV_VK_FRAME_FLAG_NONE = (1ULL << 0), > >> + AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY = (1ULL << 1) | 1ULL > >> +} AVVkFrameFlags; > >> + > >> /** > >> * Main Vulkan context, allocated as AVHWDeviceContext.hwctx. > >> * All of these can be set before init to change what the context uses > >> @@ -165,6 +176,14 @@ typedef struct AVVulkanFramesContext { > >> * extensions are present in enabled_dev_extensions. > >> */ > >> void *alloc_pnext[AV_NUM_DATA_POINTERS]; > >> + > >> + /** > >> + * Is a combination of AVVkFrameFlags. Defines the behaviour of > frame > >> + * allocation. > >> + * If no flag is set, then the flags are automatically determined > >> + * based on the device. > >> + */ > >> + int flags; > >> } AVVulkanFramesContext; > >> > >> /* > >> @@ -230,6 +249,11 @@ typedef struct AVVkFrame { > >> * Internal data. > >> */ > >> struct AVVkFrameInternal *internal; > >> + > >> + /** > >> + * Describe the offset from the memory currently bound to the > VkImage. > >> + */ > >> + size_t offset[AV_NUM_DATA_POINTERS]; > >> } AVVkFrame; > >> > >> /** > >> -- > >> 2.25.1 > >> > > > > Hi Lynn: > > (I move my comment from patch V4 to here, in case you miss it) > > I made a mistake. I should use "layout.offset + f->offset[i]". I resubmit > > the patchset. Let's discuss under this patch. > > > > There are two offsets. > > The first offset relates to the base address of the memory. This offset is the > > memoryOffset we pass to vulkan when we bind memory. > > The second offset relates to the base address of the image. This offset is set > by > > vulkan when we create vkImage. > > > > The offset we get from vkGetImageSubresourceLayout is the second offset. > > The offset drm_object need is the first offset. > > If one image is bind to one memory, these two offsets should be the same, > because the > > start of the image is the start of the memory. If we bind multiple image to > one memory, > > these two offsets are different. > > I don't find an API to get the second kind of offset, so I create a new > variable to store them. > > > > In practice, these two offsets are different. When I allocate planes to one > memory. The > > offsets I get from vkGetImageSubresourceLayout are all 0. However, I need > to set it to 2088960 > > to make command works. 2088960 is the memoryOffset value I pass to > vulkan when I bind > > memory. > > > > My question still stands: why can't the driver output the binding offset > for DRM-tiled images? > The spec explicitly allows for it, and RADV already outputs non-zero offset > via vkGetImageSubresourceLayout. > I'd rather not commit any driver-specific quirks because 2 internal > teams don't communicate with one another. If so, the behavior of the this API on two platform are different. I submit an issue to MESA https://gitlab.freedesktop.org/mesa/mesa/-/issues/5708 If it is convenient for you, could you please provide some information under this issue? Thanks a lot. > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c index a0437c9661..eef9009ae1 100644 --- a/libavutil/hwcontext_vulkan.c +++ b/libavutil/hwcontext_vulkan.c @@ -103,8 +103,14 @@ typedef struct VulkanDevicePriv { /* Settings */ int use_linear_images; + /* allocate planes in a contiguous memory */ + int contiguous_planes; + /* Nvidia */ int dev_is_nvidia; + + /* Intel */ + int dev_is_intel; } VulkanDevicePriv; typedef struct VulkanFramesPriv { @@ -153,6 +159,8 @@ typedef struct AVVkFrameInternal { av_free((void *)props); \ } +#define VKF_FLAG(x, f) (((x) & (~AV_VK_FRAME_FLAG_NONE)) & (f)) + static const struct { enum AVPixelFormat pixfmt; const VkFormat vkfmts[4]; @@ -1374,6 +1382,13 @@ static int vulkan_device_create_internal(AVHWDeviceContext *ctx, if (opt_d) p->use_linear_images = strtol(opt_d->value, NULL, 10); + opt_d = av_dict_get(opts, "contiguous_planes", NULL, 0); + if (opt_d) + p->contiguous_planes = strtol(opt_d->value, NULL, 10); + else + p->contiguous_planes = -1; + + hwctx->enabled_dev_extensions = dev_info.ppEnabledExtensionNames; hwctx->nb_enabled_dev_extensions = dev_info.enabledExtensionCount; @@ -1425,6 +1440,8 @@ static int vulkan_device_init(AVHWDeviceContext *ctx) p->dev_is_nvidia = (p->props.properties.vendorID == 0x10de); + p->dev_is_intel = (p->props.properties.vendorID == 0x8086); + vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, &queue_num, NULL); if (!queue_num) { av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n"); @@ -1742,8 +1759,12 @@ static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f, AVHWDeviceContext *ctx = hwfc->device_ctx; VulkanDevicePriv *p = ctx->internal->priv; FFVulkanFunctions *vk = &p->vkfn; + AVVulkanFramesContext *hwfctx = hwfc->hwctx; const int planes = av_pix_fmt_count_planes(hwfc->sw_format); VkBindImageMemoryInfo bind_info[AV_NUM_DATA_POINTERS] = { { 0 } }; + VkMemoryRequirements memory_requirements = { 0 }; + int mem_size = 0; + int mem_size_list[AV_NUM_DATA_POINTERS] = { 0 }; AVVulkanDeviceContext *hwctx = ctx->hwctx; @@ -1771,6 +1792,19 @@ static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f, req.memoryRequirements.size = FFALIGN(req.memoryRequirements.size, p->props.properties.limits.minMemoryMapAlignment); + if (VKF_FLAG(hwfctx->flags, AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY)) { + if (memory_requirements.size == 0) { + memory_requirements = req.memoryRequirements; + } else if (memory_requirements.memoryTypeBits != req.memoryRequirements.memoryTypeBits) { + av_log(hwfc, AV_LOG_ERROR, "the param for each planes are not the same\n"); + return AVERROR(EINVAL); + } + + mem_size_list[i] = req.memoryRequirements.size; + mem_size += mem_size_list[i]; + continue; + } + /* In case the implementation prefers/requires dedicated allocation */ use_ded_mem = ded_req.prefersDedicatedAllocation | ded_req.requiresDedicatedAllocation; @@ -1792,6 +1826,29 @@ static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f, bind_info[i].memory = f->mem[i]; } + if (VKF_FLAG(hwfctx->flags, AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY)) { + memory_requirements.size = mem_size; + + /* Allocate memory */ + if ((err = alloc_mem(ctx, &memory_requirements, + f->tiling == VK_IMAGE_TILING_LINEAR ? + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT : + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, + (void *)(((uint8_t *)alloc_pnext)), + &f->flags, &f->mem[0]))) + return err; + + f->size[0] = memory_requirements.size; + + for (int i = 0; i < planes; i++) { + bind_info[i].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO; + bind_info[i].image = f->img[i]; + bind_info[i].memory = f->mem[0]; + bind_info[i].memoryOffset = i == 0 ? 0 : mem_size_list[i-1]; + f->offset[i] = bind_info[i].memoryOffset; + } + } + /* Bind the allocated memory to the images */ ret = vk->BindImageMemory2(hwctx->act_dev, planes, bind_info); if (ret != VK_SUCCESS) { @@ -2154,6 +2211,12 @@ static int vulkan_frames_init(AVHWFramesContext *hwfc) if (!hwctx->usage) hwctx->usage = FF_VK_DEFAULT_USAGE_FLAGS; + if (!(hwctx->flags & AV_VK_FRAME_FLAG_NONE)) { + if (p->contiguous_planes == 1 || + ((p->contiguous_planes == -1) && p->dev_is_intel)) + hwctx->flags |= AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY; + } + err = create_exec_ctx(hwfc, &fp->conv_ctx, dev_hwctx->queue_family_comp_index, dev_hwctx->nb_comp_queues); @@ -3074,6 +3137,7 @@ static int vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst, FFVulkanFunctions *vk = &p->vkfn; VulkanFramesPriv *fp = hwfc->internal->priv; AVVulkanDeviceContext *hwctx = hwfc->device_ctx->hwctx; + AVVulkanFramesContext *hwfctx = hwfc->hwctx; const int planes = av_pix_fmt_count_planes(hwfc->sw_format); VkImageDrmFormatModifierPropertiesEXT drm_mod = { .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT, @@ -3142,7 +3206,9 @@ static int vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst, continue; vk->GetImageSubresourceLayout(hwctx->act_dev, f->img[i], &sub, &layout); - drm_desc->layers[i].planes[0].offset = layout.offset; + drm_desc->layers[i].planes[0].offset = layout.offset + + VKF_FLAG(hwfctx->flags, AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY) ? + f->offset[i] : 0; drm_desc->layers[i].planes[0].pitch = layout.rowPitch; } diff --git a/libavutil/hwcontext_vulkan.h b/libavutil/hwcontext_vulkan.h index fdf2a60156..acb4073e3d 100644 --- a/libavutil/hwcontext_vulkan.h +++ b/libavutil/hwcontext_vulkan.h @@ -35,6 +35,17 @@ * with the data pointer set to an AVVkFrame. */ +/** + * Defines the behaviour of frame allocation + * AV_VK_FRAME_FLAG_NONE: planes will be allocated in separte memory + * AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY: planes will be allocated in a + * contiguous memory. + */ +typedef enum { + AV_VK_FRAME_FLAG_NONE = (1ULL << 0), + AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY = (1ULL << 1) | 1ULL +} AVVkFrameFlags; + /** * Main Vulkan context, allocated as AVHWDeviceContext.hwctx. * All of these can be set before init to change what the context uses @@ -165,6 +176,14 @@ typedef struct AVVulkanFramesContext { * extensions are present in enabled_dev_extensions. */ void *alloc_pnext[AV_NUM_DATA_POINTERS]; + + /** + * Is a combination of AVVkFrameFlags. Defines the behaviour of frame + * allocation. + * If no flag is set, then the flags are automatically determined + * based on the device. + */ + int flags; } AVVulkanFramesContext; /* @@ -230,6 +249,11 @@ typedef struct AVVkFrame { * Internal data. */ struct AVVkFrameInternal *internal; + + /** + * Describe the offset from the memory currently bound to the VkImage. + */ + size_t offset[AV_NUM_DATA_POINTERS]; } AVVkFrame; /**
The vaapi can import external frame, but the planes of the external frames should be in the same drm object. A new option "contiguous_planes" is added to device. This flag tells device to allocate places in one memory. When device is derived from vaapi this flag will be enabled. A new flag frame_flag is also added to AVVulkanFramesContext. User can use this flag to force enable or disable this behaviour. A new variable "offset "is added to AVVKFrame. It describe describe the offset from the memory currently bound to the VkImage. Signed-off-by: Wenbin Chen <wenbin.chen@intel.com> --- libavutil/hwcontext_vulkan.c | 68 +++++++++++++++++++++++++++++++++++- libavutil/hwcontext_vulkan.h | 24 +++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-)