[FFmpeg-devel,v3,2/2] lavc/vulkan_av1: port to the new stable API

Most of this patch was written by Dave Airlie <airlied@redhat.com>,
with some additions by me.

On 10/02/2024 22:39, Lynne wrote:
> Most of this patch was written by Dave Airlie <airlied@redhat.com>,
> with some additions by me.
>  > From be1b73147c92a8f74e5cdf30e5325382a6f77799 Mon Sep 17 00:00:00 2001
 > From: Lynne <dev@lynne.ee>
 > Date: Fri, 19 Jan 2024 10:49:02 +1000
 > Subject: [PATCH v3 2/2] lavc/vulkan_av1: port to the new stable API
 >
 > Most of this patch was written by Dave Airlie <airlied@redhat.com>,
 > with some additions by me.
 > ---
 >  libavcodec/Makefile                           |   3 +-
 >  libavcodec/vulkan_av1.c                       | 462 ++++++++++--------
 >  libavcodec/vulkan_decode.c                    |  24 +-
 >  libavcodec/vulkan_decode.h                    |   2 +-
 >  libavcodec/vulkan_video.c                     |  33 +-
 >  libavcodec/vulkan_video.h                     |   3 +-
 >  libavcodec/vulkan_video_codec_av1std.h        | 403 ---------------
 >  libavcodec/vulkan_video_codec_av1std_decode.h |  36 --
 >  libavutil/hwcontext_vulkan.c                  |   2 +-
 >  libavutil/vulkan_functions.h                  |   2 +-
 >  libavutil/vulkan_loader.h                     |   2 +-
 >  11 files changed, 303 insertions(+), 669 deletions(-)
 >  delete mode 100644 libavcodec/vulkan_video_codec_av1std.h
 >  delete mode 100644 libavcodec/vulkan_video_codec_av1std_decode.h

You need something in configure to detect whether the AV1 header is available so it can build.  (Currently this passes configure and errors out at build time for me with 1.3.275 headers.)

 > diff --git a/libavcodec/Makefile b/libavcodec/Makefile
 > index 470d7cb9b1..581415f3b7 100644
 > --- a/libavcodec/Makefile
 > +++ b/libavcodec/Makefile
 > @@ -1262,7 +1262,6 @@ SKIPHEADERS                            += %_tablegen.h                  \
 >                                            aacenc_quantization.h         \
 >                                            aacenc_quantization_misc.h    \
 >                                            bitstream_template.h          \
 > -                                          vulkan_video_codec_av1std.h   \
 >                                            $(ARCH)/vpx_arith.h          \
 >
 >  SKIPHEADERS-$(CONFIG_AMF)              += amfenc.h
 > @@ -1285,7 +1284,7 @@ SKIPHEADERS-$(CONFIG_XVMC)             += xvmc.h
 >  SKIPHEADERS-$(CONFIG_VAAPI)            += vaapi_decode.h vaapi_hevc.h vaapi_encode.h
 >  SKIPHEADERS-$(CONFIG_VDPAU)            += vdpau.h vdpau_internal.h
 >  SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX)     += videotoolbox.h vt_internal.h
 > -SKIPHEADERS-$(CONFIG_VULKAN)           += vulkan.h vulkan_video.h vulkan_decode.h vulkan_video_codec_av1std_decode.h
 > +SKIPHEADERS-$(CONFIG_VULKAN)           += vulkan.h vulkan_video.h vulkan_decode.h
 >  SKIPHEADERS-$(CONFIG_V4L2_M2M)         += v4l2_buffers.h v4l2_context.h v4l2_m2m.h
 >  SKIPHEADERS-$(CONFIG_ZLIB)             += zlib_wrapper.h
 >
 > diff --git a/libavcodec/vulkan_av1.c b/libavcodec/vulkan_av1.c
 > index 9730e4b08d..3dcad57f21 100644
 > --- a/libavcodec/vulkan_av1.c
 > +++ b/libavcodec/vulkan_av1.c
 > @@ -35,18 +35,33 @@ typedef struct AV1VulkanDecodePicture {
 >       *Can be removed once no longer needed, and threading can be enabled. */
 >      FFVulkanDecodeContext          *dec;
 >
 > -    StdVideoAV1MESATile            tiles[MAX_TILES];
 > -    StdVideoAV1MESATileList        tile_list;
 > -    const uint32_t                *tile_offsets;
 > +    uint32_t tile_count;
 > +    uint32_t tile_offsets_s[MAX_TILES];
 > +    uint32_t tile_sizes[MAX_TILES];

I don't see any check before writing things to this array.  What happens if you get a non-conforming stream with more than 256 tiles?

 > +    const uint32_t *tile_offsets;
 >
 >      /* Current picture */
 > -    VkVideoDecodeAV1DpbSlotInfoMESA    vkav1_ref;
 > -    StdVideoAV1MESAFrameHeader         av1_frame_header;
 > -    VkVideoDecodeAV1PictureInfoMESA    av1_pic_info;
 > +    StdVideoDecodeAV1ReferenceInfo     std_ref;
 > +    VkVideoDecodeAV1DpbSlotInfoKHR     vkav1_ref;
 > +    uint16_t width_in_sbs_minus1[64];
 > +    uint16_t height_in_sbs_minus1[64];
 > +    uint16_t mi_col_starts[64];
 > +    uint16_t mi_row_starts[64];
 > +    StdVideoAV1TileInfo tile_info;
 > +    StdVideoAV1Quantization quantization;
 > +    StdVideoAV1Segmentation segmentation;
 > +    StdVideoAV1LoopFilter loop_filter;
 > +    StdVideoAV1CDEF cdef;
 > +    StdVideoAV1LoopRestoration loop_restoration;
 > +    StdVideoAV1GlobalMotion global_motion;
 > +    StdVideoAV1FilmGrain film_grain;
 > +    StdVideoDecodeAV1PictureInfo    std_pic_info;
 > +    VkVideoDecodeAV1PictureInfoKHR     av1_pic_info;
 >
 >      /* Picture refs */
 >      const AV1Frame                     *ref_src   [AV1_NUM_REF_FRAMES];
 > -    VkVideoDecodeAV1DpbSlotInfoMESA     vkav1_refs[AV1_NUM_REF_FRAMES];
 > +    StdVideoDecodeAV1ReferenceInfo     std_ref_info[AV1_NUM_REF_FRAMES];
 > +    VkVideoDecodeAV1DpbSlotInfoKHR     vkav1_refs[AV1_NUM_REF_FRAMES];
 >
 >      uint8_t frame_id_set;
 >      uint8_t frame_id;
 > @@ -55,44 +70,65 @@ typedef struct AV1VulkanDecodePicture {
 >  static int vk_av1_fill_pict(AVCodecContext *avctx, const AV1Frame **ref_src,
 >                              VkVideoReferenceSlotInfoKHR *ref_slot,      /* Main structure */
 >                              VkVideoPictureResourceInfoKHR *ref,         /* Goes in ^ */
 > -                            VkVideoDecodeAV1DpbSlotInfoMESA *vkav1_ref, /* Goes in ^ */
 > -                            const AV1Frame *pic, int is_current, int has_grain,
 > -                            int dpb_slot_index)
 > +                            StdVideoDecodeAV1ReferenceInfo *vkav1_std_ref,
 > +                            VkVideoDecodeAV1DpbSlotInfoKHR *vkav1_ref, /* Goes in ^ */
 > +                            const AV1Frame *pic, int is_current, int has_grain)
 >  {
 >      FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
 >      AV1VulkanDecodePicture *hp = pic->hwaccel_picture_private;
 >      FFVulkanDecodePicture *vkpic = &hp->vp;
 > +    AV1DecContext *s = avctx->priv_data;
 > +    CodedBitstreamAV1Context *cbs_ctx;
 >
 >      int err = ff_vk_decode_prepare_frame(dec, pic->f, vkpic, is_current,
 >                                           has_grain || dec->dedicated_dpb);
 >      if (err < 0)
 >          return err;
 >
 > -    *vkav1_ref = (VkVideoDecodeAV1DpbSlotInfoMESA) {
 > -        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_DPB_SLOT_INFO_MESA,
 > -        .frameIdx = hp->frame_id,
 > +    cbs_ctx = (CodedBitstreamAV1Context *)(s->cbc->priv_data);
 > +
 > +    *vkav1_std_ref = (StdVideoDecodeAV1ReferenceInfo) {
 > +        .flags = (StdVideoDecodeAV1ReferenceInfoFlags) {
 > +            .disable_frame_end_update_cdf = pic->raw_frame_header->disable_frame_end_update_cdf,
 > +            .segmentation_enabled = pic->raw_frame_header->segmentation_enabled,
 > +        },
 > +        .frame_type = pic->raw_frame_header->frame_type,
 > +        .OrderHint = pic->raw_frame_header->order_hint,
 > +        .RefFrameSignBias = 0x0,
 > +    };
 > +
 > +    for (int i = 0; i < AV1_TOTAL_REFS_PER_FRAME; i++) {
 > +        vkav1_std_ref->SavedOrderHints[i] = cbs_ctx->order_hints[i];

"SavedOrderHints is interpreted as defined in section 7.20 of the AV1 Specification."

This doesn't look right for frames in the DPB.  SavedOrderHints in 7.20 is a two-dimensional array containing OrderHints when that frame was decoded, not the reference order hints of the current frame.

 > +        vkav1_std_ref->RefFrameSignBias |= cbs_ctx->ref_frame_sign_bias[i] << i;
 > +    }
 > +
 > +    *vkav1_ref = (VkVideoDecodeAV1DpbSlotInfoKHR) {
 > +        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_DPB_SLOT_INFO_KHR,
 > +        .pStdReferenceInfo = vkav1_std_ref,
 >      };
 >
 >      for (unsigned i = 0; i < 7; i++) {
 >          const int idx = pic->raw_frame_header->ref_frame_idx[i];
 > -        vkav1_ref->ref_order_hint[i] = pic->raw_frame_header->ref_order_hint[idx];
 > +        vkav1_std_ref->SavedOrderHints[i] = pic->raw_frame_header->ref_order_hint[idx];

Is this overwriting some of the above?

 >      }
 >
 > -    vkav1_ref->disable_frame_end_update_cdf = pic->raw_frame_header->disable_frame_end_update_cdf;
 > +    vkav1_std_ref->flags.disable_frame_end_update_cdf = pic->raw_frame_header->disable_frame_end_update_cdf;
 > +    vkav1_std_ref->flags.segmentation_enabled = pic->raw_frame_header->segmentation_enabled;
 > +    vkav1_std_ref->frame_type = pic->raw_frame_header->frame_type;
 >
 >      *ref = (VkVideoPictureResourceInfoKHR) {
 >          .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
 >          .codedOffset = (VkOffset2D){ 0, 0 },
 >          .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
 >          .baseArrayLayer = ((has_grain || dec->dedicated_dpb) && dec->layered_dpb) ?
 > -                          dpb_slot_index : 0,
 > +                          hp->frame_id : 0,
 >          .imageViewBinding = vkpic->img_view_ref,
 >      };
 >
 >      *ref_slot = (VkVideoReferenceSlotInfoKHR) {
 >          .sType = VK_STRUCTURE_TYPE_VIDEO_REFERENCE_SLOT_INFO_KHR,
 >          .pNext = vkav1_ref,
 > -        .slotIndex = dpb_slot_index,
 > +        .slotIndex = hp->frame_id,
 >          .pPictureResource = ref,
 >      };
 >
 > @@ -110,15 +146,37 @@ static int vk_av1_create_params(AVCodecContext *avctx, AVBufferRef **buf)
 >
 >      const AV1RawSequenceHeader *seq = s->raw_seq;
 >
 > -    StdVideoAV1MESASequenceHeader av1_sequence_header;
 > -    VkVideoDecodeAV1SessionParametersAddInfoMESA av1_params_info;
 > -    VkVideoDecodeAV1SessionParametersCreateInfoMESA av1_params;
 > +    StdVideoAV1SequenceHeader av1_sequence_header;
 > +    StdVideoAV1TimingInfo av1_timing_info;
 > +    StdVideoAV1ColorConfig av1_color_config;
 > +    VkVideoDecodeAV1SessionParametersCreateInfoKHR av1_params;
 >      VkVideoSessionParametersCreateInfoKHR session_params_create;
 >
 >      int err;
 >
 > -    av1_sequence_header = (StdVideoAV1MESASequenceHeader) {
 > -        .flags = (StdVideoAV1MESASequenceHeaderFlags) {
 > +    av1_timing_info = (StdVideoAV1TimingInfo) {
 > +        .flags = (StdVideoAV1TimingInfoFlags) {
 > +            .equal_picture_interval = seq->timing_info.equal_picture_interval,
 > +        },
 > +        .num_units_in_display_tick = seq->timing_info.num_units_in_display_tick,
 > +        .time_scale = seq->timing_info.time_scale,
 > +        .num_ticks_per_picture_minus_1 = seq->timing_info.num_ticks_per_picture_minus_1,
 > +    };
 > +
 > +    av1_color_config = (StdVideoAV1ColorConfig) {
 > +        .flags = (StdVideoAV1ColorConfigFlags) {
 > +            .mono_chrome = seq->color_config.mono_chrome,
 > +            .color_range = seq->color_config.color_range,
 > +            .separate_uv_delta_q = seq->color_config.separate_uv_delta_q,
 > +        },
 > +        .BitDepth = seq->color_config.twelve_bit ? 12 :
 > +        seq->color_config.high_bitdepth ? 10 : 8,
 > +        .subsampling_x = seq->color_config.subsampling_x,
 > +        .subsampling_y = seq->color_config.subsampling_y,
 > +    };
 > +
 > +    av1_sequence_header = (StdVideoAV1SequenceHeader) {
 > +        .flags = (StdVideoAV1SequenceHeaderFlags) {
 >              .still_picture = seq->still_picture,
 >              .reduced_still_picture_header = seq->reduced_still_picture_header,
 >              .use_128x128_superblock = seq->use_128x128_superblock,
 > @@ -147,34 +205,13 @@ static int vk_av1_create_params(AVCodecContext *avctx, AVBufferRef **buf)
 >          .delta_frame_id_length_minus_2 = seq->delta_frame_id_length_minus_2,
 >          .additional_frame_id_length_minus_1 = seq->additional_frame_id_length_minus_1,
 >          .order_hint_bits_minus_1 = seq->order_hint_bits_minus_1,
 > -        .timing_info = (StdVideoAV1MESATimingInfo) {
 > -            .flags = (StdVideoAV1MESATimingInfoFlags) {
 > -                .equal_picture_interval = seq->timing_info.equal_picture_interval,
 > -            },
 > -            .num_units_in_display_tick = seq->timing_info.num_units_in_display_tick,
 > -            .time_scale = seq->timing_info.time_scale,
 > -            .num_ticks_per_picture_minus_1 = seq->timing_info.num_ticks_per_picture_minus_1,
 > -        },
 > -        .color_config = (StdVideoAV1MESAColorConfig) {
 > -            .flags = (StdVideoAV1MESAColorConfigFlags) {
 > -                .mono_chrome = seq->color_config.mono_chrome,
 > -                .color_range = seq->color_config.color_range,
 > -                .separate_uv_delta_q = seq->color_config.separate_uv_delta_q,
 > -            },
 > -            .bit_depth = seq->color_config.twelve_bit ? 12 :
 > -                         seq->color_config.high_bitdepth ? 10 : 8,
 > -            .subsampling_x = seq->color_config.subsampling_x,
 > -            .subsampling_y = seq->color_config.subsampling_y,
 > -        },
 > +        .pTimingInfo = &av1_timing_info,
 > +        .pColorConfig = &av1_color_config,
 >      };
 >
 > -    av1_params_info = (VkVideoDecodeAV1SessionParametersAddInfoMESA) {
 > -        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_SESSION_PARAMETERS_ADD_INFO_MESA,
 > -        .sequence_header = &av1_sequence_header,
 > -    };
 > -    av1_params = (VkVideoDecodeAV1SessionParametersCreateInfoMESA) {
 > -        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_SESSION_PARAMETERS_CREATE_INFO_MESA,
 > -        .pParametersAddInfo = &av1_params_info,
 > +    av1_params = (VkVideoDecodeAV1SessionParametersCreateInfoKHR) {
 > +        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_SESSION_PARAMETERS_CREATE_INFO_KHR,
 > +        .pStdSequenceHeader = &av1_sequence_header,
 >      };
 >      session_params_create = (VkVideoSessionParametersCreateInfoKHR) {
 >          .sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_PARAMETERS_CREATE_INFO_KHR,
 > @@ -208,6 +245,11 @@ static int vk_av1_start_frame(AVCodecContext          *avctx,
 >      const AV1RawFilmGrainParams *film_grain = &s->cur_frame.film_grain;
 >      const int apply_grain = !(avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) &&
 >                              film_grain->apply_grain;
 > +    StdVideoAV1FrameRestorationType remap_lr_type[4] = { STD_VIDEO_AV1_FRAME_RESTORATION_TYPE_NONE,
 > +                                                         STD_VIDEO_AV1_FRAME_RESTORATION_TYPE_SWITCHABLE,
 > +                                                         STD_VIDEO_AV1_FRAME_RESTORATION_TYPE_WIENER,
 > +                                                         STD_VIDEO_AV1_FRAME_RESTORATION_TYPE_SGRPROJ };
 > +    int uses_lr;
 >
 >      if (!dec->session_params) {
 >          err = vk_av1_create_params(avctx, &dec->session_params);
 > @@ -235,8 +277,8 @@ static int vk_av1_start_frame(AVCodecContext          *avctx,
 >              continue;
 >
 >          err = vk_av1_fill_pict(avctx, &ap->ref_src[i], &vp->ref_slots[i],
 > -                               &vp->refs[i], &ap->vkav1_refs[i],
 > -                               ref_frame, 0, 0, i);
 > +                               &vp->refs[i], &ap->std_ref_info[i], &ap->vkav1_refs[i],
 > +                               ref_frame, 0, 0);
 >          if (err < 0)
 >              return err;
 >
 > @@ -244,20 +286,25 @@ static int vk_av1_start_frame(AVCodecContext          *avctx,
 >      }
 >
 >      err = vk_av1_fill_pict(avctx, NULL, &vp->ref_slot, &vp->ref,
 > +                           &ap->std_ref,
 >                             &ap->vkav1_ref,
 > -                           pic, 1, apply_grain, 8);
 > +                           pic, 1, apply_grain);
 >      if (err < 0)
 >          return err;
 >
 > -    ap->tile_list.nb_tiles = 0;
 > -    ap->tile_list.tile_list = ap->tiles;
 > +    ap->tile_count = 0;
 >
 > -    ap->av1_pic_info = (VkVideoDecodeAV1PictureInfoMESA) {
 > -        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PICTURE_INFO_MESA,
 > -        .frame_header = &ap->av1_frame_header,
 > -        .tile_list = &ap->tile_list,
 > +    ap->av1_pic_info = (VkVideoDecodeAV1PictureInfoKHR) {
 > +        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PICTURE_INFO_KHR,
 > +        .pStdPictureInfo = &ap->std_pic_info,
 > +        .tileCount = ap->tile_count,
 > +        .pTileOffsets = ap->tile_offsets_s,
 > +        .pTileSizes = ap->tile_sizes,
 >      };
 >
 > +    for (int i = 0; i < STD_VIDEO_AV1_REFS_PER_FRAME; i++)
 > +        ap->av1_pic_info.referenceNameSlotIndices[i] = vp->ref_slots[frame_header->ref_frame_idx[i]].slotIndex;
 > +
 >      vp->decode_info = (VkVideoDecodeInfoKHR) {
 >          .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_INFO_KHR,
 >          .pNext = &ap->av1_pic_info,
 > @@ -274,9 +321,94 @@ static int vk_av1_start_frame(AVCodecContext          *avctx,
 >          },
 >      };
 >
 > +    ap->tile_info = (StdVideoAV1TileInfo) {
 > +        .flags = (StdVideoAV1TileInfoFlags) {
 > +            .uniform_tile_spacing_flag = frame_header->uniform_tile_spacing_flag,
 > +        },
 > +        .TileCols = frame_header->tile_cols,
 > +        .TileRows = frame_header->tile_rows,
 > +        .context_update_tile_id = frame_header->context_update_tile_id,
 > +        .tile_size_bytes_minus_1 = frame_header->tile_size_bytes_minus1,
 > +        .pWidthInSbsMinus1 = ap->width_in_sbs_minus1,
 > +        .pHeightInSbsMinus1 = ap->height_in_sbs_minus1,
 > +        .pMiColStarts = ap->mi_col_starts,
 > +        .pMiRowStarts = ap->mi_row_starts,
 > +    };
 > +
 > +    ap->quantization = (StdVideoAV1Quantization) {
 > +        .flags.using_qmatrix = frame_header->using_qmatrix,
 > +        .flags.diff_uv_delta = frame_header->diff_uv_delta,
 > +        .base_q_idx = frame_header->base_q_idx,
 > +        .DeltaQYDc = frame_header->delta_q_y_dc,
 > +        .DeltaQUDc = frame_header->delta_q_u_dc,
 > +        .DeltaQUAc = frame_header->delta_q_u_ac,
 > +        .DeltaQVDc = frame_header->delta_q_v_dc,
 > +        .DeltaQVAc = frame_header->delta_q_v_ac,
 > +        .qm_y = frame_header->qm_y,
 > +        .qm_u = frame_header->qm_u,
 > +        .qm_v = frame_header->qm_v,
 > +    };
 > +
 > +    for (int i = 0; i < STD_VIDEO_AV1_MAX_SEGMENTS; i++) {
 > +        for (int j = 0; j < STD_VIDEO_AV1_SEG_LVL_MAX; j++) {
 > +            ap->segmentation.FeatureEnabled[i] |= frame_header->feature_enabled[i][j] << j;

"the elements of FeatureEnabled are bitmasks where bit index i of element j corresponds to FeatureEnabled[i][j] as defined in section 6.8.13 of the AV1 Specification"

This looks like you have i and j swapped?

 > +            ap->segmentation.FeatureData[i][j] = frame_header->feature_value[i][j];
 > +        }
 > +    }
 > +
 > +    ap->loop_filter = (StdVideoAV1LoopFilter) {
 > +        .flags = (StdVideoAV1LoopFilterFlags) {
 > +            .loop_filter_delta_enabled = frame_header->loop_filter_delta_enabled,
 > +            .loop_filter_delta_update = frame_header->loop_filter_delta_update,
 > +        },
 > +        .loop_filter_sharpness = frame_header->loop_filter_sharpness,
 > +    };
 > +
 > +    for (int i = 0; i < STD_VIDEO_AV1_MAX_LOOP_FILTER_STRENGTHS; i++)
 > +        ap->loop_filter.loop_filter_level[i] = frame_header->loop_filter_level[i];
 > +    for (int i = 0; i < STD_VIDEO_AV1_LOOP_FILTER_ADJUSTMENTS; i++)
 > +        ap->loop_filter.loop_filter_mode_deltas[i] = frame_header->loop_filter_mode_deltas[i];
 > +
 > +    ap->cdef = (StdVideoAV1CDEF) {
 > +        .cdef_damping_minus_3 = frame_header->cdef_damping_minus_3,
 > +        .cdef_bits = frame_header->cdef_bits,
 > +    };
 > +
 > +    uses_lr = frame_header->lr_type[0] || frame_header->lr_type[1] || frame_header->lr_type[2],
 > +    ap->loop_restoration = (StdVideoAV1LoopRestoration) {
 > +        .FrameRestorationType[0] = remap_lr_type[frame_header->lr_type[0]],
 > +        .FrameRestorationType[1] = remap_lr_type[frame_header->lr_type[1]],
 > +        .FrameRestorationType[2] = remap_lr_type[frame_header->lr_type[2]],
 > +        .LoopRestorationSize[0] = 1 + frame_header->lr_unit_shift,
 > +        .LoopRestorationSize[1] = 1 + frame_header->lr_unit_shift - frame_header->lr_uv_shift,
 > +        .LoopRestorationSize[2] = 1 + frame_header->lr_unit_shift - frame_header->lr_uv_shift,

The standard has:

LoopRestorationSize[ 0 ] = RESTORATION_TILESIZE_MAX >> (2 - lr_unit_shift)
LoopRestorationSize[ 1 ] = LoopRestorationSize[ 0 ] >> lr_uv_shift
LoopRestorationSize[ 2 ] = LoopRestorationSize[ 0 ] >> lr_uv_shift

The values here look like log2 of these rather than what they are meant to be?

 > +    };
 > +
 > +    ap->film_grain = (StdVideoAV1FilmGrain) {
 > +        .flags = (StdVideoAV1FilmGrainFlags) {
 > +            .chroma_scaling_from_luma = film_grain->chroma_scaling_from_luma,
 > +            .overlap_flag = film_grain->overlap_flag,
 > +            .clip_to_restricted_range = film_grain->clip_to_restricted_range,
 > +        },
 > +        .grain_scaling_minus_8 = film_grain->grain_scaling_minus_8,
 > +        .ar_coeff_lag = film_grain->ar_coeff_lag,
 > +        .ar_coeff_shift_minus_6 = film_grain->ar_coeff_shift_minus_6,
 > +        .grain_scale_shift = film_grain->grain_scale_shift,
 > +        .grain_seed = film_grain->grain_seed,

.film_grain_params_ref_idx got lost here.

 > +        .num_y_points = film_grain->num_y_points,
 > +        .num_cb_points = film_grain->num_cb_points,
 > +        .num_cr_points = film_grain->num_cr_points,
 > +        .cb_mult = film_grain->cb_mult,
 > +        .cb_luma_mult = film_grain->cb_luma_mult,
 > +        .cb_offset = film_grain->cb_offset,
 > +        .cr_mult = film_grain->cr_mult,
 > +        .cr_luma_mult = film_grain->cr_luma_mult,
 > +        .cr_offset = film_grain->cr_offset,
 > +    };
 > +
 >      /* Setup frame header */
 > -    ap->av1_frame_header = (StdVideoAV1MESAFrameHeader) {
 > -        .flags = (StdVideoAV1MESAFrameHeaderFlags) {
 > +    ap->std_pic_info = (StdVideoDecodeAV1PictureInfo) {
 > +        .flags = (StdVideoDecodeAV1PictureInfoFlags) {
 >              .error_resilient_mode = frame_header->error_resilient_mode,
 >              .disable_cdf_update = frame_header->disable_cdf_update,
 >              .use_superres = frame_header->use_superres,
 > @@ -297,171 +429,87 @@ static int vk_av1_start_frame(AVCodecContext          *avctx,
 >              .reference_select = frame_header->reference_select,
 >              .skip_mode_present = frame_header->skip_mode_present,
 >              .delta_q_present = frame_header->delta_q_present,
 > +            .delta_lf_present = frame_header->delta_lf_present,
 > +            .delta_lf_multi = frame_header->delta_lf_multi,
 > +            .segmentation_enabled = frame_header->segmentation_enabled,
 > +            .segmentation_update_map = frame_header->segmentation_update_map,
 > +            .segmentation_temporal_update = frame_header->segmentation_temporal_update,
 > +            .segmentation_update_data = frame_header->segmentation_update_data,
 > +            .UsesLr = uses_lr,
 > +            .apply_grain = apply_grain,
 >          },
 > -        .frame_to_show_map_idx = frame_header->frame_to_show_map_idx,
 > -        .frame_presentation_time = frame_header->frame_presentation_time,
 > -        .display_frame_id = frame_header->display_frame_id,
 >          .frame_type = frame_header->frame_type,
 >          .current_frame_id = frame_header->current_frame_id,
 > -        .order_hint = frame_header->order_hint,
 > +        .OrderHint = frame_header->order_hint,
 >          .primary_ref_frame = frame_header->primary_ref_frame,
 > -        .frame_width_minus_1 = frame_header->frame_width_minus_1,
 > -        .frame_height_minus_1 = frame_header->frame_height_minus_1,
 > -        .coded_denom = frame_header->coded_denom,
 > -        .render_width_minus_1 = frame_header->render_width_minus_1,
 > -        .render_height_minus_1 = frame_header->render_height_minus_1,
 >          .refresh_frame_flags = frame_header->refresh_frame_flags,
 >          .interpolation_filter = frame_header->interpolation_filter,
 > -        .tx_mode = frame_header->tx_mode,
 > -        .tiling = (StdVideoAV1MESATileInfo) {
 > -            .flags = (StdVideoAV1MESATileInfoFlags) {
 > -                .uniform_tile_spacing_flag = frame_header->uniform_tile_spacing_flag,
 > -            },
 > -            .tile_cols = frame_header->tile_cols,
 > -            .tile_rows = frame_header->tile_rows,
 > -            .context_update_tile_id = frame_header->context_update_tile_id,
 > -            .tile_size_bytes_minus1 = frame_header->tile_size_bytes_minus1,
 > -        },
 > -        .quantization = (StdVideoAV1MESAQuantization) {
 > -            .flags.using_qmatrix = frame_header->using_qmatrix,
 > -            .base_q_idx = frame_header->base_q_idx,
 > -            .delta_q_y_dc = frame_header->delta_q_y_dc,
 > -            .diff_uv_delta = frame_header->diff_uv_delta,
 > -            .delta_q_u_dc = frame_header->delta_q_u_dc,
 > -            .delta_q_u_ac = frame_header->delta_q_u_ac,
 > -            .delta_q_v_dc = frame_header->delta_q_v_dc,
 > -            .delta_q_v_ac = frame_header->delta_q_v_ac,
 > -            .qm_y = frame_header->qm_y,
 > -            .qm_u = frame_header->qm_u,
 > -            .qm_v = frame_header->qm_v,
 > -        },
 > -        .delta_q = (StdVideoAV1MESADeltaQ) {
 > -            .flags = (StdVideoAV1MESADeltaQFlags) {
 > -                .delta_lf_present = frame_header->delta_lf_present,
 > -                .delta_lf_multi = frame_header->delta_lf_multi,
 > -            },
 > -            .delta_q_res = frame_header->delta_q_res,
 > -            .delta_lf_res = frame_header->delta_lf_res,
 > -        },
 > -        .loop_filter = (StdVideoAV1MESALoopFilter) {
 > -            .flags = (StdVideoAV1MESALoopFilterFlags) {
 > -                .delta_enabled = frame_header->loop_filter_delta_enabled,
 > -                .delta_update = frame_header->loop_filter_delta_update,
 > -            },
 > -            .level = {
 > -                frame_header->loop_filter_level[0], frame_header->loop_filter_level[1],
 > -                frame_header->loop_filter_level[2], frame_header->loop_filter_level[3],
 > -            },
 > -            .sharpness = frame_header->loop_filter_sharpness,
 > -            .mode_deltas = {
 > -                frame_header->loop_filter_mode_deltas[0], frame_header->loop_filter_mode_deltas[1],
 > -            },
 > -        },
 > -        .cdef = (StdVideoAV1MESACDEF) {
 > -            .damping_minus_3 = frame_header->cdef_damping_minus_3,
 > -            .bits = frame_header->cdef_bits,
 > -        },
 > -        .lr = (StdVideoAV1MESALoopRestoration) {
 > -            .lr_unit_shift = frame_header->lr_unit_shift,
 > -            .lr_uv_shift = frame_header->lr_uv_shift,
 > -            .lr_type = { frame_header->lr_type[0], frame_header->lr_type[1], frame_header->lr_type[2] },
 > -        },
 > -        .segmentation = (StdVideoAV1MESASegmentation) {
 > -            .flags = (StdVideoAV1MESASegmentationFlags) {
 > -                .enabled = frame_header->segmentation_enabled,
 > -                .update_map = frame_header->segmentation_update_map,
 > -                .temporal_update = frame_header->segmentation_temporal_update,
 > -                .update_data = frame_header->segmentation_update_data,
 > -            },
 > -        },
 > -        .film_grain = (StdVideoAV1MESAFilmGrainParameters) {
 > -            .flags = (StdVideoAV1MESAFilmGrainFlags) {
 > -                .apply_grain = apply_grain,
 > -                .chroma_scaling_from_luma = film_grain->chroma_scaling_from_luma,
 > -                .overlap_flag = film_grain->overlap_flag,
 > -                .clip_to_restricted_range = film_grain->clip_to_restricted_range,
 > -            },
 > -            .grain_scaling_minus_8 = film_grain->grain_scaling_minus_8,
 > -            .ar_coeff_lag = film_grain->ar_coeff_lag,
 > -            .ar_coeff_shift_minus_6 = film_grain->ar_coeff_shift_minus_6,
 > -            .grain_scale_shift = film_grain->grain_scale_shift,
 > -            .grain_seed = film_grain->grain_seed,
 > -            .num_y_points = film_grain->num_y_points,
 > -            .num_cb_points = film_grain->num_cb_points,
 > -            .num_cr_points = film_grain->num_cr_points,
 > -            .cb_mult = film_grain->cb_mult,
 > -            .cb_luma_mult = film_grain->cb_luma_mult,
 > -            .cb_offset = film_grain->cb_offset,
 > -            .cr_mult = film_grain->cr_mult,
 > -            .cr_luma_mult = film_grain->cr_luma_mult,
 > -            .cr_offset = film_grain->cr_offset,
 > -        },
 > +        .TxMode = frame_header->tx_mode,
 > +        .delta_q_res = frame_header->delta_q_res,
 > +        .delta_lf_res = frame_header->delta_lf_res,
 > +        .SkipModeFrame[0] = s->cur_frame.skip_mode_frame_idx[0],
 > +        .SkipModeFrame[1] = s->cur_frame.skip_mode_frame_idx[1],
 > +        .coded_denom = frame_header->coded_denom,
 > +        .pTileInfo = &ap->tile_info,
 > +        .pQuantization = &ap->quantization,
 > +        .pSegmentation = &ap->segmentation,
 > +        .pLoopFilter = &ap->loop_filter,
 > +        .pCDEF = &ap->cdef,
 > +        .pLoopRestoration = &ap->loop_restoration,
 > +        .pGlobalMotion = &ap->global_motion,
 > +        .pFilmGrain = apply_grain ? &ap->film_grain : NULL,
 >      };
 >
 >      for (int i = 0; i < 64; i++) {
 > -        ap->av1_frame_header.tiling.width_in_sbs_minus_1[i] = frame_header->width_in_sbs_minus_1[i];
 > -        ap->av1_frame_header.tiling.height_in_sbs_minus_1[i] = frame_header->height_in_sbs_minus_1[i];
 > -        ap->av1_frame_header.tiling.tile_start_col_sb[i] = frame_header->tile_start_col_sb[i];
 > -        ap->av1_frame_header.tiling.tile_start_row_sb[i] = frame_header->tile_start_row_sb[i];
 > +        ap->width_in_sbs_minus1[i] = frame_header->width_in_sbs_minus_1[i];
 > +        ap->height_in_sbs_minus1[i] = frame_header->height_in_sbs_minus_1[i];
 > +        ap->mi_col_starts[i] = frame_header->tile_start_col_sb[i];
 > +        ap->mi_row_starts[i] = frame_header->tile_start_row_sb[i];
 >      }
 >
 >      for (int i = 0; i < 8; i++) {
 > -        ap->av1_frame_header.segmentation.feature_enabled_bits[i] = 0;
 > -        for (int j = 0; j < 8; j++) {
 > -            ap->av1_frame_header.segmentation.feature_enabled_bits[i] |= (frame_header->feature_enabled[i][j] << j);
 > -            ap->av1_frame_header.segmentation.feature_data[i][j] = frame_header->feature_value[i][j];
 > +        ap->segmentation.FeatureEnabled[i] = 0x0;
 > +        for (int j = 0; j < STD_VIDEO_AV1_SEG_LVL_MAX; j++) {
 > +            ap->segmentation.FeatureEnabled[i] |= (frame_header->feature_enabled[i][j] << j);
 > +            ap->segmentation.FeatureData[i][j] = frame_header->feature_value[i][j];
 >          }
 >
 > -        ap->av1_frame_header.loop_filter.ref_deltas[i] = frame_header->loop_filter_ref_deltas[i];
 > -
 > -        ap->av1_frame_header.cdef.y_pri_strength[i] = frame_header->cdef_y_pri_strength[i];
 > -        ap->av1_frame_header.cdef.y_sec_strength[i] = frame_header->cdef_y_sec_strength[i];
 > -        ap->av1_frame_header.cdef.uv_pri_strength[i] = frame_header->cdef_uv_pri_strength[i];
 > -        ap->av1_frame_header.cdef.uv_sec_strength[i] = frame_header->cdef_uv_sec_strength[i];
 > -
 > -        ap->av1_frame_header.ref_order_hint[i] = frame_header->ref_order_hint[i];
 > -        ap->av1_frame_header.global_motion[i] = (StdVideoAV1MESAGlobalMotion) {
 > -            .flags = (StdVideoAV1MESAGlobalMotionFlags) {
 > -                .gm_invalid = s->cur_frame.gm_invalid[i],
 > -            },
 > -            .gm_type = s->cur_frame.gm_type[i],
 > -            .gm_params = {
 > -                s->cur_frame.gm_params[i][0], s->cur_frame.gm_params[i][1],
 > -                s->cur_frame.gm_params[i][2], s->cur_frame.gm_params[i][3],
 > -                s->cur_frame.gm_params[i][4], s->cur_frame.gm_params[i][5],
 > -            },
 > -        };
 > -    }
 > +        ap->loop_filter.loop_filter_ref_deltas[i] = frame_header->loop_filter_ref_deltas[i];
 >
 > -    for (int i = 0; i < 7; i++) {
 > -        ap->av1_frame_header.ref_frame_idx[i] = frame_header->ref_frame_idx[i];
 > -        ap->av1_frame_header.delta_frame_id_minus1[i] = frame_header->delta_frame_id_minus1[i];
 > -    }
 > +        ap->cdef.cdef_y_pri_strength[i] = frame_header->cdef_y_pri_strength[i];
 > +        ap->cdef.cdef_y_sec_strength[i] = frame_header->cdef_y_sec_strength[i];
 > +        ap->cdef.cdef_uv_pri_strength[i] = frame_header->cdef_uv_pri_strength[i];
 > +        ap->cdef.cdef_uv_sec_strength[i] = frame_header->cdef_uv_sec_strength[i];
 >
 > -    ap->av1_pic_info.skip_mode_frame_idx[0] = s->cur_frame.skip_mode_frame_idx[0];
 > -    ap->av1_pic_info.skip_mode_frame_idx[1] = s->cur_frame.skip_mode_frame_idx[1];
 > +        /* Reference frames */
 > +        ap->std_pic_info.OrderHints[i] = frame_header->ref_order_hint[i];
 > +        ap->global_motion.GmType[i] = s->cur_frame.gm_type[i];
 > +        for (int j = 0; j < STD_VIDEO_AV1_GLOBAL_MOTION_PARAMS; j++) {
 > +            ap->global_motion.gm_params[i][j] = s->cur_frame.gm_params[i][j];
 > +        }
 > +    }
 >
 >      if (apply_grain) {
 >          for (int i = 0; i < 14; i++) {

The Vulkan headers provide some nice constants for you to use here, use them rather than magic numbers.

(Maybe we should add our own versions of these in av1.h?)

 > -            ap->av1_frame_header.film_grain.point_y_value[i] = film_grain->point_y_value[i];
 > -            ap->av1_frame_header.film_grain.point_y_scaling[i] = film_grain->point_y_scaling[i];
 > +            ap->film_grain.point_y_value[i] = film_grain->point_y_value[i];
 > +            ap->film_grain.point_y_scaling[i] = film_grain->point_y_scaling[i];
 >          }
 >
 >          for (int i = 0; i < 10; i++) {
 > -            ap->av1_frame_header.film_grain.point_cb_value[i] = film_grain->point_cb_value[i];
 > -            ap->av1_frame_header.film_grain.point_cb_scaling[i] = film_grain->point_cb_scaling[i];
 > -            ap->av1_frame_header.film_grain.point_cr_value[i] = film_grain->point_cr_value[i];
 > -            ap->av1_frame_header.film_grain.point_cr_scaling[i] = film_grain->point_cr_scaling[i];
 > +            ap->film_grain.point_cb_value[i] = film_grain->point_cb_value[i];
 > +            ap->film_grain.point_cb_scaling[i] = film_grain->point_cb_scaling[i];
 > +            ap->film_grain.point_cr_value[i] = film_grain->point_cr_value[i];
 > +            ap->film_grain.point_cr_scaling[i] = film_grain->point_cr_scaling[i];
 >          }
 >
 >          for (int i = 0; i < 24; i++) {
 > -            ap->av1_frame_header.film_grain.ar_coeffs_y_plus_128[i] = film_grain->ar_coeffs_y_plus_128[i];
 > -            ap->av1_frame_header.film_grain.ar_coeffs_cb_plus_128[i] = film_grain->ar_coeffs_cb_plus_128[i];
 > -            ap->av1_frame_header.film_grain.ar_coeffs_cr_plus_128[i] = film_grain->ar_coeffs_cr_plus_128[i];
 > +            ap->film_grain.ar_coeffs_y_plus_128[i] = film_grain->ar_coeffs_y_plus_128[i];
 > +            ap->film_grain.ar_coeffs_cb_plus_128[i] = film_grain->ar_coeffs_cb_plus_128[i];
 > +            ap->film_grain.ar_coeffs_cr_plus_128[i] = film_grain->ar_coeffs_cr_plus_128[i];
 >          }
 >
 > -        ap->av1_frame_header.film_grain.ar_coeffs_cb_plus_128[24] = film_grain->ar_coeffs_cb_plus_128[24];
 > -        ap->av1_frame_header.film_grain.ar_coeffs_cr_plus_128[24] = film_grain->ar_coeffs_cr_plus_128[24];
 > +        ap->film_grain.ar_coeffs_cb_plus_128[24] = film_grain->ar_coeffs_cb_plus_128[24];
 > +        ap->film_grain.ar_coeffs_cr_plus_128[24] = film_grain->ar_coeffs_cr_plus_128[24];
 >      }
 >
 >      /* Workaround for a spec issue. */

Is whatever this is still present?

 > @@ -480,26 +528,22 @@ static int vk_av1_decode_slice(AVCodecContext *avctx,
 >      FFVulkanDecodePicture *vp = &ap->vp;
 >
 >      for (int i = s->tg_start; i <= s->tg_end; i++) {
 > -        ap->tiles[ap->tile_list.nb_tiles] = (StdVideoAV1MESATile) {
 > -            .size     = s->tile_group_info[i].tile_size,
 > -            .offset   = s->tile_group_info[i].tile_offset,
 > -            .row      = s->tile_group_info[i].tile_row,
 > -            .column   = s->tile_group_info[i].tile_column,
 > -            .tg_start = s->tg_start,
 > -            .tg_end   = s->tg_end,
 > -        };
 > +
 > +        ap->tile_offsets_s[ap->tile_count] = s->tile_group_info[i].tile_offset;
 > +        ap->tile_sizes[ap->tile_count] = s->tile_group_info[i].tile_size;

I'm confused by why this is indexed by ap->tile_count rather than, say, i - s->tg_start?  Isn't this repeatedly overwriting one entry in the array and never touching the others?

 >
 >          err = ff_vk_decode_add_slice(avctx, vp,
 >                                       data + s->tile_group_info[i].tile_offset,
 >                                       s->tile_group_info[i].tile_size, 0,
 > -                                     &ap->tile_list.nb_tiles,
 > +                                     &ap->tile_count,
 >                                       &ap->tile_offsets);
 >          if (err < 0)
 >              return err;
 >
 > -        ap->tiles[ap->tile_list.nb_tiles - 1].offset = ap->tile_offsets[ap->tile_list.nb_tiles - 1];
 > +        ap->tile_offsets_s[ap->tile_count - 1] = ap->tile_offsets[ap->tile_count - 1];

And it's not at all clear what this is doing.

 >      }
 >
 > +    ap->av1_pic_info.tileCount = ap->tile_count;
 >      return 0;
 >  }
 >
 > @@ -513,7 +557,7 @@ static int vk_av1_end_frame(AVCodecContext *avctx)
 >      FFVulkanDecodePicture *rvp[AV1_NUM_REF_FRAMES] = { 0 };
 >      AVFrame *rav[AV1_NUM_REF_FRAMES] = { 0 };
 >
 > -    if (!ap->tile_list.nb_tiles)
 > +    if (!ap->tile_count)
 >          return 0;
 >
 >      if (!dec->session_params) {
 > @@ -531,7 +575,7 @@ static int vk_av1_end_frame(AVCodecContext *avctx)
 >      }
 >
 >      av_log(avctx, AV_LOG_VERBOSE, "Decoding frame, %"SIZE_SPECIFIER" bytes, %i tiles\n",
 > -           vp->slices_size, ap->tile_list.nb_tiles);
 > +           vp->slices_size, ap->tile_count);
 >
 >      return ff_vk_decode_frame(avctx, pic->f, vp, rav, rvp);
 >  }
 > diff --git a/libavcodec/vulkan_decode.c b/libavcodec/vulkan_decode.c
 > index fdbcbb450a..3da2c0ed1b 100644
 > --- a/libavcodec/vulkan_decode.c
 > +++ b/libavcodec/vulkan_decode.c
 > @@ -50,7 +50,7 @@ static const VkVideoProfileInfoKHR *get_video_profile(FFVulkanDecodeShared *ctx,
 >      VkStructureType profile_struct_type =
 >          codec_id == AV_CODEC_ID_H264 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR :
 >          codec_id == AV_CODEC_ID_HEVC ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR :
 > -        codec_id == AV_CODEC_ID_AV1  ? VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_MESA :
 > +        codec_id == AV_CODEC_ID_AV1  ? VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR :
 >          0;
 >
 >      profile_list = ff_vk_find_struct(ctx->s.hwfc->create_pnext,
 > @@ -663,7 +663,7 @@ static VkResult vulkan_setup_profile(AVCodecContext *avctx,
 >                                       const struct FFVkCodecMap *vk_codec,
 >                                       VkVideoDecodeH264CapabilitiesKHR *h264_caps,
 >                                       VkVideoDecodeH265CapabilitiesKHR *h265_caps,
 > -                                     VkVideoDecodeAV1CapabilitiesMESA *av1_caps,
 > +                                     VkVideoDecodeAV1CapabilitiesKHR *av1_caps,
 >                                       VkVideoCapabilitiesKHR *caps,
 >                                       VkVideoDecodeCapabilitiesKHR *dec_caps,
 >                                       int cur_profile)
 > @@ -674,7 +674,7 @@ static VkResult vulkan_setup_profile(AVCodecContext *avctx,
 >
 >      VkVideoDecodeH264ProfileInfoKHR *h264_profile = &prof->h264_profile;
 >      VkVideoDecodeH265ProfileInfoKHR *h265_profile = &prof->h265_profile;
 > -    VkVideoDecodeAV1ProfileInfoMESA *av1_profile  = &prof->av1_profile;
 > +    VkVideoDecodeAV1ProfileInfoKHR *av1_profile  = &prof->av1_profile;
 >
 >      const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->sw_pix_fmt);
 >      if (!desc)
 > @@ -702,8 +702,8 @@ static VkResult vulkan_setup_profile(AVCodecContext *avctx,
 >      } else if (avctx->codec_id == AV_CODEC_ID_AV1) {
 >          dec_caps->pNext = av1_caps;
 >          usage->pNext = av1_profile;
 > -        av1_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_MESA;
 > -        av1_profile->stdProfileIdc = cur_profile;
 > +        av1_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR;
 > +        av1_profile->stdProfile = cur_profile;
 >      }
 >
 >      usage->sType           = VK_STRUCTURE_TYPE_VIDEO_DECODE_USAGE_INFO_KHR;
 > @@ -758,8 +758,8 @@ static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_
 >      VkVideoDecodeH265CapabilitiesKHR h265_caps = {
 >          .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_CAPABILITIES_KHR,
 >      };
 > -    VkVideoDecodeAV1CapabilitiesMESA av1_caps = {
 > -        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_CAPABILITIES_MESA,
 > +    VkVideoDecodeAV1CapabilitiesKHR av1_caps = {
 > +        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_CAPABILITIES_KHR,
 >      };
 >
 >      VkPhysicalDeviceVideoFormatInfoKHR fmt_info = {
 > @@ -782,7 +782,7 @@ static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_
 >      cur_profile = avctx->profile;
 >      base_profile = avctx->codec_id == AV_CODEC_ID_H264 ? AV_PROFILE_H264_CONSTRAINED_BASELINE :
 >                     avctx->codec_id == AV_CODEC_ID_H265 ? AV_PROFILE_HEVC_MAIN :
 > -                   avctx->codec_id == AV_CODEC_ID_AV1  ? STD_VIDEO_AV1_MESA_PROFILE_MAIN :
 > +                   avctx->codec_id == AV_CODEC_ID_AV1  ? STD_VIDEO_AV1_PROFILE_MAIN :
 >                     0;
 >
 >      ret = vulkan_setup_profile(avctx, prof, hwctx, vk, vk_codec,
 > @@ -830,7 +830,7 @@ static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_
 >
 >      max_level = avctx->codec_id == AV_CODEC_ID_H264 ? ff_vk_h264_level_to_av(h264_caps.maxLevelIdc) :
 >                  avctx->codec_id == AV_CODEC_ID_H265 ? ff_vk_h265_level_to_av(h265_caps.maxLevelIdc) :
 > -                avctx->codec_id == AV_CODEC_ID_AV1  ? av1_caps.maxLevelIdc  :
 > +                avctx->codec_id == AV_CODEC_ID_AV1  ? ff_vk_av1_level_to_av(av1_caps.maxLevel) :
 >                  0;
 >
 >      av_log(avctx, AV_LOG_VERBOSE, "Decoder capabilities for %s profile \"%s\":\n",
 > @@ -911,7 +911,7 @@ static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_
 >      /* TODO: make dedicated_dpb tunable */
 >      dec->dedicated_dpb = !(dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR);
 >      dec->layered_dpb = !(caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR);
 > -    dec->external_fg = av1_caps.flags & VK_VIDEO_DECODE_AV1_CAPABILITY_EXTERNAL_FILM_GRAIN_MESA;
 > +    //    dec->external_fg = av1_caps.flags & VK_VIDEO_DECODE_AV1_CAPABILITY_EXTERNAL_FILM_GRAIN_KHR;

What does this mean for the film grain capabilities of the current setup?

 >
 >      if (dec->dedicated_dpb) {
 >          fmt_info.imageUsage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
 > @@ -1118,8 +1118,8 @@ int ff_vk_decode_init(AVCodecContext *avctx)
 >      VkVideoDecodeH265SessionParametersCreateInfoKHR h265_params = {
 >          .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_SESSION_PARAMETERS_CREATE_INFO_KHR,
 >      };
 > -    VkVideoDecodeAV1SessionParametersCreateInfoMESA av1_params = {
 > -        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_SESSION_PARAMETERS_CREATE_INFO_MESA,
 > +    VkVideoDecodeAV1SessionParametersCreateInfoKHR av1_params = {
 > +        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_SESSION_PARAMETERS_CREATE_INFO_KHR,
 >      };
 >      VkVideoSessionParametersCreateInfoKHR session_params_create = {
 >          .sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_PARAMETERS_CREATE_INFO_KHR,
 > diff --git a/libavcodec/vulkan_decode.h b/libavcodec/vulkan_decode.h
 > index b0b1ff5614..9991de6794 100644
 > --- a/libavcodec/vulkan_decode.h
 > +++ b/libavcodec/vulkan_decode.h
 > @@ -28,7 +28,7 @@
 >  typedef struct FFVulkanDecodeProfileData {
 >      VkVideoDecodeH264ProfileInfoKHR h264_profile;
 >      VkVideoDecodeH265ProfileInfoKHR h265_profile;
 > -    VkVideoDecodeAV1ProfileInfoMESA av1_profile;
 > +    VkVideoDecodeAV1ProfileInfoKHR av1_profile;
 >      VkVideoDecodeUsageInfoKHR usage;
 >      VkVideoProfileInfoKHR profile;
 >      VkVideoProfileListInfoKHR profile_list;
 > diff --git a/libavcodec/vulkan_video.c b/libavcodec/vulkan_video.c
 > index fb20315db4..2c992b0330 100644
 > --- a/libavcodec/vulkan_video.c
 > +++ b/libavcodec/vulkan_video.c
 > @@ -37,7 +37,7 @@ const FFVkCodecMap ff_vk_codec_map[AV_CODEC_ID_FIRST_AUDIO] = {
 >                             0,
 >                             0,
 >                             FF_VK_EXT_VIDEO_DECODE_AV1,
 > -                           0x01000000 /* TODO fix this */
 > +                           VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR
 >      },
 >  };
 >
 > @@ -199,6 +199,37 @@ int ff_vk_h265_level_to_av(StdVideoH265LevelIdc level)
 >      }
 >  }
 >
 > +int ff_vk_av1_level_to_av(StdVideoAV1Level level)
 > +{
 > +    switch (level) {
 > +    case STD_VIDEO_AV1_LEVEL_2_0: return 20;
 > +    case STD_VIDEO_AV1_LEVEL_2_1: return 21;
 > +    case STD_VIDEO_AV1_LEVEL_2_2: return 22;
 > +    case STD_VIDEO_AV1_LEVEL_2_3: return 23;
 > +    case STD_VIDEO_AV1_LEVEL_3_0: return 30;
 > +    case STD_VIDEO_AV1_LEVEL_3_1: return 31;
 > +    case STD_VIDEO_AV1_LEVEL_3_2: return 32;
 > +    case STD_VIDEO_AV1_LEVEL_3_3: return 33;
 > +    case STD_VIDEO_AV1_LEVEL_4_0: return 40;
 > +    case STD_VIDEO_AV1_LEVEL_4_1: return 41;
 > +    case STD_VIDEO_AV1_LEVEL_4_2: return 42;
 > +    case STD_VIDEO_AV1_LEVEL_4_3: return 43;
 > +    case STD_VIDEO_AV1_LEVEL_5_0: return 50;
 > +    case STD_VIDEO_AV1_LEVEL_5_1: return 51;
 > +    case STD_VIDEO_AV1_LEVEL_5_2: return 52;
 > +    case STD_VIDEO_AV1_LEVEL_5_3: return 53;
 > +    case STD_VIDEO_AV1_LEVEL_6_0: return 60;
 > +    case STD_VIDEO_AV1_LEVEL_6_1: return 61;
 > +    case STD_VIDEO_AV1_LEVEL_6_2: return 62;
 > +    case STD_VIDEO_AV1_LEVEL_6_3: return 63;
 > +    case STD_VIDEO_AV1_LEVEL_7_0: return 70;
 > +    case STD_VIDEO_AV1_LEVEL_7_1: return 71;
 > +    case STD_VIDEO_AV1_LEVEL_7_2: return 72;
 > +    default:
 > +    case STD_VIDEO_AV1_LEVEL_7_3: return 73;
 > +    }
 > +}

Er, what?  Where have the numbers on the RHS of this come from?  The vulkan header shows the numbers on the left as correctly matching the spec.

 > +
 >  static void free_data_buf(void *opaque, uint8_t *data)
 >  {
 >      FFVulkanContext *ctx = opaque;
 > diff --git a/libavcodec/vulkan_video.h b/libavcodec/vulkan_video.h
 > index b28e3fe0bd..2bdb8ab97a 100644
 > --- a/libavcodec/vulkan_video.h
 > +++ b/libavcodec/vulkan_video.h
 > @@ -23,8 +23,6 @@
 >  #include "vulkan.h"
 >
 >  #include <vk_video/vulkan_video_codecs_common.h>
 > -#include "vulkan_video_codec_av1std.h"
 > -#include "vulkan_video_codec_av1std_decode.h"
 >
 >  #define CODEC_VER_MAJ(ver) (ver >> 22)
 >  #define CODEC_VER_MIN(ver) ((ver >> 12) & ((1 << 10) - 1))
 > @@ -77,6 +75,7 @@ VkVideoComponentBitDepthFlagBitsKHR ff_vk_depth_from_av_depth(int depth);
 >   */
 >  int ff_vk_h264_level_to_av(StdVideoH264LevelIdc level);
 >  int ff_vk_h265_level_to_av(StdVideoH265LevelIdc level);
 > +int ff_vk_av1_level_to_av(StdVideoAV1Level level);
 >
 >  typedef struct FFVkVideoBuffer {
 >      FFVkBuffer buf;
 > diff --git a/libavcodec/vulkan_video_codec_av1std.h b/libavcodec/vulkan_video_codec_av1std.h
 > deleted file mode 100644
 > ...
 > diff --git a/libavcodec/vulkan_video_codec_av1std_decode.h b/libavcodec/vulkan_video_codec_av1std_decode.h
 > deleted file mode 100644
 > ...
 > diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
 > index a84713e621..44b763550d 100644
 > --- a/libavutil/hwcontext_vulkan.c
 > +++ b/libavutil/hwcontext_vulkan.c
 > @@ -436,7 +436,7 @@ static const VulkanOptExtension optional_device_exts[] = {
 >      { VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME,               FF_VK_EXT_VIDEO_DECODE_QUEUE     },
 >      { VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME,                FF_VK_EXT_VIDEO_DECODE_H264      },
 >      { VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME,                FF_VK_EXT_VIDEO_DECODE_H265      },
 > -    { "VK_MESA_video_decode_av1",                             FF_VK_EXT_VIDEO_DECODE_AV1       },
 > +    { VK_KHR_VIDEO_DECODE_AV1_EXTENSION_NAME,                 FF_VK_EXT_VIDEO_DECODE_AV1       },
 >  };
 >
 >  static VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
 > diff --git a/libavutil/vulkan_functions.h b/libavutil/vulkan_functions.h
 > index 65021b04b1..6b379acf93 100644
 > --- a/libavutil/vulkan_functions.h
 > +++ b/libavutil/vulkan_functions.h
 > @@ -43,7 +43,7 @@ typedef enum FFVulkanExtensions {
 >      FF_VK_EXT_VIDEO_DECODE_QUEUE     = 1ULL << 11, /* VK_KHR_video_decode_queue */
 >      FF_VK_EXT_VIDEO_DECODE_H264      = 1ULL << 12, /* VK_EXT_video_decode_h264 */
 >      FF_VK_EXT_VIDEO_DECODE_H265      = 1ULL << 13, /* VK_EXT_video_decode_h265 */
 > -    FF_VK_EXT_VIDEO_DECODE_AV1       = 1ULL << 14, /* VK_MESA_video_decode_av1 */
 > +    FF_VK_EXT_VIDEO_DECODE_AV1       = 1ULL << 14, /* VK_KHR_video_decode_av1 */
 >      FF_VK_EXT_ATOMIC_FLOAT           = 1ULL << 15, /* VK_EXT_shader_atomic_float */
 >      FF_VK_EXT_COOP_MATRIX            = 1ULL << 16, /* VK_KHR_cooperative_matrix */
 >
 > diff --git a/libavutil/vulkan_loader.h b/libavutil/vulkan_loader.h
 > index f88722f28f..fc854242dd 100644
 > --- a/libavutil/vulkan_loader.h
 > +++ b/libavutil/vulkan_loader.h
 > @@ -56,7 +56,7 @@ static inline uint64_t ff_vk_extensions_to_mask(const char * const *extensions,
 >          { VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME,        FF_VK_EXT_VIDEO_DECODE_QUEUE     },
 >          { VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME,         FF_VK_EXT_VIDEO_DECODE_H264      },
 >          { VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME,         FF_VK_EXT_VIDEO_DECODE_H265      },
 > -        { "VK_MESA_video_decode_av1",                      FF_VK_EXT_VIDEO_DECODE_AV1       },
 > +        { VK_KHR_VIDEO_DECODE_AV1_EXTENSION_NAME,          FF_VK_EXT_VIDEO_DECODE_AV1       },
 >      };
 >
 >      FFVulkanExtensions mask = 0x0;
 > --
 > 2.43.0.381.gb435a96ce8
 >

Thanks,

- Mark

Feb 11, 2024, 12:58 by sw@jkqxz.net:

> On 10/02/2024 22:39, Lynne wrote:
>
>>  11 files changed, 303 insertions(+), 669 deletions(-)
>>  delete mode 100644 libavcodec/vulkan_video_codec_av1std.h
>>  delete mode 100644 libavcodec/vulkan_video_codec_av1std_decode.h
>>
>
> You need something in configure to detect whether the AV1 header is available so it can build.  (Currently this passes configure and errors out at build time for me with 1.3.275 headers.)
>

This affects multiple files, and it's a bit too much to add an ifdef,
so I've added a version check (1.3.277).


>> -    StdVideoAV1MESATile            tiles[MAX_TILES];
>> -    StdVideoAV1MESATileList        tile_list;
>> -    const uint32_t                *tile_offsets;
>> +    uint32_t tile_count;
>> +    uint32_t tile_offsets_s[MAX_TILES];
>> +    uint32_t tile_sizes[MAX_TILES];
>>
>
> I don't see any check before writing things to this array.  What happens if you get a non-conforming stream with more than 256 tiles?
>

Fixed, added a check in vk_av1_decode_slice().


>> +
>> +    for (int i = 0; i < AV1_TOTAL_REFS_PER_FRAME; i++) {
>> +        vkav1_std_ref->SavedOrderHints[i] = cbs_ctx->order_hints[i];
>>
>
> "SavedOrderHints is interpreted as defined in section 7.20 of the AV1 Specification."
>
> This doesn't look right for frames in the DPB.  SavedOrderHints in 7.20 is a two-dimensional array containing OrderHints when that frame was decoded, not the reference order hints of the current frame.
>

SavedOrderHints is a stack that is simply pushed at the time of decoding,
isn't it?
I think this is correct, as these are derived from the bitstream values before
decoding.


>> +        vkav1_std_ref->RefFrameSignBias |= cbs_ctx->ref_frame_sign_bias[i] << i;
>> +    }
>> +
>> +    *vkav1_ref = (VkVideoDecodeAV1DpbSlotInfoKHR) {
>> +        .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_DPB_SLOT_INFO_KHR,
>> +        .pStdReferenceInfo = vkav1_std_ref,
>>  };
>>
>>  for (unsigned i = 0; i < 7; i++) {
>>  const int idx = pic->raw_frame_header->ref_frame_idx[i];
>> -        vkav1_ref->ref_order_hint[i] = pic->raw_frame_header->ref_order_hint[idx];
>> +        vkav1_std_ref->SavedOrderHints[i] = pic->raw_frame_header->ref_order_hint[idx];
>>
>
> Is this overwriting some of the above?
>

Yes, removed.


>> +    for (int i = 0; i < STD_VIDEO_AV1_MAX_SEGMENTS; i++) {
>> +        for (int j = 0; j < STD_VIDEO_AV1_SEG_LVL_MAX; j++) {
>> +            ap->segmentation.FeatureEnabled[i] |= frame_header->feature_enabled[i][j] << j;
>>
>
> "the elements of FeatureEnabled are bitmasks where bit index i of element j corresponds to FeatureEnabled[i][j] as defined in section 6.8.13 of the AV1 Specification"
>
> This looks like you have i and j swapped?
>

The header define it as:
uint8_t FeatureEnabled[STD_VIDEO_AV1_MAX_SEGMENTS];
We define it as:
uint8_t feature_enabled[AV1_MAX_SEGMENTS][AV1_SEG_LVL_MAX];

So FeatureEnabled[i] |= feature_enabled[i][j] << j;
should be correct.


>> +            ap->segmentation.FeatureData[i][j] = frame_header->feature_value[i][j];
>> +        }
>> +    }
>> +
>> +    ap->loop_filter = (StdVideoAV1LoopFilter) {
>> +        .flags = (StdVideoAV1LoopFilterFlags) {
>> +            .loop_filter_delta_enabled = frame_header->loop_filter_delta_enabled,
>> +            .loop_filter_delta_update = frame_header->loop_filter_delta_update,
>> +        },
>> +        .loop_filter_sharpness = frame_header->loop_filter_sharpness,
>> +    };
>> +
>> +    for (int i = 0; i < STD_VIDEO_AV1_MAX_LOOP_FILTER_STRENGTHS; i++)
>> +        ap->loop_filter.loop_filter_level[i] = frame_header->loop_filter_level[i];
>> +    for (int i = 0; i < STD_VIDEO_AV1_LOOP_FILTER_ADJUSTMENTS; i++)
>> +        ap->loop_filter.loop_filter_mode_deltas[i] = frame_header->loop_filter_mode_deltas[i];
>> +
>> +    ap->cdef = (StdVideoAV1CDEF) {
>> +        .cdef_damping_minus_3 = frame_header->cdef_damping_minus_3,
>> +        .cdef_bits = frame_header->cdef_bits,
>> +    };
>> +
>> +    uses_lr = frame_header->lr_type[0] || frame_header->lr_type[1] || frame_header->lr_type[2],
>> +    ap->loop_restoration = (StdVideoAV1LoopRestoration) {
>> +        .FrameRestorationType[0] = remap_lr_type[frame_header->lr_type[0]],
>> +        .FrameRestorationType[1] = remap_lr_type[frame_header->lr_type[1]],
>> +        .FrameRestorationType[2] = remap_lr_type[frame_header->lr_type[2]],
>> +        .LoopRestorationSize[0] = 1 + frame_header->lr_unit_shift,
>> +        .LoopRestorationSize[1] = 1 + frame_header->lr_unit_shift - frame_header->lr_uv_shift,
>> +        .LoopRestorationSize[2] = 1 + frame_header->lr_unit_shift - frame_header->lr_uv_shift,
>>
>
> The standard has:
>
> LoopRestorationSize[ 0 ] = RESTORATION_TILESIZE_MAX >> (2 - lr_unit_shift)
> LoopRestorationSize[ 1 ] = LoopRestorationSize[ 0 ] >> lr_uv_shift
> LoopRestorationSize[ 2 ] = LoopRestorationSize[ 0 ] >> lr_uv_shift
>
> The values here look like log2 of these rather than what they are meant to be?
>

Correct, fixed. Added an AV1_RESTORATION_TILESIZE_MAX constant to av1.h
and calculated them as the spec requires.


>> +    };
>> +
>> +    ap->film_grain = (StdVideoAV1FilmGrain) {
>> +        .flags = (StdVideoAV1FilmGrainFlags) {
>> +            .chroma_scaling_from_luma = film_grain->chroma_scaling_from_luma,
>> +            .overlap_flag = film_grain->overlap_flag,
>> +            .clip_to_restricted_range = film_grain->clip_to_restricted_range,
>> +        },
>> +        .grain_scaling_minus_8 = film_grain->grain_scaling_minus_8,
>> +        .ar_coeff_lag = film_grain->ar_coeff_lag,
>> +        .ar_coeff_shift_minus_6 = film_grain->ar_coeff_shift_minus_6,
>> +        .grain_scale_shift = film_grain->grain_scale_shift,
>> +        .grain_seed = film_grain->grain_seed,
>>
>
> .film_grain_params_ref_idx got lost here.
>

Fixed.


>>
>>  if (apply_grain) {
>>  for (int i = 0; i < 14; i++) {
>>
>
> The Vulkan headers provide some nice constants for you to use here, use them rather than magic numbers.
>
> (Maybe we should add our own versions of these in av1.h?)
>

Fixed. Did not add our own versions, used the header's.


>>
>> -        ap->av1_frame_header.film_grain.ar_coeffs_cb_plus_128[24] = film_grain->ar_coeffs_cb_plus_128[24];
>> -        ap->av1_frame_header.film_grain.ar_coeffs_cr_plus_128[24] = film_grain->ar_coeffs_cr_plus_128[24];
>> +        ap->film_grain.ar_coeffs_cb_plus_128[24] = film_grain->ar_coeffs_cb_plus_128[24];
>> +        ap->film_grain.ar_coeffs_cr_plus_128[24] = film_grain->ar_coeffs_cr_plus_128[24];
>>  }
>>
>>  /* Workaround for a spec issue. */
>>
>
> Is whatever this is still present?
>

Yes. Removed the comment, added a small TODO at the top.
Sad, all the hardware had the same limitations of requiring a
unique index for each frame, despite supporting image addresses
for the output image.
I think it's fixable with us as the indices should correspond to e.g. LAST_FRAME,
but since we don't have threading on right now, we can fix it later.


>> @@ -480,26 +528,22 @@ static int vk_av1_decode_slice(AVCodecContext *avctx,
>>  FFVulkanDecodePicture *vp = &ap->vp;
>>
>>  for (int i = s->tg_start; i <= s->tg_end; i++) {
>> -        ap->tiles[ap->tile_list.nb_tiles] = (StdVideoAV1MESATile) {
>> -            .size     = s->tile_group_info[i].tile_size,
>> -            .offset   = s->tile_group_info[i].tile_offset,
>> -            .row      = s->tile_group_info[i].tile_row,
>> -            .column   = s->tile_group_info[i].tile_column,
>> -            .tg_start = s->tg_start,
>> -            .tg_end   = s->tg_end,
>> -        };
>> +
>> +        ap->tile_offsets_s[ap->tile_count] = s->tile_group_info[i].tile_offset;
>> +        ap->tile_sizes[ap->tile_count] = s->tile_group_info[i].tile_size;
>>
>
> I'm confused by why this is indexed by ap->tile_count rather than, say, i - s->tg_start?  Isn't this repeatedly overwriting one entry in the array and never touching the others?
>

No, the call to ff_vk_decode_add_slice() modifies ap->tile_count.
It's how it's handled in H26x as well.
The logic is tiles are only added if the function call succeeds.


>>
>> err = ff_vk_decode_add_slice(avctx, vp,
>>  data + s->tile_group_info[i].tile_offset,
>>  s->tile_group_info[i].tile_size, 0,
>> -                                     &ap->tile_list.nb_tiles,
>> +                                     &ap->tile_count,
>>  &ap->tile_offsets);
>>  if (err < 0)
>>  return err;
>>
>> -        ap->tiles[ap->tile_list.nb_tiles - 1].offset = ap->tile_offsets[ap->tile_list.nb_tiles - 1];
>> +        ap->tile_offsets_s[ap->tile_count - 1] = ap->tile_offsets[ap->tile_count - 1];
>>
>
> And it's not at all clear what this is doing.
>

Leftover code. Removed, along with tile_offsets_s.


>>  av_log(avctx, AV_LOG_VERBOSE, "Decoder capabilities for %s profile \"%s\":\n",
>> @@ -911,7 +911,7 @@ static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_
>>  /* TODO: make dedicated_dpb tunable */
>>  dec->dedicated_dpb = !(dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR);
>>  dec->layered_dpb = !(caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR);
>> -    dec->external_fg = av1_caps.flags & VK_VIDEO_DECODE_AV1_CAPABILITY_EXTERNAL_FILM_GRAIN_MESA;
>> +    //    dec->external_fg = av1_caps.flags & VK_VIDEO_DECODE_AV1_CAPABILITY_EXTERNAL_FILM_GRAIN_KHR;
>>
>
> What does this mean for the film grain capabilities of the current setup?
>

No film grain on Intel.

Drivers will likely error out on profiles with filmGrainSupport = VK_TRUE,
so avctx->export_side_data must have AV_CODEC_EXPORT_DATA_FILM_GRAIN
for filmGrainSupport to be VK_FALSE, and the application will have to support
applying film grain by itself to be compliant.

We'll have to copy libplacebo's film grain generation eventually,
but I think erroring out is fine for now.

We could force side data to be exported in case the hardware doesn't
support it in a later patch.


>> +int ff_vk_av1_level_to_av(StdVideoAV1Level level)
>> +{
>> +    switch (level) {
>> +    case STD_VIDEO_AV1_LEVEL_2_0: return 20;
>> +    case STD_VIDEO_AV1_LEVEL_2_1: return 21;
>> +    case STD_VIDEO_AV1_LEVEL_2_2: return 22;
>> +    case STD_VIDEO_AV1_LEVEL_2_3: return 23;
>> +    case STD_VIDEO_AV1_LEVEL_3_0: return 30;
>> +    case STD_VIDEO_AV1_LEVEL_3_1: return 31;
>> +    case STD_VIDEO_AV1_LEVEL_3_2: return 32;
>> +    case STD_VIDEO_AV1_LEVEL_3_3: return 33;
>> +    case STD_VIDEO_AV1_LEVEL_4_0: return 40;
>> +    case STD_VIDEO_AV1_LEVEL_4_1: return 41;
>> +    case STD_VIDEO_AV1_LEVEL_4_2: return 42;
>> +    case STD_VIDEO_AV1_LEVEL_4_3: return 43;
>> +    case STD_VIDEO_AV1_LEVEL_5_0: return 50;
>> +    case STD_VIDEO_AV1_LEVEL_5_1: return 51;
>> +    case STD_VIDEO_AV1_LEVEL_5_2: return 52;
>> +    case STD_VIDEO_AV1_LEVEL_5_3: return 53;
>> +    case STD_VIDEO_AV1_LEVEL_6_0: return 60;
>> +    case STD_VIDEO_AV1_LEVEL_6_1: return 61;
>> +    case STD_VIDEO_AV1_LEVEL_6_2: return 62;
>> +    case STD_VIDEO_AV1_LEVEL_6_3: return 63;
>> +    case STD_VIDEO_AV1_LEVEL_7_0: return 70;
>> +    case STD_VIDEO_AV1_LEVEL_7_1: return 71;
>> +    case STD_VIDEO_AV1_LEVEL_7_2: return 72;
>> +    default:
>> +    case STD_VIDEO_AV1_LEVEL_7_3: return 73;
>> +    }
>> +}
>>
>
> Er, what?  Where have the numbers on the RHS of this come from?  The vulkan header shows the numbers on the left as correctly matching the spec.
>

It's how lavc represents levels for H26x codecs, so
I thought it's the same for AV1. But checking that now, it isn't the case,
so removed it.

Thanks for the review, this caught a lot of issues.
Will send v4 to the ML shortly.