diff --git a/libavutil/video_enc_params.c b/libavutil/video_enc_params.c
index c46c0f1dc6..9e64ad4d59 100644
--- a/libavutil/video_enc_params.c
+++ b/libavutil/video_enc_params.c
@@ -32,6 +32,9 @@ AVVideoEncParams *av_video_enc_params_alloc(enum AVVideoEncParamsType type,
     AVVideoEncParams *par;
     size_t size;
 
+    // Ensure blocks is the last field in the struct
+    av_assert0(offsetof(AVVideoEncParams, blocks) == sizeof(*par) - sizeof(AVVideoBlockParams*));
+
     size = sizeof(*par);
     if (nb_blocks > SIZE_MAX / sizeof(AVVideoBlockParams) ||
         nb_blocks * sizeof(AVVideoBlockParams) > SIZE_MAX - size)
@@ -45,7 +48,7 @@ AVVideoEncParams *av_video_enc_params_alloc(enum AVVideoEncParamsType type,
     par->type          = type;
     par->nb_blocks     = nb_blocks;
     par->block_size    = sizeof(AVVideoBlockParams);
-    par->blocks_offset = sizeof(*par);
+    par->blocks        = par->nb_blocks ? (AVVideoBlockParams *)&par[1] : NULL;
 
     if (out_size)
         *out_size = size;
@@ -53,6 +56,35 @@ AVVideoEncParams *av_video_enc_params_alloc(enum AVVideoEncParamsType type,
     return par;
 }
 
+static int copy_callback(void *opaque, AVBufferRef *dst, const uint8_t *src, int size)
+{
+    AVVideoEncParams *par_dst = (AVVideoEncParams *)dst->data;
+    AVVideoEncParams *par_src = (AVVideoEncParams *)src;
+
+    memcpy(par_dst, par_src, offsetof(AVVideoEncParams, blocks));
+    par_dst->blocks = par_src->nb_blocks ? (AVVideoBlockParams *)&par_dst[1] : NULL;
+    if (par_dst->blocks)
+        memcpy(par_dst->blocks, par_src->blocks, par_src->nb_blocks * par_src->block_size);
+
+    return 0;
+}
+
+static AVBufferRef *alloc_callback(void *opaque, int size)
+{
+    AVBufferRef *buf;
+    uint8_t *par;
+
+    par = av_malloc(size);
+    if (!par)
+        return NULL;
+    buf = av_buffer_create2(par, size, alloc_callback,
+                            copy_callback, NULL, NULL, 0);
+    if (!buf)
+        av_free(par);
+
+    return buf;
+}
+
 AVVideoEncParams*
 av_video_enc_params_create_side_data(AVFrame *frame, enum AVVideoEncParamsType type,
                                      unsigned int nb_blocks)
@@ -64,7 +96,8 @@ av_video_enc_params_create_side_data(AVFrame *frame, enum AVVideoEncParamsType t
     par = av_video_enc_params_alloc(type, nb_blocks, &size);
     if (!par)
         return NULL;
-    buf = av_buffer_create((uint8_t *)par, size, NULL, NULL, 0);
+    buf = av_buffer_create2((uint8_t *)par, size, alloc_callback,
+                            copy_callback, NULL, NULL, 0);
     if (!buf) {
         av_freep(&par);
         return NULL;
diff --git a/libavutil/video_enc_params.h b/libavutil/video_enc_params.h
index 43fa443154..b443dc50d9 100644
--- a/libavutil/video_enc_params.h
+++ b/libavutil/video_enc_params.h
@@ -57,6 +57,33 @@ enum AVVideoEncParamsType {
     AV_VIDEO_ENC_PARAMS_H264,
 };
 
+/**
+ * Data structure for storing block-level encoding information.
+ * It is allocated as a part of AVVideoEncParams and should be retrieved with
+ * av_video_enc_params_block().
+ *
+ * sizeof(AVVideoBlockParams) is not a part of the ABI and new fields may be
+ * added to it.
+ */
+typedef struct AVVideoBlockParams {
+    /**
+     * Distance in luma pixels from the top-left corner of the visible frame
+     * to the top-left corner of the block.
+     * Can be negative if top/right padding is present on the coded frame.
+     */
+    int src_x, src_y;
+    /**
+     * Width and height of the block in luma pixels.
+     */
+    int w, h;
+
+    /**
+     * Difference between this block's final quantization parameter and the
+     * corresponding per-frame value.
+     */
+    int32_t delta_qp;
+} AVVideoBlockParams;
+
 /**
  * Video encoding parameters for a given frame. This struct is allocated along
  * with an optional array of per-block AVVideoBlockParams descriptors.
@@ -67,15 +94,9 @@ typedef struct AVVideoEncParams {
      * Number of blocks in the array.
      *
      * May be 0, in which case no per-block information is present. In this case
-     * the values of blocks_offset / block_size are unspecified and should not
-     * be accessed.
+     * the value of block_size is unspecified and should not be accessed.
      */
     unsigned int nb_blocks;
-    /**
-     * Offset in bytes from the beginning of this structure at which the array
-     * of blocks starts.
-     */
-    size_t blocks_offset;
     /*
      * Size of each block in bytes. May not match sizeof(AVVideoBlockParams).
      */
@@ -99,34 +120,13 @@ typedef struct AVVideoEncParams {
      * plane (first index) and AC/DC coefficients (second index).
      */
     int32_t delta_qp[4][2];
-} AVVideoEncParams;
 
-/**
- * Data structure for storing block-level encoding information.
- * It is allocated as a part of AVVideoEncParams and should be retrieved with
- * av_video_enc_params_block().
- *
- * sizeof(AVVideoBlockParams) is not a part of the ABI and new fields may be
- * added to it.
- */
-typedef struct AVVideoBlockParams {
     /**
-     * Distance in luma pixels from the top-left corner of the visible frame
-     * to the top-left corner of the block.
-     * Can be negative if top/right padding is present on the coded frame.
+     * The arary of blocks, if any was allocated. Should only be accessed using
+     * av_video_enc_params_block().
      */
-    int src_x, src_y;
-    /**
-     * Width and height of the block in luma pixels.
-     */
-    int w, h;
-
-    /**
-     * Difference between this block's final quantization parameter and the
-     * corresponding per-frame value.
-     */
-    int32_t delta_qp;
-} AVVideoBlockParams;
+    AVVideoBlockParams *blocks;
+} AVVideoEncParams;
 
 /*
  * Get the block at the specified {@code idx}. Must be between 0 and nb_blocks.
@@ -135,8 +135,7 @@ static av_always_inline AVVideoBlockParams*
 av_video_enc_params_block(AVVideoEncParams *par, unsigned int idx)
 {
     av_assert0(idx < par->nb_blocks);
-    return (AVVideoBlockParams *)((uint8_t *)par + par->blocks_offset +
-                                  idx * par->block_size);
+    return (AVVideoBlockParams *)&par->blocks[idx];
 }
 
 /**
