diff --git a/libavcodec/cbs_h2645.c b/libavcodec/cbs_h2645.c
index df2b5f3f5e..2543d5f226 100644
--- a/libavcodec/cbs_h2645.c
+++ b/libavcodec/cbs_h2645.c
@@ -527,35 +527,25 @@ static int cbs_h2645_fragment_add_nals(CodedBitstreamContext *ctx,
                                        CodedBitstreamFragment *frag,
                                        const H2645Packet *packet)
 {
+    CodedBitstreamH2645Context *h2645 = ctx->priv_data;
     int err, i;
 
     for (i = 0; i < packet->nb_nals; i++) {
         const H2645NAL *nal = &packet->nals[i];
+        AVBufferRef *ref;
         size_t size = nal->size;
         // Remove trailing zeroes.
         while (size > 0 && nal->data[size - 1] == 0)
             --size;
         av_assert0(size > 0);
 
-        if (nal->data == nal->raw_data) {
-            err = ff_cbs_insert_unit_data(ctx, frag, -1, nal->type,
-                                (uint8_t*)nal->data, size, frag->data_ref);
-            if (err < 0)
-                return err;
-        } else {
-            uint8_t *data = av_malloc(size + AV_INPUT_BUFFER_PADDING_SIZE);
-            if (!data)
-                return AVERROR(ENOMEM);
-            memcpy(data, nal->data, size);
-            memset(data + size, 0, AV_INPUT_BUFFER_PADDING_SIZE);
+        ref = (nal->data == nal->raw_data) ? frag->data_ref
+                                           : h2645->rbsp_buffer_ref;
 
-            err = ff_cbs_insert_unit_data(ctx, frag, -1, nal->type,
-                                          data, size, NULL);
-            if (err < 0) {
-                av_freep(&data);
-                return err;
-            }
-        }
+        err = ff_cbs_insert_unit_data(ctx, frag, -1, nal->type,
+                            (uint8_t*)nal->data, size, ref);
+        if (err < 0)
+            return err;
     }
 
     return 0;
@@ -609,9 +599,9 @@ static int cbs_h2645_split_fragment(CodedBitstreamContext *ctx,
         }
         end = bytestream2_tell(&gbc);
 
-        err = ff_h2645_packet_split(&priv->read_packet,
-                                    frag->data + start, end - start,
-                                    ctx->log_ctx, 1, 2, AV_CODEC_ID_H264, 1, NULL);
+        err = ff_h2645_packet_split(&priv->read_packet, frag->data + start,
+                                    end - start, ctx->log_ctx, 1, 2,
+                                    AV_CODEC_ID_H264, 1, &priv->rbsp_buffer_ref);
         if (err < 0) {
             av_log(ctx->log_ctx, AV_LOG_ERROR, "Failed to split AVCC SPS array.\n");
             return err;
@@ -633,9 +623,9 @@ static int cbs_h2645_split_fragment(CodedBitstreamContext *ctx,
         }
         end = bytestream2_tell(&gbc);
 
-        err = ff_h2645_packet_split(&priv->read_packet,
-                                    frag->data + start, end - start,
-                                    ctx->log_ctx, 1, 2, AV_CODEC_ID_H264, 1, NULL);
+        err = ff_h2645_packet_split(&priv->read_packet, frag->data + start,
+                                    end - start, ctx->log_ctx, 1, 2,
+                                    AV_CODEC_ID_H264, 1, &priv->rbsp_buffer_ref);
         if (err < 0) {
             av_log(ctx->log_ctx, AV_LOG_ERROR, "Failed to split AVCC PPS array.\n");
             return err;
@@ -687,9 +677,9 @@ static int cbs_h2645_split_fragment(CodedBitstreamContext *ctx,
             }
             end = bytestream2_tell(&gbc);
 
-            err = ff_h2645_packet_split(&priv->read_packet,
-                                        frag->data + start, end - start,
-                                        ctx->log_ctx, 1, 2, AV_CODEC_ID_HEVC, 1, NULL);
+            err = ff_h2645_packet_split(&priv->read_packet, frag->data + start,
+                                        end - start, ctx->log_ctx, 1, 2,
+                                        AV_CODEC_ID_HEVC, 1, &priv->rbsp_buffer_ref);
             if (err < 0) {
                 av_log(ctx->log_ctx, AV_LOG_ERROR, "Failed to split "
                        "HVCC array %d (%d NAL units of type %d).\n",
@@ -708,7 +698,7 @@ static int cbs_h2645_split_fragment(CodedBitstreamContext *ctx,
                                     frag->data, frag->data_size,
                                     ctx->log_ctx,
                                     priv->mp4, priv->nal_length_size,
-                                    codec_id, 1, NULL);
+                                    codec_id, 1, &priv->rbsp_buffer_ref);
         if (err < 0)
             return err;
 
@@ -1510,7 +1500,9 @@ static void cbs_h264_close(CodedBitstreamContext *ctx)
     CodedBitstreamH264Context *h264 = ctx->priv_data;
     int i;
 
-    ff_h2645_packet_uninit(&h264->common.read_packet, NULL);
+    ff_h2645_packet_uninit(&h264->common.read_packet,
+                            h264->common.rbsp_buffer_ref);
+    av_buffer_unref(&h264->common.rbsp_buffer_ref);
 
     av_freep(&h264->common.write_buffer);
 
@@ -1525,7 +1517,9 @@ static void cbs_h265_close(CodedBitstreamContext *ctx)
     CodedBitstreamH265Context *h265 = ctx->priv_data;
     int i;
 
-    ff_h2645_packet_uninit(&h265->common.read_packet, NULL);
+    ff_h2645_packet_uninit(&h265->common.read_packet,
+                            h265->common.rbsp_buffer_ref);
+    av_buffer_unref(&h265->common.rbsp_buffer_ref);
 
     av_freep(&h265->common.write_buffer);
 
diff --git a/libavcodec/cbs_h2645.h b/libavcodec/cbs_h2645.h
index f4cf65bdde..2ce6176454 100644
--- a/libavcodec/cbs_h2645.h
+++ b/libavcodec/cbs_h2645.h
@@ -33,6 +33,8 @@ typedef struct CodedBitstreamH2645Context {
     int nal_length_size;
     // Packet reader.
     H2645Packet read_packet;
+    // An AVBufferRef for the rbsp_buffer of read_packet.
+    AVBufferRef *rbsp_buffer_ref;
 
     // Write buffer
     uint8_t *write_buffer;
