diff --git a/libavcodec/av1_metadata_bsf.c b/libavcodec/av1_metadata_bsf.c
index 52d383661f..c3c56afeab 100644
--- a/libavcodec/av1_metadata_bsf.c
+++ b/libavcodec/av1_metadata_bsf.c
@@ -170,7 +170,7 @@ static int av1_metadata_filter(AVBSFContext *bsf, AVPacket *out)
 
     err = 0;
 fail:
-    ff_cbs_fragment_uninit(ctx->cbc, frag);
+    ff_cbs_fragment_uninit(ctx->cbc, frag, 1);
 
     if (err < 0)
         av_packet_unref(out);
@@ -215,7 +215,7 @@ static int av1_metadata_init(AVBSFContext *bsf)
 
     err = 0;
 fail:
-    ff_cbs_fragment_uninit(ctx->cbc, frag);
+    ff_cbs_fragment_uninit(ctx->cbc, frag, 1);
     return err;
 }
 
diff --git a/libavcodec/av1_parser.c b/libavcodec/av1_parser.c
index 8df66498f4..d78e4b3f3a 100644
--- a/libavcodec/av1_parser.c
+++ b/libavcodec/av1_parser.c
@@ -72,7 +72,7 @@ static int av1_parser_parse(AVCodecParserContext *ctx,
             goto end;
         }
 
-        ff_cbs_fragment_uninit(s->cbc, td);
+        ff_cbs_fragment_uninit(s->cbc, td, 1);
     }
 
     ret = ff_cbs_read(s->cbc, td, data, size);
@@ -159,7 +159,7 @@ static int av1_parser_parse(AVCodecParserContext *ctx,
     }
 
 end:
-    ff_cbs_fragment_uninit(s->cbc, td);
+    ff_cbs_fragment_uninit(s->cbc, td, 1);
 
     s->cbc->log_ctx = NULL;
 
diff --git a/libavcodec/cbs.c b/libavcodec/cbs.c
index ecbf57c293..b61dedb1eb 100644
--- a/libavcodec/cbs.c
+++ b/libavcodec/cbs.c
@@ -137,15 +137,20 @@ static void cbs_unit_uninit(CodedBitstreamContext *ctx,
 }
 
 void ff_cbs_fragment_uninit(CodedBitstreamContext *ctx,
-                            CodedBitstreamFragment *frag)
+                            CodedBitstreamFragment *frag,
+                            int completely)
 {
     int i;
 
     for (i = 0; i < frag->nb_units; i++)
         cbs_unit_uninit(ctx, &frag->units[i]);
-    av_freep(&frag->units);
     frag->nb_units = 0;
 
+    if (completely) {
+        av_freep(&frag->units);
+        frag->units_allocated = 0;
+    }
+
     av_buffer_unref(&frag->data_ref);
     frag->data             = NULL;
     frag->data_size        = 0;
@@ -548,20 +553,34 @@ static int cbs_insert_unit(CodedBitstreamContext *ctx,
 {
     CodedBitstreamUnit *units;
 
-    units = av_malloc_array(frag->nb_units + 1, sizeof(*units));
-    if (!units)
-        return AVERROR(ENOMEM);
+    if (frag->nb_units < frag->units_allocated) {
+        units = frag->units;
+
+        if (position < frag->nb_units)
+            memmove(units + position + 1, frag->units + position,
+                    (frag->nb_units - position) * sizeof(*units));
+    } else {
+        units = av_malloc_array(frag->nb_units + 1, sizeof(*units));
+        if (!units)
+            return AVERROR(ENOMEM);
+
+        ++frag->units_allocated;
 
-    if (position > 0)
-        memcpy(units, frag->units, position * sizeof(*units));
-    if (position < frag->nb_units)
-        memcpy(units + position + 1, frag->units + position,
-               (frag->nb_units - position) * sizeof(*units));
+        if (position > 0)
+            memcpy(units, frag->units, position * sizeof(*units));
+
+        if (position < frag->nb_units)
+            memcpy(units + position + 1, frag->units + position,
+                   (frag->nb_units - position) * sizeof(*units));
+    }
 
     memset(units + position, 0, sizeof(*units));
 
-    av_freep(&frag->units);
-    frag->units = units;
+    if (units != frag->units) {
+        av_free(frag->units);
+        frag->units = units;
+    }
+
     ++frag->nb_units;
 
     return 0;
@@ -652,16 +671,10 @@ int ff_cbs_delete_unit(CodedBitstreamContext *ctx,
 
     --frag->nb_units;
 
-    if (frag->nb_units == 0) {
-        av_freep(&frag->units);
-
-    } else {
+    if (frag->nb_units > 0)
         memmove(frag->units + position,
                 frag->units + position + 1,
                 (frag->nb_units - position) * sizeof(*frag->units));
 
-        // Don't bother reallocating the unit array.
-    }
-
     return 0;
 }
diff --git a/libavcodec/cbs.h b/libavcodec/cbs.h
index 53ac360bb1..229cb129aa 100644
--- a/libavcodec/cbs.h
+++ b/libavcodec/cbs.h
@@ -145,10 +145,19 @@ typedef struct CodedBitstreamFragment {
      * and has not been decomposed.
      */
     int              nb_units;
+
+    /**
+     * Number of allocated units.
+     *
+     * Must always be >= nb_units; designed for internal use by cbs.
+     */
+     int      units_allocated;
+
     /**
-     * Pointer to an array of units of length nb_units.
+     * Pointer to an array of units of length units_allocated.
+     * Only the first nb_units are valid.
      *
-     * Must be NULL if nb_units is zero.
+     * Must be NULL if units_allocated is zero.
      */
     CodedBitstreamUnit *units;
 } CodedBitstreamFragment;
@@ -294,10 +303,12 @@ int ff_cbs_write_packet(CodedBitstreamContext *ctx,
 
 
 /**
- * Free all allocated memory in a fragment.
+ * Free all allocated memory in a fragment except possibly the unit array
+ * itself. The unit array is only freed if completely is set.
  */
 void ff_cbs_fragment_uninit(CodedBitstreamContext *ctx,
-                            CodedBitstreamFragment *frag);
+                            CodedBitstreamFragment *frag,
+                            int completely);
 
 
 /**
diff --git a/libavcodec/filter_units_bsf.c b/libavcodec/filter_units_bsf.c
index 1ee0afdf2b..a3b25cb944 100644
--- a/libavcodec/filter_units_bsf.c
+++ b/libavcodec/filter_units_bsf.c
@@ -139,7 +139,7 @@ static int filter_units_filter(AVBSFContext *bsf, AVPacket *out)
 
         // Don't return packets with nothing in them.
         av_packet_free(&in);
-        ff_cbs_fragment_uninit(ctx->cbc, frag);
+        ff_cbs_fragment_uninit(ctx->cbc, frag, 1);
     }
 
     err = ff_cbs_write_packet(ctx->cbc, out, frag);
@@ -153,7 +153,7 @@ static int filter_units_filter(AVBSFContext *bsf, AVPacket *out)
         goto fail;
 
 fail:
-    ff_cbs_fragment_uninit(ctx->cbc, frag);
+    ff_cbs_fragment_uninit(ctx->cbc, frag, 1);
     av_packet_free(&in);
 
     return err;
@@ -210,7 +210,7 @@ static int filter_units_init(AVBSFContext *bsf)
                 av_log(bsf, AV_LOG_ERROR, "Failed to write extradata.\n");
         }
 
-        ff_cbs_fragment_uninit(ctx->cbc, &ps);
+        ff_cbs_fragment_uninit(ctx->cbc, &ps, 1);
     }
 
     return err;
diff --git a/libavcodec/h264_metadata_bsf.c b/libavcodec/h264_metadata_bsf.c
index e674f2a88d..c4cfc6094f 100644
--- a/libavcodec/h264_metadata_bsf.c
+++ b/libavcodec/h264_metadata_bsf.c
@@ -604,7 +604,7 @@ static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *out)
 
     err = 0;
 fail:
-    ff_cbs_fragment_uninit(ctx->cbc, au);
+    ff_cbs_fragment_uninit(ctx->cbc, au, 1);
     av_freep(&displaymatrix_side_data);
 
     if (err < 0)
@@ -648,7 +648,7 @@ static int h264_metadata_init(AVBSFContext *bsf)
 
     err = 0;
 fail:
-    ff_cbs_fragment_uninit(ctx->cbc, au);
+    ff_cbs_fragment_uninit(ctx->cbc, au, 1);
     return err;
 }
 
diff --git a/libavcodec/h264_redundant_pps_bsf.c b/libavcodec/h264_redundant_pps_bsf.c
index 0b7888c97e..1c929cd44b 100644
--- a/libavcodec/h264_redundant_pps_bsf.c
+++ b/libavcodec/h264_redundant_pps_bsf.c
@@ -118,7 +118,7 @@ static int h264_redundant_pps_filter(AVBSFContext *bsf, AVPacket *out)
 
     err = 0;
 fail:
-    ff_cbs_fragment_uninit(ctx->output, au);
+    ff_cbs_fragment_uninit(ctx->output, au, 1);
     av_packet_free(&in);
     if (err < 0)
         av_packet_unref(out);
@@ -167,7 +167,7 @@ static int h264_redundant_pps_init(AVBSFContext *bsf)
 
     err = 0;
 fail:
-    ff_cbs_fragment_uninit(ctx->output, au);
+    ff_cbs_fragment_uninit(ctx->output, au, 1);
     return err;
 }
 
diff --git a/libavcodec/h265_metadata_bsf.c b/libavcodec/h265_metadata_bsf.c
index 26eb2d05d0..0ad2ea80b8 100644
--- a/libavcodec/h265_metadata_bsf.c
+++ b/libavcodec/h265_metadata_bsf.c
@@ -322,7 +322,7 @@ static int h265_metadata_filter(AVBSFContext *bsf, AVPacket *out)
 
     err = 0;
 fail:
-    ff_cbs_fragment_uninit(ctx->cbc, au);
+    ff_cbs_fragment_uninit(ctx->cbc, au, 1);
 
     if (err < 0)
         av_packet_unref(out);
@@ -370,7 +370,7 @@ static int h265_metadata_init(AVBSFContext *bsf)
 
     err = 0;
 fail:
-    ff_cbs_fragment_uninit(ctx->cbc, au);
+    ff_cbs_fragment_uninit(ctx->cbc, au, 1);
     return err;
 }
 
diff --git a/libavcodec/mpeg2_metadata_bsf.c b/libavcodec/mpeg2_metadata_bsf.c
index e787cb3782..420b620f6a 100644
--- a/libavcodec/mpeg2_metadata_bsf.c
+++ b/libavcodec/mpeg2_metadata_bsf.c
@@ -214,7 +214,7 @@ static int mpeg2_metadata_filter(AVBSFContext *bsf, AVPacket *out)
 
     err = 0;
 fail:
-    ff_cbs_fragment_uninit(ctx->cbc, frag);
+    ff_cbs_fragment_uninit(ctx->cbc, frag, 1);
 
     if (err < 0)
         av_packet_unref(out);
@@ -255,7 +255,7 @@ static int mpeg2_metadata_init(AVBSFContext *bsf)
 
     err = 0;
 fail:
-    ff_cbs_fragment_uninit(ctx->cbc, frag);
+    ff_cbs_fragment_uninit(ctx->cbc, frag, 1);
     return err;
 }
 
diff --git a/libavcodec/trace_headers_bsf.c b/libavcodec/trace_headers_bsf.c
index 8322229d4c..f9667f0456 100644
--- a/libavcodec/trace_headers_bsf.c
+++ b/libavcodec/trace_headers_bsf.c
@@ -50,7 +50,7 @@ static int trace_headers_init(AVBSFContext *bsf)
 
         err = ff_cbs_read_extradata(ctx->cbc, &ps, bsf->par_in);
 
-        ff_cbs_fragment_uninit(ctx->cbc, &ps);
+        ff_cbs_fragment_uninit(ctx->cbc, &ps, 1);
     }
 
     return err;
@@ -94,7 +94,7 @@ static int trace_headers(AVBSFContext *bsf, AVPacket *pkt)
 
     err = ff_cbs_read_packet(ctx->cbc, &au, pkt);
 
-    ff_cbs_fragment_uninit(ctx->cbc, &au);
+    ff_cbs_fragment_uninit(ctx->cbc, &au, 1);
 
     if (err < 0)
         av_packet_unref(pkt);
diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c
index 4ea62d96f3..b185b389dc 100644
--- a/libavcodec/vaapi_encode_h264.c
+++ b/libavcodec/vaapi_encode_h264.c
@@ -174,7 +174,7 @@ static int vaapi_encode_h264_write_sequence_header(AVCodecContext *avctx,
 
     err = vaapi_encode_h264_write_access_unit(avctx, data, data_len, au);
 fail:
-    ff_cbs_fragment_uninit(priv->cbc, au);
+    ff_cbs_fragment_uninit(priv->cbc, au, 1);
     return err;
 }
 
@@ -200,7 +200,7 @@ static int vaapi_encode_h264_write_slice_header(AVCodecContext *avctx,
 
     err = vaapi_encode_h264_write_access_unit(avctx, data, data_len, au);
 fail:
-    ff_cbs_fragment_uninit(priv->cbc, au);
+    ff_cbs_fragment_uninit(priv->cbc, au, 1);
     return err;
 }
 
@@ -264,7 +264,7 @@ static int vaapi_encode_h264_write_extra_header(AVCodecContext *avctx,
         if (err < 0)
             goto fail;
 
-        ff_cbs_fragment_uninit(priv->cbc, au);
+        ff_cbs_fragment_uninit(priv->cbc, au, 1);
 
         *type = VAEncPackedHeaderRawData;
         return 0;
@@ -286,7 +286,7 @@ static int vaapi_encode_h264_write_extra_header(AVCodecContext *avctx,
     }
 
 fail:
-    ff_cbs_fragment_uninit(priv->cbc, au);
+    ff_cbs_fragment_uninit(priv->cbc, au, 1);
     return err;
 }
 
diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c
index 19e7104e9e..cccbf2d8ec 100644
--- a/libavcodec/vaapi_encode_h265.c
+++ b/libavcodec/vaapi_encode_h265.c
@@ -159,7 +159,7 @@ static int vaapi_encode_h265_write_sequence_header(AVCodecContext *avctx,
 
     err = vaapi_encode_h265_write_access_unit(avctx, data, data_len, au);
 fail:
-    ff_cbs_fragment_uninit(priv->cbc, au);
+    ff_cbs_fragment_uninit(priv->cbc, au, 1);
     return err;
 }
 
@@ -185,7 +185,7 @@ static int vaapi_encode_h265_write_slice_header(AVCodecContext *avctx,
 
     err = vaapi_encode_h265_write_access_unit(avctx, data, data_len, au);
 fail:
-    ff_cbs_fragment_uninit(priv->cbc, au);
+    ff_cbs_fragment_uninit(priv->cbc, au, 1);
     return err;
 }
 
@@ -242,7 +242,7 @@ static int vaapi_encode_h265_write_extra_header(AVCodecContext *avctx,
         if (err < 0)
             goto fail;
 
-        ff_cbs_fragment_uninit(priv->cbc, au);
+        ff_cbs_fragment_uninit(priv->cbc, au, 1);
 
         *type = VAEncPackedHeaderRawData;
         return 0;
@@ -251,7 +251,7 @@ static int vaapi_encode_h265_write_extra_header(AVCodecContext *avctx,
     }
 
 fail:
-    ff_cbs_fragment_uninit(priv->cbc, au);
+    ff_cbs_fragment_uninit(priv->cbc, au, 1);
     return err;
 }
 
diff --git a/libavcodec/vaapi_encode_mjpeg.c b/libavcodec/vaapi_encode_mjpeg.c
index f0ea292098..730b8776dc 100644
--- a/libavcodec/vaapi_encode_mjpeg.c
+++ b/libavcodec/vaapi_encode_mjpeg.c
@@ -142,7 +142,7 @@ static int vaapi_encode_mjpeg_write_image_header(AVCodecContext *avctx,
 
     err = 0;
 fail:
-    ff_cbs_fragment_uninit(priv->cbc, frag);
+    ff_cbs_fragment_uninit(priv->cbc, frag, 1);
     return err;
 }
 
diff --git a/libavcodec/vaapi_encode_mpeg2.c b/libavcodec/vaapi_encode_mpeg2.c
index 9d42c3e644..4c4ba2432f 100644
--- a/libavcodec/vaapi_encode_mpeg2.c
+++ b/libavcodec/vaapi_encode_mpeg2.c
@@ -135,7 +135,7 @@ static int vaapi_encode_mpeg2_write_sequence_header(AVCodecContext *avctx,
 
     err = vaapi_encode_mpeg2_write_fragment(avctx, data, data_len, frag);
 fail:
-    ff_cbs_fragment_uninit(priv->cbc, frag);
+    ff_cbs_fragment_uninit(priv->cbc, frag, 1);
     return 0;
 }
 
@@ -159,7 +159,7 @@ static int vaapi_encode_mpeg2_write_picture_header(AVCodecContext *avctx,
 
     err = vaapi_encode_mpeg2_write_fragment(avctx, data, data_len, frag);
 fail:
-    ff_cbs_fragment_uninit(priv->cbc, frag);
+    ff_cbs_fragment_uninit(priv->cbc, frag, 1);
     return 0;
 }
 
diff --git a/libavcodec/vp9_metadata_bsf.c b/libavcodec/vp9_metadata_bsf.c
index be010edc3f..b275a07800 100644
--- a/libavcodec/vp9_metadata_bsf.c
+++ b/libavcodec/vp9_metadata_bsf.c
@@ -86,7 +86,7 @@ static int vp9_metadata_filter(AVBSFContext *bsf, AVPacket *out)
 
     err = 0;
 fail:
-    ff_cbs_fragment_uninit(ctx->cbc, frag);
+    ff_cbs_fragment_uninit(ctx->cbc, frag, 1);
 
     if (err < 0)
         av_packet_unref(out);
