[FFmpeg-devel,03/11] avformat/avformat: use the side data from AVStream.codecpar

diff --git a/libavdevice/android_camera.c b/libavdevice/android_camera.c
index 0425b27518..602b59047d 100644
--- a/libavdevice/android_camera.c
+++ b/libavdevice/android_camera.c
@@ -639,7 +639,7 @@  static int wait_for_image_format(AVFormatContext *avctx)
 static int add_display_matrix(AVFormatContext *avctx, AVStream *st)
 {
     AndroidCameraCtx *ctx = avctx->priv_data;
-    uint8_t *side_data;
+    AVPacketSideData *side_data;
     int32_t display_matrix[9];
 
     av_display_rotation_set(display_matrix, ctx->sensor_orientation);
@@ -648,14 +648,16 @@  static int add_display_matrix(AVFormatContext *avctx, AVStream *st)
         av_display_matrix_flip(display_matrix, 1, 0);
     }
 
-    side_data = av_stream_new_side_data(st,
-            AV_PKT_DATA_DISPLAYMATRIX, sizeof(display_matrix));
+    side_data = av_packet_side_data_new(&st->codecpar->side_data,
+                                        &st->codecpar->nb_side_data,
+                                        AV_PKT_DATA_DISPLAYMATRIX,
+                                        sizeof(display_matrix), 0);
 
     if (!side_data) {
         return AVERROR(ENOMEM);
     }
 
-    memcpy(side_data, display_matrix, sizeof(display_matrix));
+    memcpy(side_data->data, display_matrix, sizeof(display_matrix));
 
     return 0;
 }
diff --git a/libavformat/avformat.c b/libavformat/avformat.c
index 7ff0cf3429..5b8bb7879e 100644
--- a/libavformat/avformat.c
+++ b/libavformat/avformat.c
@@ -48,9 +48,13 @@  void ff_free_stream(AVStream **pst)
     if (!st)
         return;
 
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
     for (int i = 0; i < st->nb_side_data; i++)
         av_freep(&st->side_data[i].data);
     av_freep(&st->side_data);
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
 
     if (st->attached_pic.data)
         av_packet_unref(&st->attached_pic);
@@ -140,6 +144,8 @@  void avformat_free_context(AVFormatContext *s)
     av_free(s);
 }
 
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
 uint8_t *av_stream_get_side_data(const AVStream *st,
                                  enum AVPacketSideDataType type, size_t *size)
 {
@@ -207,36 +213,8 @@  uint8_t *av_stream_new_side_data(AVStream *st, enum AVPacketSideDataType type,
 
     return data;
 }
-
-int ff_stream_side_data_copy(AVStream *dst, const AVStream *src)
-{
-    /* Free existing side data*/
-    for (int i = 0; i < dst->nb_side_data; i++)
-        av_free(dst->side_data[i].data);
-    av_freep(&dst->side_data);
-    dst->nb_side_data = 0;
-
-    /* Copy side data if present */
-    if (src->nb_side_data) {
-        dst->side_data = av_calloc(src->nb_side_data,
-                                   sizeof(*dst->side_data));
-        if (!dst->side_data)
-            return AVERROR(ENOMEM);
-        dst->nb_side_data = src->nb_side_data;
-
-        for (int i = 0; i < src->nb_side_data; i++) {
-            uint8_t *data = av_memdup(src->side_data[i].data,
-                                      src->side_data[i].size);
-            if (!data)
-                return AVERROR(ENOMEM);
-            dst->side_data[i].type = src->side_data[i].type;
-            dst->side_data[i].size = src->side_data[i].size;
-            dst->side_data[i].data = data;
-        }
-    }
-
-    return 0;
-}
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
 
 /**
  * Copy all stream parameters from source to destination stream, with the
@@ -272,10 +250,6 @@  static int stream_params_copy(AVStream *dst, const AVStream *src)
     if (ret < 0)
         return ret;
 
-    ret = ff_stream_side_data_copy(dst, src);
-    if (ret < 0)
-        return ret;
-
     av_packet_unref(&dst->attached_pic);
     if (src->attached_pic.data) {
         ret = av_packet_ref(&dst->attached_pic, &src->attached_pic);
diff --git a/libavformat/avformat.h b/libavformat/avformat.h
index 43175ba332..9100d310b9 100644
--- a/libavformat/avformat.h
+++ b/libavformat/avformat.h
@@ -938,6 +938,7 @@  typedef struct AVStream {
      */
     AVPacket attached_pic;
 
+#if FF_API_AVSTREAM_SIDE_DATA
     /**
      * An array of side data that applies to the whole stream (i.e. the
      * container does not allow it to change between packets).
@@ -954,13 +955,20 @@  typedef struct AVStream {
      *
      * Freed by libavformat in avformat_free_context().
      *
-     * @see av_format_inject_global_side_data()
+     * @deprecated use AVStream's @ref AVCodecParameters.side_data
+     *             "codecpar side data".
      */
+    attribute_deprecated
     AVPacketSideData *side_data;
     /**
      * The number of elements in the AVStream.side_data array.
+     *
+     * @deprecated use AVStream's @ref AVCodecParameters.side_data
+     *             "codecpar side data".
      */
+    attribute_deprecated
     int            nb_side_data;
+#endif
 
     /**
      * Flags indicating events happening on the stream, a combination of
@@ -1718,11 +1726,18 @@  typedef struct AVFormatContext {
     int (*io_close2)(struct AVFormatContext *s, AVIOContext *pb);
 } AVFormatContext;
 
+#if FF_API_AVSTREAM_SIDE_DATA
 /**
  * This function will cause global side data to be injected in the next packet
  * of each stream as well as after any subsequent seek.
+ *
+ * @deprecated global side data is always available in every AVStream's
+ *             @ref AVCodecParameters.side_data "codecpar side data" array.
+ * @see av_packet_side_data_get()
  */
+attribute_deprecated
 void av_format_inject_global_side_data(AVFormatContext *s);
+#endif
 
 /**
  * Returns the method used to set ctx->duration.
@@ -1847,6 +1862,7 @@  const AVClass *av_stream_get_class(void);
  */
 AVStream *avformat_new_stream(AVFormatContext *s, const struct AVCodec *c);
 
+#if FF_API_AVSTREAM_SIDE_DATA
 /**
  * Wrap an existing array as stream side data.
  *
@@ -1859,7 +1875,10 @@  AVStream *avformat_new_stream(AVFormatContext *s, const struct AVCodec *c);
  *
  * @return zero on success, a negative AVERROR code on failure. On failure,
  *         the stream is unchanged and the data remains owned by the caller.
+ * @deprecated use av_packet_side_data_add() with the stream's
+ *             @ref AVCodecParameters.side_data "codecpar side data"
  */
+attribute_deprecated
 int av_stream_add_side_data(AVStream *st, enum AVPacketSideDataType type,
                             uint8_t *data, size_t size);
 
@@ -1871,7 +1890,10 @@  int av_stream_add_side_data(AVStream *st, enum AVPacketSideDataType type,
  * @param size   side information size
  *
  * @return pointer to fresh allocated data or NULL otherwise
+ * @deprecated use av_packet_side_data_new() with the stream's
+ *             @ref AVCodecParameters.side_data "codecpar side data"
  */
+attribute_deprecated
 uint8_t *av_stream_new_side_data(AVStream *stream,
                                  enum AVPacketSideDataType type, size_t size);
 /**
@@ -1883,9 +1905,13 @@  uint8_t *av_stream_new_side_data(AVStream *stream,
  *               or to zero if the desired side data is not present.
  *
  * @return pointer to data if present or NULL otherwise
+ * @deprecated use av_packet_side_data_get() with the stream's
+ *             @ref AVCodecParameters.side_data "codecpar side data"
  */
+attribute_deprecated
 uint8_t *av_stream_get_side_data(const AVStream *stream,
                                  enum AVPacketSideDataType type, size_t *size);
+#endif
 
 AVProgram *av_new_program(AVFormatContext *s, int id);
 
diff --git a/libavformat/concatdec.c b/libavformat/concatdec.c
index 5d4f67d0ac..2aeb4f1dfd 100644
--- a/libavformat/concatdec.c
+++ b/libavformat/concatdec.c
@@ -194,7 +194,6 @@  static int copy_stream_props(AVStream *st, AVStream *source_st)
     avpriv_set_pts_info(st, 64, source_st->time_base.num, source_st->time_base.den);
 
     av_dict_copy(&st->metadata, source_st->metadata, 0);
-    ff_stream_side_data_copy(st, source_st);
     return 0;
 }
 
diff --git a/libavformat/dashdec.c b/libavformat/dashdec.c
index cbe34f4f50..1215407f3c 100644
--- a/libavformat/dashdec.c
+++ b/libavformat/dashdec.c
@@ -1954,17 +1954,6 @@  static int open_demux_for_component(AVFormatContext *s, struct representation *p
 
         // copy disposition
         st->disposition = ist->disposition;
-
-        // copy side data
-        for (int i = 0; i < ist->nb_side_data; i++) {
-            const AVPacketSideData *sd_src = &ist->side_data[i];
-            uint8_t *dst_data;
-
-            dst_data = av_stream_new_side_data(st, sd_src->type, sd_src->size);
-            if (!dst_data)
-                return AVERROR(ENOMEM);
-            memcpy(dst_data, sd_src->data, sd_src->size);
-        }
     }
 
     return 0;
diff --git a/libavformat/demux.c b/libavformat/demux.c
index 6decb08698..e8767e9212 100644
--- a/libavformat/demux.c
+++ b/libavformat/demux.c
@@ -1409,9 +1409,10 @@  FF_ENABLE_DEPRECATION_WARNINGS
             sti->skip_samples = 0;
         }
 
+#if FF_API_AVSTREAM_SIDE_DATA
         if (sti->inject_global_side_data) {
-            for (int i = 0; i < st->nb_side_data; i++) {
-                const AVPacketSideData *const src_sd = &st->side_data[i];
+            for (int i = 0; i < st->codecpar->nb_side_data; i++) {
+                const AVPacketSideData *const src_sd = &st->codecpar->side_data[i];
                 uint8_t *dst_data;
 
                 if (av_packet_get_side_data(pkt, src_sd->type, NULL))
@@ -1427,6 +1428,7 @@  FF_ENABLE_DEPRECATION_WARNINGS
             }
             sti->inject_global_side_data = 0;
         }
+#endif
     }
 
     if (!si->metafree) {
@@ -2431,19 +2433,6 @@  static int extract_extradata(FFFormatContext *si, AVStream *st, const AVPacket *
     return 0;
 }
 
-static int add_coded_side_data(AVStream *st, AVCodecContext *avctx)
-{
-    for (int i = 0; i < avctx->nb_coded_side_data; i++) {
-        const AVPacketSideData *const sd_src = &avctx->coded_side_data[i];
-        uint8_t *dst_data;
-        dst_data = av_stream_new_side_data(st, sd_src->type, sd_src->size);
-        if (!dst_data)
-            return AVERROR(ENOMEM);
-        memcpy(dst_data, sd_src->data, sd_src->size);
-    }
-    return 0;
-}
-
 int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
 {
     FFFormatContext *const si = ffformatcontext(ic);
@@ -2969,9 +2958,6 @@  int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
 
         if (sti->avctx_inited) {
             ret = avcodec_parameters_from_context(st->codecpar, sti->avctx);
-            if (ret < 0)
-                goto find_stream_info_err;
-            ret = add_coded_side_data(st, sti->avctx);
             if (ret < 0)
                 goto find_stream_info_err;
 
@@ -2986,14 +2972,41 @@  int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
                         props->min_bitrate = sti->avctx->rc_min_rate;
                     if (sti->avctx->rc_max_rate > 0)
                         props->max_bitrate = sti->avctx->rc_max_rate;
-                    if (av_stream_add_side_data(st, AV_PKT_DATA_CPB_PROPERTIES,
-                                                (uint8_t *)props, cpb_size))
+                    if (!av_packet_side_data_add(&st->codecpar->side_data,
+                                                 &st->codecpar->nb_side_data,
+                                                 AV_PKT_DATA_CPB_PROPERTIES,
+                                                 (uint8_t *)props, cpb_size, 0))
                         av_free(props);
                 }
             }
         }
 
         sti->avctx_inited = 0;
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
+        if (st->codecpar->nb_side_data > 0) {
+            av_assert0(!st->side_data && !st->nb_side_data);
+            st->side_data = av_calloc(st->codecpar->nb_side_data, sizeof(*st->side_data));
+            if (!st->side_data) {
+                ret = AVERROR(ENOMEM);
+                goto find_stream_info_err;
+            }
+
+            for (int j = 0; j < st->codecpar->nb_side_data; j++) {
+                uint8_t *data = av_memdup(st->codecpar->side_data[j].data,
+                                          st->codecpar->side_data[j].size);
+                if (!data) {
+                    ret = AVERROR(ENOMEM);
+                    goto find_stream_info_err;
+                }
+                st->side_data[j].type = st->codecpar->side_data[j].type;
+                st->side_data[j].size = st->codecpar->side_data[j].size;
+                st->side_data[j].data = data;
+                st->nb_side_data++;
+            }
+        }
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
     }
 
 find_stream_info_err:
diff --git a/libavformat/demux_utils.c b/libavformat/demux_utils.c
index 56cc6e15d8..2946e82295 100644
--- a/libavformat/demux_utils.c
+++ b/libavformat/demux_utils.c
@@ -80,6 +80,8 @@  AVChapter *avpriv_new_chapter(AVFormatContext *s, int64_t id, AVRational time_ba
     return chapter;
 }
 
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
 void av_format_inject_global_side_data(AVFormatContext *s)
 {
     FFFormatContext *const si = ffformatcontext(s);
@@ -89,6 +91,8 @@  void av_format_inject_global_side_data(AVFormatContext *s)
         ffstream(st)->inject_global_side_data = 1;
     }
 }
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
 
 int avformat_queue_attached_pictures(AVFormatContext *s)
 {
diff --git a/libavformat/dovi_isom.c b/libavformat/dovi_isom.c
index c8fdf566e4..53c43649b4 100644
--- a/libavformat/dovi_isom.c
+++ b/libavformat/dovi_isom.c
@@ -34,7 +34,6 @@  int ff_isom_parse_dvcc_dvvc(void *logctx, AVStream *st,
     uint32_t buf;
     AVDOVIDecoderConfigurationRecord *dovi;
     size_t dovi_size;
-    int ret;
 
     if (size > (1 << 30) || size < 4)
         return AVERROR_INVALIDDATA;
@@ -64,11 +63,10 @@  int ff_isom_parse_dvcc_dvvc(void *logctx, AVStream *st,
         dovi->dv_bl_signal_compatibility_id = 0;
     }
 
-    ret = av_stream_add_side_data(st, AV_PKT_DATA_DOVI_CONF,
-                                  (uint8_t *)dovi, dovi_size);
-    if (ret < 0) {
+    if (!av_packet_side_data_add(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                 AV_PKT_DATA_DOVI_CONF, (uint8_t *)dovi, dovi_size, 0)) {
         av_free(dovi);
-        return ret;
+        return AVERROR(ENOMEM);
     }
 
     av_log(logctx, AV_LOG_TRACE, "DOVI in dvcC/dvvC/dvwC box, version: %d.%d, profile: %d, level: %d, "
diff --git a/libavformat/dump.c b/libavformat/dump.c
index d31e4c2ec6..22fc07a655 100644
--- a/libavformat/dump.c
+++ b/libavformat/dump.c
@@ -431,11 +431,11 @@  static void dump_sidedata(void *ctx, const AVStream *st, const char *indent)
 {
     int i;
 
-    if (st->nb_side_data)
+    if (st->codecpar->nb_side_data)
         av_log(ctx, AV_LOG_INFO, "%sSide data:\n", indent);
 
-    for (i = 0; i < st->nb_side_data; i++) {
-        const AVPacketSideData *sd = &st->side_data[i];
+    for (i = 0; i < st->codecpar->nb_side_data; i++) {
+        const AVPacketSideData *sd = &st->codecpar->side_data[i];
         av_log(ctx, AV_LOG_INFO, "%s  ", indent);
 
         switch (sd->type) {
diff --git a/libavformat/hls.c b/libavformat/hls.c
index 332a5a15bb..f5f549b24d 100644
--- a/libavformat/hls.c
+++ b/libavformat/hls.c
@@ -1852,17 +1852,6 @@  static int set_stream_info_from_input_stream(AVStream *st, struct playlist *pls,
 
     av_dict_copy(&st->metadata, ist->metadata, 0);
 
-    // copy side data
-    for (int i = 0; i < ist->nb_side_data; i++) {
-        const AVPacketSideData *sd_src = &ist->side_data[i];
-        uint8_t *dst_data;
-
-        dst_data = av_stream_new_side_data(st, sd_src->type, sd_src->size);
-        if (!dst_data)
-            return AVERROR(ENOMEM);
-        memcpy(dst_data, sd_src->data, sd_src->size);
-    }
-
     ffstream(st)->need_context_update = 1;
 
     return 0;
diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c
index 08f3746ce7..038d08d207 100644
--- a/libavformat/hlsenc.c
+++ b/libavformat/hlsenc.c
@@ -1353,16 +1353,17 @@  static const char* get_relative_url(const char *master_url, const char *media_ur
 
 static int64_t get_stream_bit_rate(AVStream *stream)
 {
-    AVCPBProperties *props = (AVCPBProperties*)av_stream_get_side_data(
-        stream,
-        AV_PKT_DATA_CPB_PROPERTIES,
-        NULL
+    const AVPacketSideData *sd = av_packet_side_data_get(
+        stream->codecpar->side_data, stream->codecpar->nb_side_data,
+        AV_PKT_DATA_CPB_PROPERTIES
     );
 
     if (stream->codecpar->bit_rate)
         return stream->codecpar->bit_rate;
-    else if (props)
+    else if (sd) {
+        AVCPBProperties *props = (AVCPBProperties*)sd->data;
         return props->max_bitrate;
+    }
 
     return 0;
 }
diff --git a/libavformat/internal.h b/libavformat/internal.h
index 901a8b51c6..7702986c9c 100644
--- a/libavformat/internal.h
+++ b/libavformat/internal.h
@@ -147,7 +147,9 @@  typedef struct FFFormatContext {
     int missing_ts_warning;
 #endif
 
+#if FF_API_AVSTREAM_SIDE_DATA
     int inject_global_side_data;
+#endif
 
     int avoid_negative_ts_use_pts;
 
@@ -356,10 +358,12 @@  typedef struct FFStream {
     uint8_t dts_ordered;
     uint8_t dts_misordered;
 
+#if FF_API_AVSTREAM_SIDE_DATA
     /**
      * Internal data to inject global side data
      */
     int inject_global_side_data;
+#endif
 
     /**
      * display aspect ratio (0 if unknown)
diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c
index 941c0bcdc9..e1ae71e758 100644
--- a/libavformat/matroskadec.c
+++ b/libavformat/matroskadec.c
@@ -2249,7 +2249,6 @@  static int mkv_stereo3d_conv(AVStream *st, MatroskaVideoStereoModeType stereo_mo
         STEREOMODE_STEREO3D_MAPPING(STEREO_MODE_CONV, NOTHING)
     };
     AVStereo3D *stereo;
-    int ret;
 
     stereo = av_stereo3d_alloc();
     if (!stereo)
@@ -2258,11 +2257,10 @@  static int mkv_stereo3d_conv(AVStream *st, MatroskaVideoStereoModeType stereo_mo
     stereo->type  = stereo_mode_conv[stereo_mode].type;
     stereo->flags = stereo_mode_conv[stereo_mode].flags;
 
-    ret = av_stream_add_side_data(st, AV_PKT_DATA_STEREO3D, (uint8_t *)stereo,
-                                  sizeof(*stereo));
-    if (ret < 0) {
+    if (!av_packet_side_data_add(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                 AV_PKT_DATA_STEREO3D, stereo, sizeof(*stereo), 0)) {
         av_freep(&stereo);
-        return ret;
+        return AVERROR(ENOMEM);
     }
 
     return 0;
@@ -2309,28 +2307,27 @@  static int mkv_parse_video_color(AVStream *st, const MatroskaTrack *track) {
     }
     if (color->max_cll && color->max_fall) {
         size_t size = 0;
-        int ret;
         AVContentLightMetadata *metadata = av_content_light_metadata_alloc(&size);
         if (!metadata)
             return AVERROR(ENOMEM);
-        ret = av_stream_add_side_data(st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
-                                      (uint8_t *)metadata, size);
-        if (ret < 0) {
+        if (!av_packet_side_data_add(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                     AV_PKT_DATA_CONTENT_LIGHT_LEVEL, metadata, size, 0)) {
             av_freep(&metadata);
-            return ret;
+            return AVERROR(ENOMEM);
         }
         metadata->MaxCLL  = color->max_cll;
         metadata->MaxFALL = color->max_fall;
     }
 
     if (has_mastering_primaries || has_mastering_luminance) {
-        AVMasteringDisplayMetadata *metadata =
-            (AVMasteringDisplayMetadata*) av_stream_new_side_data(
-                st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
-                sizeof(AVMasteringDisplayMetadata));
-        if (!metadata) {
+        AVMasteringDisplayMetadata *metadata;
+        AVPacketSideData *sd = av_packet_side_data_new(&st->codecpar->side_data,
+                                                       &st->codecpar->nb_side_data,
+                                                       AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
+                                                       sizeof(AVMasteringDisplayMetadata), 0);
+        if (!sd)
             return AVERROR(ENOMEM);
-        }
+        metadata = (AVMasteringDisplayMetadata*)sd->data;
         memset(metadata, 0, sizeof(AVMasteringDisplayMetadata));
         if (has_mastering_primaries) {
             metadata->display_primaries[0][0] = av_d2q(mastering_meta->r_x, INT_MAX);
@@ -2356,6 +2353,7 @@  static int mkv_create_display_matrix(AVStream *st,
                                      const MatroskaTrackVideoProjection *proj,
                                      void *logctx)
 {
+    AVPacketSideData *sd;
     double pitch = proj->pitch, yaw = proj->yaw, roll = proj->roll;
     int32_t *matrix;
     int hflip;
@@ -2372,10 +2370,13 @@  static int mkv_create_display_matrix(AVStream *st,
                st->index, yaw, pitch, roll);
         return 0;
     }
-    matrix = (int32_t*)av_stream_new_side_data(st, AV_PKT_DATA_DISPLAYMATRIX,
-                                               9 * sizeof(*matrix));
-    if (!matrix)
+    sd = av_packet_side_data_new(&st->codecpar->side_data,
+                                 &st->codecpar->nb_side_data,
+                                 AV_PKT_DATA_DISPLAYMATRIX,
+                                 9 * sizeof(*matrix), 0);
+    if (!sd)
         return AVERROR(ENOMEM);
+    matrix = (int32_t*)sd->data;
 
     hflip = yaw != 0.0;
     /* ProjectionPoseRoll is in the counter-clockwise direction
@@ -2400,7 +2401,6 @@  static int mkv_parse_video_projection(AVStream *st, const MatroskaTrack *track,
     size_t spherical_size;
     uint32_t l = 0, t = 0, r = 0, b = 0;
     uint32_t padding = 0;
-    int ret;
 
     if (mkv_projection->private.size && priv_data[0] != 0) {
         av_log(logctx, AV_LOG_WARNING, "Unknown spherical metadata\n");
@@ -2476,11 +2476,10 @@  static int mkv_parse_video_projection(AVStream *st, const MatroskaTrack *track,
     spherical->bound_right  = r;
     spherical->bound_bottom = b;
 
-    ret = av_stream_add_side_data(st, AV_PKT_DATA_SPHERICAL, (uint8_t *)spherical,
-                                  spherical_size);
-    if (ret < 0) {
+    if (!av_packet_side_data_add(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                 AV_PKT_DATA_SPHERICAL, spherical, spherical_size, 0)) {
         av_freep(&spherical);
-        return ret;
+        return AVERROR(ENOMEM);
     }
 
     return 0;
diff --git a/libavformat/matroskaenc.c b/libavformat/matroskaenc.c
index ba54f5f98e..82fbd38f79 100644
--- a/libavformat/matroskaenc.c
+++ b/libavformat/matroskaenc.c
@@ -1327,7 +1327,7 @@  fail:
 static void mkv_write_video_color(EbmlWriter *writer, const AVStream *st,
                                   const AVCodecParameters *par)
 {
-    const void *side_data;
+    const AVPacketSideData *side_data;
 
     ebml_writer_open_master(writer, MATROSKA_ID_VIDEOCOLOR);
 
@@ -1361,20 +1361,20 @@  static void mkv_write_video_color(EbmlWriter *writer, const AVStream *st,
                              (ypos >> 7) + 1);
     }
 
-    side_data = av_stream_get_side_data(st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
-                                        NULL);
+    side_data = av_packet_side_data_get(st->codecpar->side_data, st->codecpar->nb_side_data,
+                                        AV_PKT_DATA_CONTENT_LIGHT_LEVEL);
     if (side_data) {
-        const AVContentLightMetadata *metadata = side_data;
+        const AVContentLightMetadata *metadata = (AVContentLightMetadata *)side_data->data;
         ebml_writer_add_uint(writer, MATROSKA_ID_VIDEOCOLORMAXCLL,
                              metadata->MaxCLL);
         ebml_writer_add_uint(writer, MATROSKA_ID_VIDEOCOLORMAXFALL,
                              metadata->MaxFALL);
     }
 
-    side_data = av_stream_get_side_data(st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
-                                        NULL);
+    side_data = av_packet_side_data_get(st->codecpar->side_data, st->codecpar->nb_side_data,
+                                        AV_PKT_DATA_MASTERING_DISPLAY_METADATA);
     if (side_data) {
-        const AVMasteringDisplayMetadata *metadata = side_data;
+        const AVMasteringDisplayMetadata *metadata = (AVMasteringDisplayMetadata *)side_data->data;
         ebml_writer_open_master(writer, MATROSKA_ID_VIDEOCOLORMASTERINGMETA);
         if (metadata->has_primaries) {
             ebml_writer_add_float(writer, MATROSKA_ID_VIDEOCOLOR_RX,
@@ -1410,12 +1410,16 @@  static void mkv_write_video_color(EbmlWriter *writer, const AVStream *st,
 static void mkv_handle_rotation(void *logctx, const AVStream *st,
                                 double *yaw, double *roll)
 {
-    const int32_t *matrix =
-        (const int32_t*)av_stream_get_side_data(st, AV_PKT_DATA_DISPLAYMATRIX, NULL);
+    const int32_t *matrix;
+    const AVPacketSideData *side_data =
+        av_packet_side_data_get(st->codecpar->side_data, st->codecpar->nb_side_data,
+                                AV_PKT_DATA_DISPLAYMATRIX);
 
-    if (!matrix)
+    if (!side_data)
         return;
 
+    matrix = (int32_t *)side_data->data;
+
     /* Check whether this is an affine transformation */
     if (matrix[2] || matrix[5])
         goto ignore;
@@ -1462,13 +1466,15 @@  static int mkv_handle_spherical(void *logctx, EbmlWriter *writer,
                                 const AVStream *st, uint8_t private[],
                                 double *yaw, double *pitch, double *roll)
 {
-    const AVSphericalMapping *spherical =
-        (const AVSphericalMapping *)av_stream_get_side_data(st, AV_PKT_DATA_SPHERICAL,
-                                                            NULL);
+    const AVPacketSideData *sd = av_packet_side_data_get(st->codecpar->side_data,
+                                                         st->codecpar->nb_side_data,
+                                                         AV_PKT_DATA_SPHERICAL);
+    const AVSphericalMapping *spherical;
 
-    if (!spherical)
+    if (!sd)
         return 0;
 
+    spherical = (const AVSphericalMapping *)sd->data;
     if (spherical->projection != AV_SPHERICAL_EQUIRECTANGULAR      &&
         spherical->projection != AV_SPHERICAL_EQUIRECTANGULAR_TILE &&
         spherical->projection != AV_SPHERICAL_CUBEMAP) {
@@ -1628,6 +1634,7 @@  static int mkv_write_stereo_mode(AVFormatContext *s, EbmlWriter *writer,
             format = stereo_mode;
         }
     } else {
+        const AVPacketSideData *sd;
         const AVStereo3D *stereo;
         /* The following macro presumes all MATROSKA_VIDEO_STEREOMODE_TYPE_*
          * values to be in the range 0..254. */
@@ -1639,11 +1646,13 @@  static int mkv_write_stereo_mode(AVFormatContext *s, EbmlWriter *writer,
         };
         int fmt;
 
-        stereo = (const AVStereo3D*)av_stream_get_side_data(st, AV_PKT_DATA_STEREO3D,
-                                                            NULL);
-        if (!stereo)
+        sd = av_packet_side_data_get(st->codecpar->side_data, st->codecpar->nb_side_data,
+                                     AV_PKT_DATA_STEREO3D);
+        if (!sd)
             return 0;
 
+        stereo = (const AVStereo3D*)sd->data;
+
         /* A garbage AVStereo3D or something with no Matroska analogon. */
         if ((unsigned)stereo->type >= FF_ARRAY_ELEMS(conversion_table))
             return 0;
@@ -1681,6 +1690,7 @@  static void mkv_write_blockadditionmapping(AVFormatContext *s, const MatroskaMux
 {
 #if CONFIG_MATROSKA_MUXER
     const AVDOVIDecoderConfigurationRecord *dovi;
+    const AVPacketSideData *sd;
 
     if (IS_SEEKABLE(s->pb, mkv)) {
         track->blockadditionmapping_offset = avio_tell(pb);
@@ -1697,9 +1707,14 @@  static void mkv_write_blockadditionmapping(AVFormatContext *s, const MatroskaMux
         }
     }
 
-    dovi = (const AVDOVIDecoderConfigurationRecord *)
-           av_stream_get_side_data(st, AV_PKT_DATA_DOVI_CONF, NULL);
-    if (dovi && dovi->dv_profile <= 10) {
+    sd = av_packet_side_data_get(st->codecpar->side_data, st->codecpar->nb_side_data,
+                                 AV_PKT_DATA_DOVI_CONF);
+
+    if (!sd)
+        return;
+
+    dovi = (const AVDOVIDecoderConfigurationRecord *)sd->data;
+    if (dovi->dv_profile <= 10) {
         ebml_master mapping;
         uint8_t buf[ISOM_DVCC_DVVC_SIZE];
         uint32_t type;
diff --git a/libavformat/mov.c b/libavformat/mov.c
index 1996e0028c..db528d9907 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -799,6 +799,7 @@  static int mov_read_esds(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 static int mov_read_dac3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 {
     AVStream *st;
+    AVPacketSideData *sd;
     enum AVAudioServiceType *ast;
     int ac3info, acmod, lfeon, bsmod;
     uint64_t mask;
@@ -807,11 +808,14 @@  static int mov_read_dac3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
         return 0;
     st = c->fc->streams[c->fc->nb_streams-1];
 
-    ast = (enum AVAudioServiceType*)av_stream_new_side_data(st, AV_PKT_DATA_AUDIO_SERVICE_TYPE,
-                                                            sizeof(*ast));
-    if (!ast)
+    sd = av_packet_side_data_new(&st->codecpar->side_data,
+                                 &st->codecpar->nb_side_data,
+                                     AV_PKT_DATA_AUDIO_SERVICE_TYPE,
+                                     sizeof(*ast), 0);
+    if (!sd)
         return AVERROR(ENOMEM);
 
+    ast = (enum AVAudioServiceType*)sd->data;
     ac3info = avio_rb24(pb);
     bsmod = (ac3info >> 14) & 0x7;
     acmod = (ac3info >> 11) & 0x7;
@@ -833,6 +837,7 @@  static int mov_read_dac3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 static int mov_read_dec3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 {
     AVStream *st;
+    AVPacketSideData *sd;
     enum AVAudioServiceType *ast;
     int eac3info, acmod, lfeon, bsmod;
     uint64_t mask;
@@ -841,11 +846,15 @@  static int mov_read_dec3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
         return 0;
     st = c->fc->streams[c->fc->nb_streams-1];
 
-    ast = (enum AVAudioServiceType*)av_stream_new_side_data(st, AV_PKT_DATA_AUDIO_SERVICE_TYPE,
-                                                            sizeof(*ast));
-    if (!ast)
+    sd = av_packet_side_data_new(&st->codecpar->side_data,
+                                 &st->codecpar->nb_side_data,
+                                     AV_PKT_DATA_AUDIO_SERVICE_TYPE,
+                                     sizeof(*ast), 0);
+    if (!sd)
         return AVERROR(ENOMEM);
 
+    ast = (enum AVAudioServiceType*)sd->data;
+
     /* No need to parse fields for additional independent substreams and its
      * associated dependent substreams since libavcodec's E-AC-3 decoder
      * does not support them yet. */
@@ -1747,7 +1756,6 @@  static int mov_read_pcmc(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 static int mov_read_colr(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 {
     AVStream *st;
-    uint8_t *icc_profile;
     char color_parameter_type[5] = { 0 };
     uint16_t color_primaries, color_trc, color_matrix;
     int ret;
@@ -1768,10 +1776,13 @@  static int mov_read_colr(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     }
 
     if (!strncmp(color_parameter_type, "prof", 4)) {
-        icc_profile = av_stream_new_side_data(st, AV_PKT_DATA_ICC_PROFILE, atom.size - 4);
-        if (!icc_profile)
+        AVPacketSideData *sd = av_packet_side_data_new(&st->codecpar->side_data,
+                                                       &st->codecpar->nb_side_data,
+                                                           AV_PKT_DATA_ICC_PROFILE,
+                                                           atom.size - 4, 0);
+        if (!sd)
             return AVERROR(ENOMEM);
-        ret = ffio_read_size(pb, icc_profile, atom.size - 4);
+        ret = ffio_read_size(pb, sd->data, atom.size - 4);
         if (ret < 0)
             return ret;
     } else {
@@ -6841,8 +6852,9 @@  static int mov_read_pssh(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     AVEncryptionInitInfo *info, *old_init_info;
     uint8_t **key_ids;
     AVStream *st;
-    uint8_t *side_data, *extra_data, *old_side_data;
-    size_t side_data_size, old_side_data_size;
+    const AVPacketSideData *old_side_data;
+    uint8_t *side_data, *extra_data;
+    size_t side_data_size;
     int ret = 0;
     unsigned int version, kid_count, extra_data_size, alloc_size = 0;
 
@@ -6920,9 +6932,10 @@  static int mov_read_pssh(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     info->data_size = extra_data_size;
 
     // If there is existing initialization data, append to the list.
-    old_side_data = av_stream_get_side_data(st, AV_PKT_DATA_ENCRYPTION_INIT_INFO, &old_side_data_size);
+    old_side_data = av_packet_side_data_get(st->codecpar->side_data, st->codecpar->nb_side_data,
+                                            AV_PKT_DATA_ENCRYPTION_INIT_INFO);
     if (old_side_data) {
-        old_init_info = av_encryption_init_info_get_side_data(old_side_data, old_side_data_size);
+        old_init_info = av_encryption_init_info_get_side_data(old_side_data->data, old_side_data->size);
         if (old_init_info) {
             // Append to the end of the list.
             for (AVEncryptionInitInfo *cur = old_init_info;; cur = cur->next) {
@@ -6944,9 +6957,9 @@  static int mov_read_pssh(MOVContext *c, AVIOContext *pb, MOVAtom atom)
         ret = AVERROR(ENOMEM);
         goto finish;
     }
-    ret = av_stream_add_side_data(st, AV_PKT_DATA_ENCRYPTION_INIT_INFO,
-                                  side_data, side_data_size);
-    if (ret < 0)
+    if (!av_packet_side_data_add(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                 AV_PKT_DATA_ENCRYPTION_INIT_INFO,
+                                     side_data, side_data_size, 0))
         av_free(side_data);
 
 finish:
@@ -8725,46 +8738,42 @@  static int mov_read_header(AVFormatContext *s)
             break;
         case AVMEDIA_TYPE_VIDEO:
             if (sc->display_matrix) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_DISPLAYMATRIX, (uint8_t*)sc->display_matrix,
-                                              sizeof(int32_t) * 9);
-                if (err < 0)
-                    return err;
+                if (!av_packet_side_data_add(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                             AV_PKT_DATA_DISPLAYMATRIX,
+                                                 (uint8_t*)sc->display_matrix, sizeof(int32_t) * 9, 0))
+                    return AVERROR(ENOMEM);
 
                 sc->display_matrix = NULL;
             }
             if (sc->stereo3d) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_STEREO3D,
-                                              (uint8_t *)sc->stereo3d,
-                                              sizeof(*sc->stereo3d));
-                if (err < 0)
-                    return err;
+                if (!av_packet_side_data_add(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                             AV_PKT_DATA_STEREO3D,
+                                                 (uint8_t *)sc->stereo3d, sizeof(*sc->stereo3d), 0))
+                    return AVERROR(ENOMEM);
 
                 sc->stereo3d = NULL;
             }
             if (sc->spherical) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_SPHERICAL,
-                                              (uint8_t *)sc->spherical,
-                                              sc->spherical_size);
-                if (err < 0)
-                    return err;
+                if (!av_packet_side_data_add(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                             AV_PKT_DATA_SPHERICAL,
+                                                 (uint8_t *)sc->spherical, sc->spherical_size, 0))
+                    return AVERROR(ENOMEM);
 
                 sc->spherical = NULL;
             }
             if (sc->mastering) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
-                                              (uint8_t *)sc->mastering,
-                                              sizeof(*sc->mastering));
-                if (err < 0)
-                    return err;
+                if (!av_packet_side_data_add(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                             AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
+                                                 (uint8_t *)sc->mastering, sizeof(*sc->mastering), 0))
+                    return AVERROR(ENOMEM);
 
                 sc->mastering = NULL;
             }
             if (sc->coll) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
-                                              (uint8_t *)sc->coll,
-                                              sc->coll_size);
-                if (err < 0)
-                    return err;
+                if (!av_packet_side_data_add(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                             AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
+                                                 (uint8_t *)sc->coll, sc->coll_size, 0))
+                    return AVERROR(ENOMEM);
 
                 sc->coll = NULL;
             }
diff --git a/libavformat/movenc.c b/libavformat/movenc.c
index 7e96e7435f..7100ae3b59 100644
--- a/libavformat/movenc.c
+++ b/libavformat/movenc.c
@@ -672,11 +672,12 @@  struct mpeg4_bit_rate_values {
 
 static struct mpeg4_bit_rate_values calculate_mpeg4_bit_rates(MOVTrack *track)
 {
-    AVCPBProperties *props = track->st ?
-        (AVCPBProperties*)av_stream_get_side_data(track->st,
-                                                  AV_PKT_DATA_CPB_PROPERTIES,
-                                                  NULL) :
+    const AVPacketSideData *sd = track->st ?
+        av_packet_side_data_get(track->st->codecpar->side_data,
+                                track->st->codecpar->nb_side_data,
+                                         AV_PKT_DATA_CPB_PROPERTIES) :
         NULL;
+    AVCPBProperties *props = sd ? (AVCPBProperties *)sd->data : NULL;
     struct mpeg4_bit_rate_values bit_rates = { 0 };
 
     bit_rates.avg_bit_rate = compute_avg_bitrate(track);
@@ -2129,18 +2130,17 @@  static int mov_write_colr_tag(AVIOContext *pb, MOVTrack *track, int prefer_icc)
     // Ref (MOV): https://developer.apple.com/library/mac/technotes/tn2162/_index.html#//apple_ref/doc/uid/DTS40013070-CH1-TNTAG9
     // Ref (MP4): ISO/IEC 14496-12:2012
 
-    const uint8_t *icc_profile;
-    size_t icc_profile_size;
-
     if (prefer_icc) {
-        icc_profile = av_stream_get_side_data(track->st, AV_PKT_DATA_ICC_PROFILE, &icc_profile_size);
+        AVPacketSideData *sd = av_packet_side_data_get(track->st->codecpar->side_data,
+                                                       track->st->codecpar->nb_side_data,
+                                                                AV_PKT_DATA_ICC_PROFILE);
 
-        if (icc_profile) {
-            avio_wb32(pb, 12 + icc_profile_size);
+        if (sd) {
+            avio_wb32(pb, 12 + sd->size);
             ffio_wfourcc(pb, "colr");
             ffio_wfourcc(pb, "prof");
-            avio_write(pb, icc_profile, icc_profile_size);
-            return 12 + icc_profile_size;
+            avio_write(pb, sd->data, sd->size);
+            return 12 + sd->size;
         }
         else {
             av_log(NULL, AV_LOG_INFO, "no ICC profile found, will write nclx/nclc colour info instead\n");
@@ -2173,14 +2173,15 @@  static int mov_write_colr_tag(AVIOContext *pb, MOVTrack *track, int prefer_icc)
 
 static int mov_write_clli_tag(AVIOContext *pb, MOVTrack *track)
 {
-    const uint8_t *side_data;
+    const AVPacketSideData *side_data;
     const AVContentLightMetadata *content_light_metadata;
 
-    side_data = av_stream_get_side_data(track->st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL, NULL);
+    side_data = av_packet_side_data_get(track->st->codecpar->side_data, track->st->codecpar->nb_side_data,
+                                        AV_PKT_DATA_CONTENT_LIGHT_LEVEL);
     if (!side_data) {
         return 0;
     }
-    content_light_metadata = (const AVContentLightMetadata*)side_data;
+    content_light_metadata = (const AVContentLightMetadata*)side_data->data;
 
     avio_wb32(pb, 12); // size
     ffio_wfourcc(pb, "clli");
@@ -2198,11 +2199,13 @@  static int mov_write_mdcv_tag(AVIOContext *pb, MOVTrack *track)
 {
     const int chroma_den = 50000;
     const int luma_den = 10000;
-    const uint8_t *side_data;
-    const AVMasteringDisplayMetadata *metadata;
+    const AVPacketSideData *side_data;
+    const AVMasteringDisplayMetadata *metadata = NULL;
 
-    side_data = av_stream_get_side_data(track->st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA, NULL);
-    metadata = (const AVMasteringDisplayMetadata*)side_data;
+    side_data = av_packet_side_data_get(track->st->codecpar->side_data, track->st->codecpar->nb_side_data,
+                                        AV_PKT_DATA_MASTERING_DISPLAY_METADATA);
+    if (side_data)
+        metadata = (const AVMasteringDisplayMetadata*)side_data->data;
     if (!metadata || !metadata->has_primaries || !metadata->has_luminance) {
         return 0;
     }
@@ -2421,7 +2424,8 @@  static int mov_write_video_tag(AVFormatContext *s, AVIOContext *pb, MOVMuxContex
                              track->par->color_trc != AVCOL_TRC_UNSPECIFIED &&
                              track->par->color_space != AVCOL_SPC_UNSPECIFIED;
         if (has_color_info || mov->flags & FF_MOV_FLAG_WRITE_COLR ||
-            av_stream_get_side_data(track->st, AV_PKT_DATA_ICC_PROFILE, NULL)) {
+            av_packet_side_data_get(track->st->codecpar->side_data,  track->st->codecpar->nb_side_data,
+                                    AV_PKT_DATA_ICC_PROFILE)) {
             int prefer_icc = mov->flags & FF_MOV_FLAG_PREFER_ICC || !has_color_info;
             mov_write_colr_tag(pb, track, prefer_icc);
         }
@@ -2435,17 +2439,19 @@  static int mov_write_video_tag(AVFormatContext *s, AVIOContext *pb, MOVMuxContex
     }
 
     if (track->mode == MODE_MP4 && mov->fc->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL) {
-        AVStereo3D* stereo_3d = (AVStereo3D*) av_stream_get_side_data(track->st, AV_PKT_DATA_STEREO3D, NULL);
-        AVSphericalMapping* spherical_mapping = (AVSphericalMapping*)av_stream_get_side_data(track->st, AV_PKT_DATA_SPHERICAL, NULL);
-        AVDOVIDecoderConfigurationRecord *dovi = (AVDOVIDecoderConfigurationRecord *)
-                                                 av_stream_get_side_data(track->st, AV_PKT_DATA_DOVI_CONF, NULL);
+        const AVPacketSideData *stereo_3d = av_packet_side_data_get(track->st->codecpar->side_data, track->st->codecpar->nb_side_data,
+                                                                    AV_PKT_DATA_STEREO3D);
+        const AVPacketSideData *spherical_mapping = av_packet_side_data_get(track->st->codecpar->side_data, track->st->codecpar->nb_side_data,
+                                                                            AV_PKT_DATA_SPHERICAL);
+        const AVPacketSideData *dovi = av_packet_side_data_get(track->st->codecpar->side_data, track->st->codecpar->nb_side_data,
+                                                               AV_PKT_DATA_DOVI_CONF);
 
         if (stereo_3d)
-            mov_write_st3d_tag(s, pb, stereo_3d);
+            mov_write_st3d_tag(s, pb, (AVStereo3D*)stereo_3d->data);
         if (spherical_mapping)
-            mov_write_sv3d_tag(mov->fc, pb, spherical_mapping);
+            mov_write_sv3d_tag(mov->fc, pb, (AVSphericalMapping*)spherical_mapping->data);
         if (dovi)
-            mov_write_dvcc_dvvc_tag(s, pb, dovi);
+            mov_write_dvcc_dvvc_tag(s, pb, (AVDOVIDecoderConfigurationRecord *)dovi->data);
     }
 
     if (track->par->sample_aspect_ratio.den && track->par->sample_aspect_ratio.num) {
@@ -3392,7 +3398,6 @@  static int mov_write_tkhd_tag(AVIOContext *pb, MOVMuxContext *mov,
     int group   = 0;
 
     uint32_t *display_matrix = NULL;
-    size_t display_matrix_size;
     int       i;
 
     if (mov->mode == MODE_AVIF)
@@ -3402,15 +3407,16 @@  static int mov_write_tkhd_tag(AVIOContext *pb, MOVMuxContext *mov,
             duration *= mov->avif_loop_count;
 
     if (st) {
+        const AVPacketSideData *sd;
         if (mov->per_stream_grouping)
             group = st->index;
         else
             group = st->codecpar->codec_type;
 
-        display_matrix = (uint32_t*)av_stream_get_side_data(st, AV_PKT_DATA_DISPLAYMATRIX,
-                                                            &display_matrix_size);
-        if (display_matrix && display_matrix_size < 9 * sizeof(*display_matrix))
-            display_matrix = NULL;
+        sd = av_packet_side_data_get(st->codecpar->side_data, st->codecpar->nb_side_data,
+                                     AV_PKT_DATA_DISPLAYMATRIX);
+        if (sd && sd->size == 9 * sizeof(*display_matrix))
+            display_matrix = (uint32_t *)sd->data;
     }
 
     if (track->flags & MOV_TRACK_ENABLED)
@@ -4608,12 +4614,11 @@  static int mov_write_moov_tag(AVIOContext *pb, MOVMuxContext *mov,
             track->tref_tag = MKTAG('h','i','n','t');
             track->tref_id = mov->tracks[track->src_track].track_id;
         } else if (track->par->codec_type == AVMEDIA_TYPE_AUDIO) {
-            size_t size;
-            int *fallback;
-            fallback = (int*)av_stream_get_side_data(track->st,
-                                                     AV_PKT_DATA_FALLBACK_TRACK,
-                                                     &size);
-            if (fallback != NULL && size == sizeof(int)) {
+            const AVPacketSideData *sd = av_packet_side_data_get(track->st->codecpar->side_data,
+                                                                 track->st->codecpar->nb_side_data,
+                                                                    AV_PKT_DATA_FALLBACK_TRACK );
+            if (sd && sd->size == sizeof(int)) {
+                int *fallback = (int *)sd->data;
                 if (*fallback >= 0 && *fallback < mov->nb_streams) {
                     track->tref_tag = MKTAG('f','a','l','l');
                     track->tref_id = mov->tracks[*fallback].track_id;
@@ -5446,7 +5451,8 @@  static int mov_write_ftyp_tag(AVIOContext *pb, AVFormatContext *s)
         if (st->codecpar->codec_id == AV_CODEC_ID_AC3 ||
             st->codecpar->codec_id == AV_CODEC_ID_EAC3 ||
             st->codecpar->codec_id == AV_CODEC_ID_TRUEHD ||
-            av_stream_get_side_data(st, AV_PKT_DATA_DOVI_CONF, NULL))
+            av_packet_side_data_get(st->codecpar->side_data, st->codecpar->nb_side_data,
+                                    AV_PKT_DATA_DOVI_CONF))
             has_dolby = 1;
     }
 
diff --git a/libavformat/mp3enc.c b/libavformat/mp3enc.c
index 5e81f72a59..ece431e67d 100644
--- a/libavformat/mp3enc.c
+++ b/libavformat/mp3enc.c
@@ -400,10 +400,10 @@  static int mp3_queue_flush(AVFormatContext *s)
 static void mp3_update_xing(AVFormatContext *s)
 {
     MP3Context  *mp3 = s->priv_data;
+    const AVPacketSideData *sd;
     AVReplayGain *rg;
     uint16_t tag_crc;
     uint8_t *toc;
-    size_t rg_size;
     int i;
     int64_t old_pos = avio_tell(s->pb);
 
@@ -423,11 +423,13 @@  static void mp3_update_xing(AVFormatContext *s)
     }
 
     /* write replaygain */
-    rg = (AVReplayGain*)av_stream_get_side_data(s->streams[0], AV_PKT_DATA_REPLAYGAIN,
-                                                &rg_size);
-    if (rg && rg_size >= sizeof(*rg)) {
+    sd = av_packet_side_data_get(s->streams[0]->codecpar->side_data,
+                                 s->streams[0]->codecpar->nb_side_data,
+                                 AV_PKT_DATA_REPLAYGAIN);
+    if (sd && sd->size >= sizeof(*rg)) {
         uint16_t val;
 
+        rg = (AVReplayGain *)sd->data;
         AV_WB32(mp3->xing_frame + mp3->xing_offset + 131,
                 av_rescale(rg->track_peak, 1 << 23, 100000));
 
diff --git a/libavformat/mpegenc.c b/libavformat/mpegenc.c
index c06e308296..b6f5f1026d 100644
--- a/libavformat/mpegenc.c
+++ b/libavformat/mpegenc.c
@@ -342,8 +342,6 @@  static av_cold int mpeg_mux_init(AVFormatContext *ctx)
     lpcm_id = LPCM_ID;
 
     for (i = 0; i < ctx->nb_streams; i++) {
-        AVCPBProperties *props;
-
         st     = ctx->streams[i];
         stream = av_mallocz(sizeof(StreamInfo));
         if (!stream)
@@ -430,13 +428,18 @@  static av_cold int mpeg_mux_init(AVFormatContext *ctx)
             stream->max_buffer_size = 4 * 1024;
             s->audio_bound++;
             break;
-        case AVMEDIA_TYPE_VIDEO:
+        case AVMEDIA_TYPE_VIDEO: {
+            const AVPacketSideData *sd;
+            AVCPBProperties *props = NULL;
             if (st->codecpar->codec_id == AV_CODEC_ID_H264)
                 stream->id = h264_id++;
             else
                 stream->id = mpv_id++;
 
-            props = (AVCPBProperties*)av_stream_get_side_data(st, AV_PKT_DATA_CPB_PROPERTIES, NULL);
+            sd = av_packet_side_data_get(st->codecpar->side_data, st->codecpar->nb_side_data,
+                                         AV_PKT_DATA_CPB_PROPERTIES);
+            if (sd)
+                props = (AVCPBProperties*)sd->data;
             if (props && props->buffer_size)
                 stream->max_buffer_size = 6 * 1024 + props->buffer_size / 8;
             else {
@@ -453,6 +456,7 @@  static av_cold int mpeg_mux_init(AVFormatContext *ctx)
             }
             s->video_bound++;
             break;
+        }
         case AVMEDIA_TYPE_SUBTITLE:
             stream->id              = mps_id++;
             stream->max_buffer_size = 16 * 1024;
@@ -470,12 +474,16 @@  static av_cold int mpeg_mux_init(AVFormatContext *ctx)
     audio_bitrate = 0;
     video_bitrate = 0;
     for (i = 0; i < ctx->nb_streams; i++) {
-        AVCPBProperties *props;
+        const AVPacketSideData *sd;
+        AVCPBProperties *props = NULL;
         int codec_rate;
         st     = ctx->streams[i];
         stream = (StreamInfo *)st->priv_data;
 
-        props = (AVCPBProperties*)av_stream_get_side_data(st, AV_PKT_DATA_CPB_PROPERTIES, NULL);
+        sd = av_packet_side_data_get(st->codecpar->side_data, st->codecpar->nb_side_data,
+                                     AV_PKT_DATA_CPB_PROPERTIES);
+        if (sd)
+            props = (AVCPBProperties*)sd->data;
         if (props)
             codec_rate = props->max_bitrate;
         else
diff --git a/libavformat/mpegts.c b/libavformat/mpegts.c
index d825bf0b5d..3460a3b312 100644
--- a/libavformat/mpegts.c
+++ b/libavformat/mpegts.c
@@ -2187,7 +2187,6 @@  int ff_parse_mpeg2_descriptor(AVFormatContext *fc, AVStream *st, int stream_type
             uint32_t buf;
             AVDOVIDecoderConfigurationRecord *dovi;
             size_t dovi_size;
-            int ret;
             int dependency_pid;
 
             if (desc_end - *pp < 4) // (8 + 8 + 7 + 6 + 1 + 1 + 1) / 8
@@ -2218,11 +2217,11 @@  int ff_parse_mpeg2_descriptor(AVFormatContext *fc, AVStream *st, int stream_type
                 dovi->dv_bl_signal_compatibility_id = 0;
             }
 
-            ret = av_stream_add_side_data(st, AV_PKT_DATA_DOVI_CONF,
-                                          (uint8_t *)dovi, dovi_size);
-            if (ret < 0) {
+            if (!av_packet_side_data_add(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                         AV_PKT_DATA_DOVI_CONF,
+                                             (uint8_t *)dovi, dovi_size, 0)) {
                 av_free(dovi);
-                return ret;
+                return AVERROR(ENOMEM);
             }
 
             av_log(fc, AV_LOG_TRACE, "DOVI, version: %d.%d, profile: %d, level: %d, "
diff --git a/libavformat/mux.c b/libavformat/mux.c
index d3779202f0..43bade97ec 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -279,6 +279,26 @@  FF_ENABLE_DEPRECATION_WARNINGS
             break;
         }
 
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
+        /* if the caller is using the deprecated AVStream side_data API,
+         * copy its contents to AVStream.codecpar, giving it priority
+           over existing side data in the latter */
+        for (int i = 0; i < st->nb_side_data; i++) {
+            const AVPacketSideData *sd_src = &st->side_data[i];
+            AVPacketSideData *sd_dst;
+
+            sd_dst = av_packet_side_data_new(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                             sd_src->type, sd_src->size, 0);
+            if (!sd_dst) {
+                ret = AVERROR(ENOMEM);
+                goto fail;
+            }
+            memcpy(sd_dst->data, sd_src->data, sd_src->size);
+        }
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
+
         desc = avcodec_descriptor_get(par->codec_id);
         if (desc && desc->props & AV_CODEC_PROP_REORDER)
             sti->reorder = 1;
diff --git a/libavformat/mxfdec.c b/libavformat/mxfdec.c
index 4846c5d206..ce41a98536 100644
--- a/libavformat/mxfdec.c
+++ b/libavformat/mxfdec.c
@@ -2579,10 +2579,12 @@  static int parse_mca_labels(MXFContext *mxf, MXFTrack *source_track, MXFDescript
 
     if (service_type != AV_AUDIO_SERVICE_TYPE_NB && service_type != AV_AUDIO_SERVICE_TYPE_MAIN && !ambigous_service_type) {
         enum AVAudioServiceType *ast;
-        uint8_t* side_data = av_stream_new_side_data(st, AV_PKT_DATA_AUDIO_SERVICE_TYPE, sizeof(*ast));
+        AVPacketSideData *side_data = av_packet_side_data_new(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                                                  AV_PKT_DATA_AUDIO_SERVICE_TYPE,
+                                                                  sizeof(*ast), 0);
         if (!side_data)
             return AVERROR(ENOMEM);
-        ast = (enum AVAudioServiceType*)side_data;
+        ast = (enum AVAudioServiceType*)side_data->data;
         *ast = service_type;
     }
 
@@ -2982,19 +2984,21 @@  static int mxf_parse_structural_metadata(MXFContext *mxf)
             st->codecpar->color_trc       = mxf_get_codec_ul(ff_mxf_color_trc_uls, &descriptor->color_trc_ul)->id;
             st->codecpar->color_space     = mxf_get_codec_ul(ff_mxf_color_space_uls, &descriptor->color_space_ul)->id;
             if (descriptor->mastering) {
-                ret = av_stream_add_side_data(st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
-                                              (uint8_t *)descriptor->mastering,
-                                              sizeof(*descriptor->mastering));
-                if (ret < 0)
+                if (!av_packet_side_data_add(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                             AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
+                                                 (uint8_t *)descriptor->mastering, sizeof(*descriptor->mastering), 0)) {
+                    ret = AVERROR(ENOMEM);
                     goto fail_and_free;
+                }
                 descriptor->mastering = NULL;
             }
             if (descriptor->coll) {
-                ret = av_stream_add_side_data(st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
-                                              (uint8_t *)descriptor->coll,
-                                              descriptor->coll_size);
-                if (ret < 0)
+                if (!av_packet_side_data_add(&st->codecpar->side_data, &st->codecpar->nb_side_data,
+                                             AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
+                                                 (uint8_t *)descriptor->coll, descriptor->coll_size, 0)) {
+                    ret = AVERROR(ENOMEM);
                     goto fail_and_free;
+                }
                 descriptor->coll = NULL;
             }
         } else if (st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
diff --git a/libavformat/mxfenc.c b/libavformat/mxfenc.c
index cb299412e2..7d5d4c180d 100644
--- a/libavformat/mxfenc.c
+++ b/libavformat/mxfenc.c
@@ -556,7 +556,8 @@  static void mxf_write_primer_pack(AVFormatContext *s)
         if (s->streams[i]->codecpar->codec_id == AV_CODEC_ID_H264 && !sc->avc_intra) {
             will_have_avc_tags = 1;
         }
-        if (av_stream_get_side_data(s->streams[i], AV_PKT_DATA_MASTERING_DISPLAY_METADATA, NULL)) {
+        if (av_packet_side_data_get(s->streams[i]->codecpar->side_data, s->streams[i]->codecpar->nb_side_data,
+                                    AV_PKT_DATA_MASTERING_DISPLAY_METADATA)) {
             will_have_mastering_tags = 1;
         }
         if (s->streams[i]->codecpar->codec_id == AV_CODEC_ID_FFV1) {
@@ -1158,7 +1159,7 @@  static int64_t mxf_write_cdci_common(AVFormatContext *s, AVStream *st, const UID
     const MXFCodecUL *color_trc_ul;
     const MXFCodecUL *color_space_ul;
     int64_t pos = mxf_write_generic_desc(s, st, key);
-    uint8_t *side_data;
+    const AVPacketSideData *side_data;
 
     color_primaries_ul = mxf_get_codec_ul_by_id(ff_mxf_color_primaries_uls, st->codecpar->color_primaries);
     color_trc_ul       = mxf_get_codec_ul_by_id(ff_mxf_color_trc_uls, st->codecpar->color_trc);
@@ -1344,9 +1345,10 @@  static int64_t mxf_write_cdci_common(AVFormatContext *s, AVStream *st, const UID
     avio_write(pb, *sc->codec_ul, 16);
 
     // Mastering Display metadata
-    side_data = av_stream_get_side_data(st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA, NULL);
+    side_data = av_packet_side_data_get(st->codecpar->side_data, st->codecpar->nb_side_data,
+                                        AV_PKT_DATA_MASTERING_DISPLAY_METADATA);
     if (side_data) {
-        const AVMasteringDisplayMetadata *metadata = (const AVMasteringDisplayMetadata*)side_data;
+        const AVMasteringDisplayMetadata *metadata = (const AVMasteringDisplayMetadata*)side_data->data;
         if (metadata->has_primaries) {
             mxf_write_local_tag(s, 12, 0x8301);
             avio_wb16(pb, rescale_mastering_chroma(metadata->display_primaries[0][0]));
diff --git a/libavformat/options.c b/libavformat/options.c
index 194042820e..0f79fe0fa4 100644
--- a/libavformat/options.c
+++ b/libavformat/options.c
@@ -311,7 +311,9 @@  AVStream *avformat_new_stream(AVFormatContext *s, const AVCodec *c)
 
     st->sample_aspect_ratio = (AVRational) { 0, 1 };
 
+#if FF_API_AVSTREAM_SIDE_DATA
     sti->inject_global_side_data = si->inject_global_side_data;
+#endif
 
     sti->need_context_update = 1;
 
diff --git a/libavformat/replaygain.c b/libavformat/replaygain.c
index 915bcb2382..ab62090ffe 100644
--- a/libavformat/replaygain.c
+++ b/libavformat/replaygain.c
@@ -69,16 +69,20 @@  static int32_t parse_value(const char *value, int32_t min)
 int ff_replaygain_export_raw(AVStream *st, int32_t tg, uint32_t tp,
                              int32_t ag, uint32_t ap)
 {
+    AVPacketSideData *sd;
     AVReplayGain *replaygain;
 
     if (tg == INT32_MIN && ag == INT32_MIN)
         return 0;
 
-    replaygain = (AVReplayGain*)av_stream_new_side_data(st, AV_PKT_DATA_REPLAYGAIN,
-                                                        sizeof(*replaygain));
-    if (!replaygain)
+    sd = av_packet_side_data_new(&st->codecpar->side_data,
+                                 &st->codecpar->nb_side_data,
+                                     AV_PKT_DATA_REPLAYGAIN,
+                                     sizeof(*replaygain), 0);
+    if (!sd)
         return AVERROR(ENOMEM);
 
+    replaygain = (AVReplayGain*)sd->data;
     replaygain->track_gain = tg;
     replaygain->track_peak = tp;
     replaygain->album_gain = ag;
diff --git a/libavformat/seek.c b/libavformat/seek.c
index 386312cd3a..0180188595 100644
--- a/libavformat/seek.c
+++ b/libavformat/seek.c
@@ -745,8 +745,10 @@  void ff_read_frame_flush(AVFormatContext *s)
         for (int j = 0; j < MAX_REORDER_DELAY + 1; j++)
             sti->pts_buffer[j] = AV_NOPTS_VALUE;
 
+#if FF_API_AVSTREAM_SIDE_DATA
         if (si->inject_global_side_data)
             sti->inject_global_side_data = 1;
+#endif
 
         sti->skip_samples = 0;
     }
diff --git a/libavformat/version_major.h b/libavformat/version_major.h
index d1d6f95b15..3c04b38871 100644
--- a/libavformat/version_major.h
+++ b/libavformat/version_major.h
@@ -47,6 +47,7 @@ 
 #define FF_API_AVFORMAT_IO_CLOSE        (LIBAVFORMAT_VERSION_MAJOR < 61)
 #define FF_API_AVIO_WRITE_NONCONST      (LIBAVFORMAT_VERSION_MAJOR < 61)
 #define FF_API_LAVF_SHORTEST            (LIBAVFORMAT_VERSION_MAJOR < 61)
+#define FF_API_AVSTREAM_SIDE_DATA       (LIBAVFORMAT_VERSION_MAJOR < 61)
 
 
 #define FF_API_R_FRAME_RATE            1