[FFmpeg-devel,03/13] avformat/avformat: use the side data from AVStream.codecpar

diff --git a/libavdevice/android_camera.c b/libavdevice/android_camera.c
index 1934999c18..7ceb1e1fc2 100644
--- a/libavdevice/android_camera.c
+++ b/libavdevice/android_camera.c
@@ -638,7 +638,7 @@  static int wait_for_image_format(AVFormatContext *avctx)
 static int add_display_matrix(AVFormatContext *avctx, AVStream *st)
 {
     AndroidCameraCtx *ctx = avctx->priv_data;
-    uint8_t *side_data;
+    AVPacketSideData *side_data;
     int32_t display_matrix[9];
 
     av_display_rotation_set(display_matrix, ctx->sensor_orientation);
@@ -647,14 +647,14 @@  static int add_display_matrix(AVFormatContext *avctx, AVStream *st)
         av_display_matrix_flip(display_matrix, 1, 0);
     }
 
-    side_data = av_stream_new_side_data(st,
+    side_data = av_packet_new_side_data_to_set(&st->codecpar.side_data_set,
             AV_PKT_DATA_DISPLAYMATRIX, sizeof(display_matrix));
 
     if (!side_data) {
         return AVERROR(ENOMEM);
     }
 
-    memcpy(side_data, display_matrix, sizeof(display_matrix));
+    memcpy(side_data->data, display_matrix, sizeof(display_matrix));
 
     return 0;
 }
diff --git a/libavformat/avformat.c b/libavformat/avformat.c
index 356b4de931..3afc5afd7f 100644
--- a/libavformat/avformat.c
+++ b/libavformat/avformat.c
@@ -46,9 +46,13 @@  void ff_free_stream(AVStream **pst)
     if (!st)
         return;
 
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
     for (int i = 0; i < st->nb_side_data; i++)
         av_freep(&st->side_data[i].data);
     av_freep(&st->side_data);
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
 
     if (st->attached_pic.data)
         av_packet_unref(&st->attached_pic);
@@ -138,6 +142,8 @@  void avformat_free_context(AVFormatContext *s)
     av_free(s);
 }
 
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
 uint8_t *av_stream_get_side_data(const AVStream *st,
                                  enum AVPacketSideDataType type, size_t *size)
 {
@@ -205,36 +211,8 @@  uint8_t *av_stream_new_side_data(AVStream *st, enum AVPacketSideDataType type,
 
     return data;
 }
-
-int ff_stream_side_data_copy(AVStream *dst, const AVStream *src)
-{
-    /* Free existing side data*/
-    for (int i = 0; i < dst->nb_side_data; i++)
-        av_free(dst->side_data[i].data);
-    av_freep(&dst->side_data);
-    dst->nb_side_data = 0;
-
-    /* Copy side data if present */
-    if (src->nb_side_data) {
-        dst->side_data = av_calloc(src->nb_side_data,
-                                   sizeof(*dst->side_data));
-        if (!dst->side_data)
-            return AVERROR(ENOMEM);
-        dst->nb_side_data = src->nb_side_data;
-
-        for (int i = 0; i < src->nb_side_data; i++) {
-            uint8_t *data = av_memdup(src->side_data[i].data,
-                                      src->side_data[i].size);
-            if (!data)
-                return AVERROR(ENOMEM);
-            dst->side_data[i].type = src->side_data[i].type;
-            dst->side_data[i].size = src->side_data[i].size;
-            dst->side_data[i].data = data;
-        }
-    }
-
-    return 0;
-}
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
 
 /**
  * Copy all stream parameters from source to destination stream, with the
@@ -270,10 +248,6 @@  static int stream_params_copy(AVStream *dst, const AVStream *src)
     if (ret < 0)
         return ret;
 
-    ret = ff_stream_side_data_copy(dst, src);
-    if (ret < 0)
-        return ret;
-
     av_packet_unref(&dst->attached_pic);
     if (src->attached_pic.data) {
         ret = av_packet_ref(&dst->attached_pic, &src->attached_pic);
diff --git a/libavformat/avformat.h b/libavformat/avformat.h
index 1916aa2dc5..e67ed4205e 100644
--- a/libavformat/avformat.h
+++ b/libavformat/avformat.h
@@ -164,6 +164,12 @@ 
  * decoding functions avcodec_send_packet() or avcodec_decode_subtitle2() if the
  * caller wishes to decode the data.
  *
+ * There may be no overlap between AVCodecParameters.side_data_set and side data
+ * in packets. I.e. a given side data is either exported by the demuxer in
+ * AVCodecParameters, then it never appears in the packets, or the side data is
+ * exported through the packets (always in the first packet where the value
+ * becomes known or changes), then it does not appear in AVCodecParameters.
+ *
  * AVPacket.pts, AVPacket.dts and AVPacket.duration timing information will be
  * set if known. They may also be unset (i.e. AV_NOPTS_VALUE for
  * pts/dts, 0 for duration) if the stream does not provide them. The timing
@@ -209,6 +215,11 @@ 
  *   AVCodecParameters, rather than using @ref avcodec_parameters_copy() during
  *   remuxing: there is no guarantee that the codec context values remain valid
  *   for both input and output format contexts.
+ * - There may be no overlap between AVCodecParameters.side_data_set and side data
+ *   in packets. I.e. a given side data is either set by the caller in
+ *   AVCodecParameters, then it never appears in the packets, or the side data is
+ *   sent through the packets (always in the first packet where the value becomes
+ *   known or changes), then it does not appear in AVCodecParameters.
  * - The caller may fill in additional information, such as @ref
  *   AVFormatContext.metadata "global" or @ref AVStream.metadata "per-stream"
  *   metadata, @ref AVFormatContext.chapters "chapters", @ref
@@ -937,6 +948,7 @@  typedef struct AVStream {
      */
     AVPacket attached_pic;
 
+#if FF_API_AVSTREAM_SIDE_DATA
     /**
      * An array of side data that applies to the whole stream (i.e. the
      * container does not allow it to change between packets).
@@ -953,13 +965,20 @@  typedef struct AVStream {
      *
      * Freed by libavformat in avformat_free_context().
      *
+     * @deprecated use AVStream's codecpar.side_data_set
+     *
      * @see av_format_inject_global_side_data()
      */
+    attribute_deprecated
     AVPacketSideData *side_data;
     /**
      * The number of elements in the AVStream.side_data array.
+     *
+     * @deprecated use AVStream's codecpar.side_data_set
      */
+    attribute_deprecated
     int            nb_side_data;
+#endif
 
     /**
      * Flags indicating events happening on the stream, a combination of
@@ -1715,11 +1734,14 @@  typedef struct AVFormatContext {
     int (*io_close2)(struct AVFormatContext *s, AVIOContext *pb);
 } AVFormatContext;
 
+#if FF_API_AVSTREAM_SIDE_DATA
 /**
  * This function will cause global side data to be injected in the next packet
  * of each stream as well as after any subsequent seek.
  */
+attribute_deprecated
 void av_format_inject_global_side_data(AVFormatContext *s);
+#endif
 
 /**
  * Returns the method used to set ctx->duration.
@@ -1856,7 +1878,9 @@  AVStream *avformat_new_stream(AVFormatContext *s, const AVCodec *c);
  *
  * @return zero on success, a negative AVERROR code on failure. On failure,
  *         the stream is unchanged and the data remains owned by the caller.
+ * @deprecated use av_packet_add_side_data_to_set() and stream->codecpar->side_data_set
  */
+attribute_deprecated
 int av_stream_add_side_data(AVStream *st, enum AVPacketSideDataType type,
                             uint8_t *data, size_t size);
 
@@ -1868,7 +1892,9 @@  int av_stream_add_side_data(AVStream *st, enum AVPacketSideDataType type,
  * @param size   side information size
  *
  * @return pointer to fresh allocated data or NULL otherwise
+ * @deprecated use av_packet_new_side_data_to_set() and stream->codecpar->side_data_set
  */
+attribute_deprecated
 uint8_t *av_stream_new_side_data(AVStream *stream,
                                  enum AVPacketSideDataType type, size_t size);
 /**
@@ -1880,7 +1906,9 @@  uint8_t *av_stream_new_side_data(AVStream *stream,
  *               or to zero if the desired side data is not present.
  *
  * @return pointer to data if present or NULL otherwise
+ * @deprecated use av_packet_get_side_data_from_set() and stream->codecpar->side_data_set
  */
+attribute_deprecated
 uint8_t *av_stream_get_side_data(const AVStream *stream,
                                  enum AVPacketSideDataType type, size_t *size);
 
diff --git a/libavformat/concatdec.c b/libavformat/concatdec.c
index 5d4f67d0ac..2aeb4f1dfd 100644
--- a/libavformat/concatdec.c
+++ b/libavformat/concatdec.c
@@ -194,7 +194,6 @@  static int copy_stream_props(AVStream *st, AVStream *source_st)
     avpriv_set_pts_info(st, 64, source_st->time_base.num, source_st->time_base.den);
 
     av_dict_copy(&st->metadata, source_st->metadata, 0);
-    ff_stream_side_data_copy(st, source_st);
     return 0;
 }
 
diff --git a/libavformat/dashdec.c b/libavformat/dashdec.c
index 29d4680c68..2441087606 100644
--- a/libavformat/dashdec.c
+++ b/libavformat/dashdec.c
@@ -1952,17 +1952,6 @@  static int open_demux_for_component(AVFormatContext *s, struct representation *p
 
         // copy disposition
         st->disposition = ist->disposition;
-
-        // copy side data
-        for (int i = 0; i < ist->nb_side_data; i++) {
-            const AVPacketSideData *sd_src = &ist->side_data[i];
-            uint8_t *dst_data;
-
-            dst_data = av_stream_new_side_data(st, sd_src->type, sd_src->size);
-            if (!dst_data)
-                return AVERROR(ENOMEM);
-            memcpy(dst_data, sd_src->data, sd_src->size);
-        }
     }
 
     return 0;
diff --git a/libavformat/demux.c b/libavformat/demux.c
index 1ef297d5e7..7253196a82 100644
--- a/libavformat/demux.c
+++ b/libavformat/demux.c
@@ -1408,9 +1408,10 @@  FF_ENABLE_DEPRECATION_WARNINGS
             sti->skip_samples = 0;
         }
 
+#if FF_API_AVSTREAM_SIDE_DATA
         if (sti->inject_global_side_data) {
-            for (int i = 0; i < st->nb_side_data; i++) {
-                const AVPacketSideData *const src_sd = &st->side_data[i];
+            for (int i = 0; i < st->codecpar->side_data_set.nb_side_data; i++) {
+                const AVPacketSideData *const src_sd = &st->codecpar->side_data_set.side_data[i];
                 uint8_t *dst_data;
 
                 if (av_packet_get_side_data(pkt, src_sd->type, NULL))
@@ -1426,6 +1427,7 @@  FF_ENABLE_DEPRECATION_WARNINGS
             }
             sti->inject_global_side_data = 0;
         }
+#endif
     }
 
     if (!si->metafree) {
@@ -2434,11 +2436,11 @@  static int add_coded_side_data(AVStream *st, AVCodecContext *avctx)
 {
     for (int i = 0; i < avctx->nb_coded_side_data; i++) {
         const AVPacketSideData *const sd_src = &avctx->coded_side_data[i];
-        uint8_t *dst_data;
-        dst_data = av_stream_new_side_data(st, sd_src->type, sd_src->size);
-        if (!dst_data)
+        AVPacketSideData *sd_dst = av_packet_new_side_data_to_set(&st->codecpar->side_data_set,
+                                                                  sd_src->type, sd_src->size);
+        if (!sd_dst)
             return AVERROR(ENOMEM);
-        memcpy(dst_data, sd_src->data, sd_src->size);
+        memcpy(sd_dst->data, sd_src->data, sd_src->size);
     }
     return 0;
 }
@@ -2975,6 +2977,32 @@  int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
                 goto find_stream_info_err;
         }
 
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
+        if (st->codecpar->side_data_set.nb_side_data > 0) {
+            av_assert0(!st->side_data && !st->nb_side_data);
+            st->side_data = av_calloc(st->codecpar->side_data_set.nb_side_data,
+                                      sizeof(*st->side_data));
+            if (!st->side_data) {
+                ret = AVERROR(ENOMEM);
+                goto find_stream_info_err;
+            }
+
+            for (int i = 0; i < st->codecpar->side_data_set.nb_side_data; i++) {
+                uint8_t *data = av_memdup(st->codecpar->side_data_set.side_data[i].data,
+                                          st->codecpar->side_data_set.side_data[i].size);
+                if (!data) {
+                    ret = AVERROR(ENOMEM);
+                    goto find_stream_info_err;
+                }
+                st->side_data[i].type = st->codecpar->side_data_set.side_data[i].type;
+                st->side_data[i].size = st->codecpar->side_data_set.side_data[i].size;
+                st->side_data[i].data = data;
+                st->nb_side_data++;
+            }
+        }
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
         sti->avctx_inited = 0;
     }
 
diff --git a/libavformat/demux_utils.c b/libavformat/demux_utils.c
index 56cc6e15d8..2946e82295 100644
--- a/libavformat/demux_utils.c
+++ b/libavformat/demux_utils.c
@@ -80,6 +80,8 @@  AVChapter *avpriv_new_chapter(AVFormatContext *s, int64_t id, AVRational time_ba
     return chapter;
 }
 
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
 void av_format_inject_global_side_data(AVFormatContext *s)
 {
     FFFormatContext *const si = ffformatcontext(s);
@@ -89,6 +91,8 @@  void av_format_inject_global_side_data(AVFormatContext *s)
         ffstream(st)->inject_global_side_data = 1;
     }
 }
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
 
 int avformat_queue_attached_pictures(AVFormatContext *s)
 {
diff --git a/libavformat/dovi_isom.c b/libavformat/dovi_isom.c
index 76681b9451..031f19dba5 100644
--- a/libavformat/dovi_isom.c
+++ b/libavformat/dovi_isom.c
@@ -63,7 +63,7 @@  int ff_isom_parse_dvcc_dvvc(AVFormatContext *s, AVStream *st, const uint8_t *buf
         dovi->dv_bl_signal_compatibility_id = 0;
     }
 
-    ret = av_stream_add_side_data(st, AV_PKT_DATA_DOVI_CONF,
+    ret = av_packet_add_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_DOVI_CONF,
                                   (uint8_t *)dovi, dovi_size);
     if (ret < 0) {
         av_free(dovi);
diff --git a/libavformat/dump.c b/libavformat/dump.c
index d31e4c2ec6..76981fb45f 100644
--- a/libavformat/dump.c
+++ b/libavformat/dump.c
@@ -431,11 +431,11 @@  static void dump_sidedata(void *ctx, const AVStream *st, const char *indent)
 {
     int i;
 
-    if (st->nb_side_data)
+    if (st->codecpar->side_data_set.nb_side_data)
         av_log(ctx, AV_LOG_INFO, "%sSide data:\n", indent);
 
-    for (i = 0; i < st->nb_side_data; i++) {
-        const AVPacketSideData *sd = &st->side_data[i];
+    for (i = 0; i < st->codecpar->side_data_set.nb_side_data; i++) {
+        const AVPacketSideData *sd = &st->codecpar->side_data_set.side_data[i];
         av_log(ctx, AV_LOG_INFO, "%s  ", indent);
 
         switch (sd->type) {
diff --git a/libavformat/hls.c b/libavformat/hls.c
index 2a2fe28a54..d792833290 100644
--- a/libavformat/hls.c
+++ b/libavformat/hls.c
@@ -1851,17 +1851,6 @@  static int set_stream_info_from_input_stream(AVStream *st, struct playlist *pls,
 
     av_dict_copy(&st->metadata, ist->metadata, 0);
 
-    // copy side data
-    for (int i = 0; i < ist->nb_side_data; i++) {
-        const AVPacketSideData *sd_src = &ist->side_data[i];
-        uint8_t *dst_data;
-
-        dst_data = av_stream_new_side_data(st, sd_src->type, sd_src->size);
-        if (!dst_data)
-            return AVERROR(ENOMEM);
-        memcpy(dst_data, sd_src->data, sd_src->size);
-    }
-
     ffstream(st)->need_context_update = 1;
 
     return 0;
diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c
index 27d97f5f72..b61079bd76 100644
--- a/libavformat/hlsenc.c
+++ b/libavformat/hlsenc.c
@@ -1351,16 +1351,17 @@  static const char* get_relative_url(const char *master_url, const char *media_ur
 
 static int64_t get_stream_bit_rate(AVStream *stream)
 {
-    AVCPBProperties *props = (AVCPBProperties*)av_stream_get_side_data(
-        stream,
-        AV_PKT_DATA_CPB_PROPERTIES,
-        NULL
+    AVPacketSideData *sd = av_packet_get_side_data_from_set(
+        &stream->codecpar->side_data_set,
+        AV_PKT_DATA_CPB_PROPERTIES
     );
 
     if (stream->codecpar->bit_rate)
         return stream->codecpar->bit_rate;
-    else if (props)
+    else if (sd) {
+        AVCPBProperties *props = (AVCPBProperties*)sd->data;
         return props->max_bitrate;
+    }
 
     return 0;
 }
diff --git a/libavformat/internal.h b/libavformat/internal.h
index 9fc980601f..e625a6c654 100644
--- a/libavformat/internal.h
+++ b/libavformat/internal.h
@@ -149,7 +149,9 @@  typedef struct FFFormatContext {
     int missing_ts_warning;
 #endif
 
+#if FF_API_AVSTREAM_SIDE_DATA
     int inject_global_side_data;
+#endif
 
     int avoid_negative_ts_use_pts;
 
@@ -356,10 +358,12 @@  typedef struct FFStream {
     uint8_t dts_ordered;
     uint8_t dts_misordered;
 
+#if FF_API_AVSTREAM_SIDE_DATA
     /**
      * Internal data to inject global side data
      */
     int inject_global_side_data;
+#endif
 
     /**
      * display aspect ratio (0 if unknown)
diff --git a/libavformat/matroska.c b/libavformat/matroska.c
index 79b2d09984..d217b68584 100644
--- a/libavformat/matroska.c
+++ b/libavformat/matroska.c
@@ -198,7 +198,7 @@  int ff_mkv_stereo3d_conv(AVStream *st, MatroskaVideoStereoModeType stereo_mode)
         break;
     }
 
-    ret = av_stream_add_side_data(st, AV_PKT_DATA_STEREO3D, (uint8_t *)stereo,
+    ret = av_packet_add_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_STEREO3D, (uint8_t *)stereo,
                                   sizeof(*stereo));
     if (ret < 0) {
         av_freep(&stereo);
diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c
index 49950956b6..598cf8a7bf 100644
--- a/libavformat/matroskadec.c
+++ b/libavformat/matroskadec.c
@@ -2202,7 +2202,7 @@  static int mkv_parse_video_color(AVStream *st, const MatroskaTrack *track) {
         AVContentLightMetadata *metadata = av_content_light_metadata_alloc(&size);
         if (!metadata)
             return AVERROR(ENOMEM);
-        ret = av_stream_add_side_data(st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
+        ret = av_packet_add_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
                                       (uint8_t *)metadata, size);
         if (ret < 0) {
             av_freep(&metadata);
@@ -2213,13 +2213,13 @@  static int mkv_parse_video_color(AVStream *st, const MatroskaTrack *track) {
     }
 
     if (has_mastering_primaries || has_mastering_luminance) {
-        AVMasteringDisplayMetadata *metadata =
-            (AVMasteringDisplayMetadata*) av_stream_new_side_data(
-                st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
+        AVMasteringDisplayMetadata *metadata;
+        AVPacketSideData *sd = av_packet_new_side_data_to_set(
+                &st->codecpar->side_data_set, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
                 sizeof(AVMasteringDisplayMetadata));
-        if (!metadata) {
+        if (!sd)
             return AVERROR(ENOMEM);
-        }
+        metadata = (AVMasteringDisplayMetadata*)sd->data;
         memset(metadata, 0, sizeof(AVMasteringDisplayMetadata));
         if (has_mastering_primaries) {
             metadata->display_primaries[0][0] = av_d2q(mastering_meta->r_x, INT_MAX);
@@ -2245,6 +2245,7 @@  static int mkv_create_display_matrix(AVStream *st,
                                      const MatroskaTrackVideoProjection *proj,
                                      void *logctx)
 {
+    AVPacketSideData *sd;
     double pitch = proj->pitch, yaw = proj->yaw, roll = proj->roll;
     int32_t *matrix;
     int hflip;
@@ -2261,10 +2262,11 @@  static int mkv_create_display_matrix(AVStream *st,
                st->index, yaw, pitch, roll);
         return 0;
     }
-    matrix = (int32_t*)av_stream_new_side_data(st, AV_PKT_DATA_DISPLAYMATRIX,
+    sd = av_packet_new_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_DISPLAYMATRIX,
                                                9 * sizeof(*matrix));
-    if (!matrix)
+    if (!sd)
         return AVERROR(ENOMEM);
+    matrix = (int32_t*)sd->data;
 
     hflip = yaw != 0.0;
     /* ProjectionPoseRoll is in the counter-clockwise direction
@@ -2365,7 +2367,7 @@  static int mkv_parse_video_projection(AVStream *st, const MatroskaTrack *track,
     spherical->bound_right  = r;
     spherical->bound_bottom = b;
 
-    ret = av_stream_add_side_data(st, AV_PKT_DATA_SPHERICAL, (uint8_t *)spherical,
+    ret = av_packet_add_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_SPHERICAL, (uint8_t *)spherical,
                                   spherical_size);
     if (ret < 0) {
         av_freep(&spherical);
diff --git a/libavformat/matroskaenc.c b/libavformat/matroskaenc.c
index 41e13b273d..a382cdd120 100644
--- a/libavformat/matroskaenc.c
+++ b/libavformat/matroskaenc.c
@@ -1323,7 +1323,7 @@  fail:
 static void mkv_write_video_color(EbmlWriter *writer, const AVStream *st,
                                   const AVCodecParameters *par)
 {
-    const void *side_data;
+    const AVPacketSideData *side_data;
 
     ebml_writer_open_master(writer, MATROSKA_ID_VIDEOCOLOR);
 
@@ -1357,20 +1357,18 @@  static void mkv_write_video_color(EbmlWriter *writer, const AVStream *st,
                              (ypos >> 7) + 1);
     }
 
-    side_data = av_stream_get_side_data(st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
-                                        NULL);
+    side_data = av_packet_get_side_data_from_set(&st->codecpar->side_data_set, AV_PKT_DATA_CONTENT_LIGHT_LEVEL);
     if (side_data) {
-        const AVContentLightMetadata *metadata = side_data;
+        const AVContentLightMetadata *metadata = (AVContentLightMetadata *)side_data->data;
         ebml_writer_add_uint(writer, MATROSKA_ID_VIDEOCOLORMAXCLL,
                              metadata->MaxCLL);
         ebml_writer_add_uint(writer, MATROSKA_ID_VIDEOCOLORMAXFALL,
                              metadata->MaxFALL);
     }
 
-    side_data = av_stream_get_side_data(st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
-                                        NULL);
+    side_data = av_packet_get_side_data_from_set(&st->codecpar->side_data_set, AV_PKT_DATA_MASTERING_DISPLAY_METADATA);
     if (side_data) {
-        const AVMasteringDisplayMetadata *metadata = side_data;
+        const AVMasteringDisplayMetadata *metadata = (AVMasteringDisplayMetadata *)side_data->data;
         ebml_writer_open_master(writer, MATROSKA_ID_VIDEOCOLORMASTERINGMETA);
         if (metadata->has_primaries) {
             ebml_writer_add_float(writer, MATROSKA_ID_VIDEOCOLOR_RX,
@@ -1406,13 +1404,13 @@  static void mkv_write_video_color(EbmlWriter *writer, const AVStream *st,
 static void mkv_write_video_projection(AVFormatContext *s, EbmlWriter *writer,
                                        const AVStream *st, uint8_t private[])
 {
-    const AVSphericalMapping *spherical =
-        (const AVSphericalMapping *)av_stream_get_side_data(st, AV_PKT_DATA_SPHERICAL,
-                                                            NULL);
+    AVPacketSideData *sd = av_packet_get_side_data_from_set(&st->codecpar->side_data_set, AV_PKT_DATA_SPHERICAL);
+    const AVSphericalMapping *spherical;
 
-    if (!spherical)
+    if (!sd)
         return;
 
+    spherical = (const AVSphericalMapping *)sd->data;
     if (spherical->projection != AV_SPHERICAL_EQUIRECTANGULAR      &&
         spherical->projection != AV_SPHERICAL_EQUIRECTANGULAR_TILE &&
         spherical->projection != AV_SPHERICAL_CUBEMAP) {
@@ -1513,9 +1511,9 @@  static int mkv_write_stereo_mode(AVFormatContext *s, EbmlWriter *writer,
                                  AVStream *st, int is_webm,
                                  int *h_width, int *h_height)
 {
+    AVPacketSideData *sd;
     const AVDictionaryEntry *tag;
     MatroskaVideoStereoModeType format = MATROSKA_VIDEO_STEREOMODE_TYPE_NB;
-    const AVStereo3D *stereo;
 
     *h_width = 1;
     *h_height = 1;
@@ -1538,9 +1536,9 @@  static int mkv_write_stereo_mode(AVFormatContext *s, EbmlWriter *writer,
         }
     }
 
-    stereo = (const AVStereo3D*)av_stream_get_side_data(st, AV_PKT_DATA_STEREO3D,
-                                                        NULL);
-    if (stereo) {
+    sd = av_packet_get_side_data_from_set(&st->codecpar->side_data_set, AV_PKT_DATA_STEREO3D);
+    if (sd) {
+        const AVStereo3D *stereo = (const AVStereo3D*)sd->data;
         switch (stereo->type) {
         case AV_STEREO3D_2D:
             format = MATROSKA_VIDEO_STEREOMODE_TYPE_MONO;
@@ -1605,8 +1603,11 @@  static void mkv_write_blockadditionmapping(AVFormatContext *s, MatroskaMuxContex
                                            AVIOContext *pb, mkv_track *track, AVStream *st)
 {
 #if CONFIG_MATROSKA_MUXER
-    AVDOVIDecoderConfigurationRecord *dovi = (AVDOVIDecoderConfigurationRecord *)
-                                             av_stream_get_side_data(st, AV_PKT_DATA_DOVI_CONF, NULL);
+    AVPacketSideData *sd = av_packet_get_side_data_from_set(&st->codecpar->side_data_set, AV_PKT_DATA_DOVI_CONF);
+    AVDOVIDecoderConfigurationRecord *dovi = NULL;
+
+    if (sd)
+        dovi = (AVDOVIDecoderConfigurationRecord *)sd->data;
 
     if (IS_SEEKABLE(s->pb, mkv)) {
         track->blockadditionmapping_offset = avio_tell(pb);
diff --git a/libavformat/mov.c b/libavformat/mov.c
index ac19b3157a..4669ab998c 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -799,6 +799,7 @@  static int mov_read_esds(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 static int mov_read_dac3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 {
     AVStream *st;
+    AVPacketSideData *sd;
     enum AVAudioServiceType *ast;
     int ac3info, acmod, lfeon, bsmod;
     uint64_t mask;
@@ -807,11 +808,12 @@  static int mov_read_dac3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
         return 0;
     st = c->fc->streams[c->fc->nb_streams-1];
 
-    ast = (enum AVAudioServiceType*)av_stream_new_side_data(st, AV_PKT_DATA_AUDIO_SERVICE_TYPE,
+    sd = av_packet_new_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_AUDIO_SERVICE_TYPE,
                                                             sizeof(*ast));
-    if (!ast)
+    if (!sd)
         return AVERROR(ENOMEM);
 
+    ast = (enum AVAudioServiceType*)sd->data;
     ac3info = avio_rb24(pb);
     bsmod = (ac3info >> 14) & 0x7;
     acmod = (ac3info >> 11) & 0x7;
@@ -833,6 +835,7 @@  static int mov_read_dac3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 static int mov_read_dec3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 {
     AVStream *st;
+    AVPacketSideData *sd;
     enum AVAudioServiceType *ast;
     int eac3info, acmod, lfeon, bsmod;
     uint64_t mask;
@@ -841,11 +844,13 @@  static int mov_read_dec3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
         return 0;
     st = c->fc->streams[c->fc->nb_streams-1];
 
-    ast = (enum AVAudioServiceType*)av_stream_new_side_data(st, AV_PKT_DATA_AUDIO_SERVICE_TYPE,
+    sd = av_packet_new_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_AUDIO_SERVICE_TYPE,
                                                             sizeof(*ast));
-    if (!ast)
+    if (!sd)
         return AVERROR(ENOMEM);
 
+    ast = (enum AVAudioServiceType*)sd->data;
+
     /* No need to parse fields for additional independent substreams and its
      * associated dependent substreams since libavcodec's E-AC-3 decoder
      * does not support them yet. */
@@ -1747,7 +1752,6 @@  static int mov_read_pcmc(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 static int mov_read_colr(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 {
     AVStream *st;
-    uint8_t *icc_profile;
     char color_parameter_type[5] = { 0 };
     uint16_t color_primaries, color_trc, color_matrix;
     int ret;
@@ -1768,10 +1772,10 @@  static int mov_read_colr(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     }
 
     if (!strncmp(color_parameter_type, "prof", 4)) {
-        icc_profile = av_stream_new_side_data(st, AV_PKT_DATA_ICC_PROFILE, atom.size - 4);
-        if (!icc_profile)
+        AVPacketSideData *sd = av_packet_new_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_ICC_PROFILE, atom.size - 4);
+        if (!sd)
             return AVERROR(ENOMEM);
-        ret = ffio_read_size(pb, icc_profile, atom.size - 4);
+        ret = ffio_read_size(pb, sd->data, atom.size - 4);
         if (ret < 0)
             return ret;
     } else {
@@ -6831,8 +6835,9 @@  static int mov_read_pssh(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     AVEncryptionInitInfo *info, *old_init_info;
     uint8_t **key_ids;
     AVStream *st;
-    uint8_t *side_data, *extra_data, *old_side_data;
-    size_t side_data_size, old_side_data_size;
+    AVPacketSideData *old_side_data;
+    uint8_t *side_data, *extra_data;
+    size_t side_data_size;
     int ret = 0;
     unsigned int version, kid_count, extra_data_size, alloc_size = 0;
 
@@ -6910,9 +6915,9 @@  static int mov_read_pssh(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     info->data_size = extra_data_size;
 
     // If there is existing initialization data, append to the list.
-    old_side_data = av_stream_get_side_data(st, AV_PKT_DATA_ENCRYPTION_INIT_INFO, &old_side_data_size);
+    old_side_data = av_packet_get_side_data_from_set(&st->codecpar->side_data_set, AV_PKT_DATA_ENCRYPTION_INIT_INFO);
     if (old_side_data) {
-        old_init_info = av_encryption_init_info_get_side_data(old_side_data, old_side_data_size);
+        old_init_info = av_encryption_init_info_get_side_data(old_side_data->data, old_side_data->size);
         if (old_init_info) {
             // Append to the end of the list.
             for (AVEncryptionInitInfo *cur = old_init_info;; cur = cur->next) {
@@ -6934,7 +6939,7 @@  static int mov_read_pssh(MOVContext *c, AVIOContext *pb, MOVAtom atom)
         ret = AVERROR(ENOMEM);
         goto finish;
     }
-    ret = av_stream_add_side_data(st, AV_PKT_DATA_ENCRYPTION_INIT_INFO,
+    ret = av_packet_add_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_ENCRYPTION_INIT_INFO,
                                   side_data, side_data_size);
     if (ret < 0)
         av_free(side_data);
@@ -8713,7 +8718,7 @@  static int mov_read_header(AVFormatContext *s)
             break;
         case AVMEDIA_TYPE_VIDEO:
             if (sc->display_matrix) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_DISPLAYMATRIX, (uint8_t*)sc->display_matrix,
+                err = av_packet_add_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_DISPLAYMATRIX, (uint8_t*)sc->display_matrix,
                                               sizeof(int32_t) * 9);
                 if (err < 0)
                     return err;
@@ -8721,7 +8726,7 @@  static int mov_read_header(AVFormatContext *s)
                 sc->display_matrix = NULL;
             }
             if (sc->stereo3d) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_STEREO3D,
+                err = av_packet_add_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_STEREO3D,
                                               (uint8_t *)sc->stereo3d,
                                               sizeof(*sc->stereo3d));
                 if (err < 0)
@@ -8730,7 +8735,7 @@  static int mov_read_header(AVFormatContext *s)
                 sc->stereo3d = NULL;
             }
             if (sc->spherical) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_SPHERICAL,
+                err = av_packet_add_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_SPHERICAL,
                                               (uint8_t *)sc->spherical,
                                               sc->spherical_size);
                 if (err < 0)
@@ -8739,7 +8744,7 @@  static int mov_read_header(AVFormatContext *s)
                 sc->spherical = NULL;
             }
             if (sc->mastering) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
+                err = av_packet_add_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
                                               (uint8_t *)sc->mastering,
                                               sizeof(*sc->mastering));
                 if (err < 0)
@@ -8748,7 +8753,7 @@  static int mov_read_header(AVFormatContext *s)
                 sc->mastering = NULL;
             }
             if (sc->coll) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
+                err = av_packet_add_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
                                               (uint8_t *)sc->coll,
                                               sc->coll_size);
                 if (err < 0)
diff --git a/libavformat/movenc.c b/libavformat/movenc.c
index f1cc80b1b3..06939d40d0 100644
--- a/libavformat/movenc.c
+++ b/libavformat/movenc.c
@@ -672,11 +672,11 @@  struct mpeg4_bit_rate_values {
 
 static struct mpeg4_bit_rate_values calculate_mpeg4_bit_rates(MOVTrack *track)
 {
-    AVCPBProperties *props = track->st ?
-        (AVCPBProperties*)av_stream_get_side_data(track->st,
-                                                  AV_PKT_DATA_CPB_PROPERTIES,
-                                                  NULL) :
+    AVPacketSideData *sd = track->st ?
+        av_packet_get_side_data_from_set(&track->st->codecpar->side_data_set,
+                                         AV_PKT_DATA_CPB_PROPERTIES) :
         NULL;
+    AVCPBProperties *props = sd ? (AVCPBProperties *)sd->data : NULL;
     struct mpeg4_bit_rate_values bit_rates = { 0 };
 
     bit_rates.avg_bit_rate = compute_avg_bitrate(track);
@@ -2127,18 +2127,16 @@  static int mov_write_colr_tag(AVIOContext *pb, MOVTrack *track, int prefer_icc)
     // Ref (MOV): https://developer.apple.com/library/mac/technotes/tn2162/_index.html#//apple_ref/doc/uid/DTS40013070-CH1-TNTAG9
     // Ref (MP4): ISO/IEC 14496-12:2012
 
-    const uint8_t *icc_profile;
-    size_t icc_profile_size;
-
     if (prefer_icc) {
-        icc_profile = av_stream_get_side_data(track->st, AV_PKT_DATA_ICC_PROFILE, &icc_profile_size);
+        AVPacketSideData *sd = av_packet_get_side_data_from_set(&track->st->codecpar->side_data_set,
+                                                                AV_PKT_DATA_ICC_PROFILE);
 
-        if (icc_profile) {
-            avio_wb32(pb, 12 + icc_profile_size);
+        if (sd) {
+            avio_wb32(pb, 12 + sd->size);
             ffio_wfourcc(pb, "colr");
             ffio_wfourcc(pb, "prof");
-            avio_write(pb, icc_profile, icc_profile_size);
-            return 12 + icc_profile_size;
+            avio_write(pb, sd->data, sd->size);
+            return 12 + sd->size;
         }
         else {
             av_log(NULL, AV_LOG_INFO, "no ICC profile found, will write nclx/nclc colour info instead\n");
@@ -2171,14 +2169,14 @@  static int mov_write_colr_tag(AVIOContext *pb, MOVTrack *track, int prefer_icc)
 
 static int mov_write_clli_tag(AVIOContext *pb, MOVTrack *track)
 {
-    const uint8_t *side_data;
+    const AVPacketSideData *side_data;
     const AVContentLightMetadata *content_light_metadata;
 
-    side_data = av_stream_get_side_data(track->st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL, NULL);
+    side_data = av_packet_get_side_data_from_set(&track->st->codecpar->side_data_set, AV_PKT_DATA_CONTENT_LIGHT_LEVEL);
     if (!side_data) {
         return 0;
     }
-    content_light_metadata = (const AVContentLightMetadata*)side_data;
+    content_light_metadata = (const AVContentLightMetadata*)side_data->data;
 
     avio_wb32(pb, 12); // size
     ffio_wfourcc(pb, "clli");
@@ -2196,11 +2194,12 @@  static int mov_write_mdcv_tag(AVIOContext *pb, MOVTrack *track)
 {
     const int chroma_den = 50000;
     const int luma_den = 10000;
-    const uint8_t *side_data;
-    const AVMasteringDisplayMetadata *metadata;
+    const AVPacketSideData *side_data;
+    const AVMasteringDisplayMetadata *metadata = NULL;
 
-    side_data = av_stream_get_side_data(track->st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA, NULL);
-    metadata = (const AVMasteringDisplayMetadata*)side_data;
+    side_data = av_packet_get_side_data_from_set(&track->st->codecpar->side_data_set, AV_PKT_DATA_MASTERING_DISPLAY_METADATA);
+    if (side_data)
+        metadata = (const AVMasteringDisplayMetadata*)side_data->data;
     if (!metadata || !metadata->has_primaries || !metadata->has_luminance) {
         return 0;
     }
@@ -2419,7 +2418,7 @@  static int mov_write_video_tag(AVFormatContext *s, AVIOContext *pb, MOVMuxContex
                              track->par->color_trc != AVCOL_TRC_UNSPECIFIED &&
                              track->par->color_space != AVCOL_SPC_UNSPECIFIED;
         if (has_color_info || mov->flags & FF_MOV_FLAG_WRITE_COLR ||
-            av_stream_get_side_data(track->st, AV_PKT_DATA_ICC_PROFILE, NULL)) {
+            av_packet_get_side_data_from_set(&track->st->codecpar->side_data_set, AV_PKT_DATA_ICC_PROFILE)) {
             int prefer_icc = mov->flags & FF_MOV_FLAG_PREFER_ICC || !has_color_info;
             mov_write_colr_tag(pb, track, prefer_icc);
         }
@@ -2433,17 +2432,16 @@  static int mov_write_video_tag(AVFormatContext *s, AVIOContext *pb, MOVMuxContex
     }
 
     if (track->mode == MODE_MP4 && mov->fc->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL) {
-        AVStereo3D* stereo_3d = (AVStereo3D*) av_stream_get_side_data(track->st, AV_PKT_DATA_STEREO3D, NULL);
-        AVSphericalMapping* spherical_mapping = (AVSphericalMapping*)av_stream_get_side_data(track->st, AV_PKT_DATA_SPHERICAL, NULL);
-        AVDOVIDecoderConfigurationRecord *dovi = (AVDOVIDecoderConfigurationRecord *)
-                                                 av_stream_get_side_data(track->st, AV_PKT_DATA_DOVI_CONF, NULL);
+        AVPacketSideData *stereo_3d = av_packet_get_side_data_from_set(&track->st->codecpar->side_data_set, AV_PKT_DATA_STEREO3D);
+        AVPacketSideData *spherical_mapping = av_packet_get_side_data_from_set(&track->st->codecpar->side_data_set, AV_PKT_DATA_SPHERICAL);
+        AVPacketSideData *dovi = av_packet_get_side_data_from_set(&track->st->codecpar->side_data_set, AV_PKT_DATA_DOVI_CONF);
 
         if (stereo_3d)
-            mov_write_st3d_tag(s, pb, stereo_3d);
+            mov_write_st3d_tag(s, pb, (AVStereo3D*)stereo_3d->data);
         if (spherical_mapping)
-            mov_write_sv3d_tag(mov->fc, pb, spherical_mapping);
+            mov_write_sv3d_tag(mov->fc, pb, (AVSphericalMapping*)spherical_mapping->data);
         if (dovi)
-            mov_write_dvcc_dvvc_tag(s, pb, dovi);
+            mov_write_dvcc_dvvc_tag(s, pb, (AVDOVIDecoderConfigurationRecord *)dovi->data);
     }
 
     if (track->par->sample_aspect_ratio.den && track->par->sample_aspect_ratio.num) {
@@ -3390,7 +3388,6 @@  static int mov_write_tkhd_tag(AVIOContext *pb, MOVMuxContext *mov,
     int group   = 0;
 
     uint32_t *display_matrix = NULL;
-    size_t display_matrix_size;
     int       i;
 
     if (mov->mode == MODE_AVIF)
@@ -3400,15 +3397,15 @@  static int mov_write_tkhd_tag(AVIOContext *pb, MOVMuxContext *mov,
             duration *= mov->avif_loop_count;
 
     if (st) {
+        AVPacketSideData *sd;
         if (mov->per_stream_grouping)
             group = st->index;
         else
             group = st->codecpar->codec_type;
 
-        display_matrix = (uint32_t*)av_stream_get_side_data(st, AV_PKT_DATA_DISPLAYMATRIX,
-                                                            &display_matrix_size);
-        if (display_matrix && display_matrix_size < 9 * sizeof(*display_matrix))
-            display_matrix = NULL;
+        sd = av_packet_get_side_data_from_set(&st->codecpar->side_data_set, AV_PKT_DATA_DISPLAYMATRIX);
+        if (sd && sd->size == 9 * sizeof(*display_matrix))
+            display_matrix = (uint32_t *)sd->data;
     }
 
     if (track->flags & MOV_TRACK_ENABLED)
@@ -4606,12 +4603,10 @@  static int mov_write_moov_tag(AVIOContext *pb, MOVMuxContext *mov,
             track->tref_tag = MKTAG('h','i','n','t');
             track->tref_id = mov->tracks[track->src_track].track_id;
         } else if (track->par->codec_type == AVMEDIA_TYPE_AUDIO) {
-            size_t size;
-            int *fallback;
-            fallback = (int*)av_stream_get_side_data(track->st,
-                                                     AV_PKT_DATA_FALLBACK_TRACK,
-                                                     &size);
-            if (fallback != NULL && size == sizeof(int)) {
+            AVPacketSideData *sd = av_packet_get_side_data_from_set(&track->st->codecpar->side_data_set,
+                                                                    AV_PKT_DATA_FALLBACK_TRACK );
+            if (sd && sd->size == sizeof(int)) {
+                int *fallback = (int *)sd->data;
                 if (*fallback >= 0 && *fallback < mov->nb_streams) {
                     track->tref_tag = MKTAG('f','a','l','l');
                     track->tref_id = mov->tracks[*fallback].track_id;
@@ -5444,7 +5439,7 @@  static int mov_write_ftyp_tag(AVIOContext *pb, AVFormatContext *s)
         if (st->codecpar->codec_id == AV_CODEC_ID_AC3 ||
             st->codecpar->codec_id == AV_CODEC_ID_EAC3 ||
             st->codecpar->codec_id == AV_CODEC_ID_TRUEHD ||
-            av_stream_get_side_data(st, AV_PKT_DATA_DOVI_CONF, NULL))
+            av_packet_get_side_data_from_set(&st->codecpar->side_data_set, AV_PKT_DATA_DOVI_CONF))
             has_dolby = 1;
     }
 
diff --git a/libavformat/mp3enc.c b/libavformat/mp3enc.c
index 5e81f72a59..7b666b7db0 100644
--- a/libavformat/mp3enc.c
+++ b/libavformat/mp3enc.c
@@ -400,10 +400,10 @@  static int mp3_queue_flush(AVFormatContext *s)
 static void mp3_update_xing(AVFormatContext *s)
 {
     MP3Context  *mp3 = s->priv_data;
+    AVPacketSideData *sd;
     AVReplayGain *rg;
     uint16_t tag_crc;
     uint8_t *toc;
-    size_t rg_size;
     int i;
     int64_t old_pos = avio_tell(s->pb);
 
@@ -423,11 +423,11 @@  static void mp3_update_xing(AVFormatContext *s)
     }
 
     /* write replaygain */
-    rg = (AVReplayGain*)av_stream_get_side_data(s->streams[0], AV_PKT_DATA_REPLAYGAIN,
-                                                &rg_size);
-    if (rg && rg_size >= sizeof(*rg)) {
+    sd = av_packet_get_side_data_from_set(&s->streams[0]->codecpar->side_data_set, AV_PKT_DATA_REPLAYGAIN);
+    if (sd && sd->size >= sizeof(*rg)) {
         uint16_t val;
 
+        rg = (AVReplayGain *)sd->data;
         AV_WB32(mp3->xing_frame + mp3->xing_offset + 131,
                 av_rescale(rg->track_peak, 1 << 23, 100000));
 
diff --git a/libavformat/mpegenc.c b/libavformat/mpegenc.c
index c06e308296..b28486821b 100644
--- a/libavformat/mpegenc.c
+++ b/libavformat/mpegenc.c
@@ -342,8 +342,6 @@  static av_cold int mpeg_mux_init(AVFormatContext *ctx)
     lpcm_id = LPCM_ID;
 
     for (i = 0; i < ctx->nb_streams; i++) {
-        AVCPBProperties *props;
-
         st     = ctx->streams[i];
         stream = av_mallocz(sizeof(StreamInfo));
         if (!stream)
@@ -431,12 +429,16 @@  static av_cold int mpeg_mux_init(AVFormatContext *ctx)
             s->audio_bound++;
             break;
         case AVMEDIA_TYPE_VIDEO:
+            AVPacketSideData *sd;
+            AVCPBProperties *props = NULL;
             if (st->codecpar->codec_id == AV_CODEC_ID_H264)
                 stream->id = h264_id++;
             else
                 stream->id = mpv_id++;
 
-            props = (AVCPBProperties*)av_stream_get_side_data(st, AV_PKT_DATA_CPB_PROPERTIES, NULL);
+            sd = av_packet_get_side_data_from_set(&st->codecpar->side_data_set, AV_PKT_DATA_CPB_PROPERTIES);
+            if (sd)
+                props = (AVCPBProperties*)sd->data;
             if (props && props->buffer_size)
                 stream->max_buffer_size = 6 * 1024 + props->buffer_size / 8;
             else {
@@ -470,12 +472,15 @@  static av_cold int mpeg_mux_init(AVFormatContext *ctx)
     audio_bitrate = 0;
     video_bitrate = 0;
     for (i = 0; i < ctx->nb_streams; i++) {
-        AVCPBProperties *props;
+        AVPacketSideData *sd;
+        AVCPBProperties *props = NULL;
         int codec_rate;
         st     = ctx->streams[i];
         stream = (StreamInfo *)st->priv_data;
 
-        props = (AVCPBProperties*)av_stream_get_side_data(st, AV_PKT_DATA_CPB_PROPERTIES, NULL);
+        sd = av_packet_get_side_data_from_set(&st->codecpar->side_data_set, AV_PKT_DATA_CPB_PROPERTIES);
+        if (sd)
+            props = (AVCPBProperties*)sd->data;
         if (props)
             codec_rate = props->max_bitrate;
         else
diff --git a/libavformat/mpegts.c b/libavformat/mpegts.c
index 0b3edda817..093ee98447 100644
--- a/libavformat/mpegts.c
+++ b/libavformat/mpegts.c
@@ -2221,7 +2221,7 @@  int ff_parse_mpeg2_descriptor(AVFormatContext *fc, AVStream *st, int stream_type
                 dovi->dv_bl_signal_compatibility_id = 0;
             }
 
-            ret = av_stream_add_side_data(st, AV_PKT_DATA_DOVI_CONF,
+            ret = av_packet_add_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_DOVI_CONF,
                                           (uint8_t *)dovi, dovi_size);
             if (ret < 0) {
                 av_free(dovi);
diff --git a/libavformat/mux.c b/libavformat/mux.c
index 415bd3948f..21ab32dc99 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -277,6 +277,25 @@  FF_ENABLE_DEPRECATION_WARNINGS
             break;
         }
 
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
+        /* if the caller is using the deprecated AVStream side_data API,
+         * copy its contents to AVStream.codecpar, giving it priority
+           over existing side data in the latter */
+        for (int i = 0; i < st->nb_side_data; i++) {
+            const AVPacketSideData *sd_src = &st->side_data[i];
+            AVPacketSideData *sd_dst;
+
+            sd_dst = av_packet_new_side_data_to_set(&st->codecpar->side_data_set, sd_src->type, sd_src->size);
+            if (!sd_dst) {
+                ret = AVERROR(EINVAL);
+                goto fail;
+            }
+            memcpy(sd_dst->data, sd_src->data, sd_src->size);
+        }
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
+
         desc = avcodec_descriptor_get(par->codec_id);
         if (desc && desc->props & AV_CODEC_PROP_REORDER)
             sti->reorder = 1;
diff --git a/libavformat/mxfdec.c b/libavformat/mxfdec.c
index 34230ece98..bb542207d1 100644
--- a/libavformat/mxfdec.c
+++ b/libavformat/mxfdec.c
@@ -2577,10 +2577,11 @@  static int parse_mca_labels(MXFContext *mxf, MXFTrack *source_track, MXFDescript
 
     if (service_type != AV_AUDIO_SERVICE_TYPE_NB && service_type != AV_AUDIO_SERVICE_TYPE_MAIN && !ambigous_service_type) {
         enum AVAudioServiceType *ast;
-        uint8_t* side_data = av_stream_new_side_data(st, AV_PKT_DATA_AUDIO_SERVICE_TYPE, sizeof(*ast));
+        AVPacketSideData *side_data = av_packet_new_side_data_to_set(&st->codecpar->side_data_set,
+                                                                     AV_PKT_DATA_AUDIO_SERVICE_TYPE, sizeof(*ast));
         if (!side_data)
             return AVERROR(ENOMEM);
-        ast = (enum AVAudioServiceType*)side_data;
+        ast = (enum AVAudioServiceType*)side_data->data;
         *ast = service_type;
     }
 
@@ -2980,7 +2981,7 @@  static int mxf_parse_structural_metadata(MXFContext *mxf)
             st->codecpar->color_trc       = mxf_get_codec_ul(ff_mxf_color_trc_uls, &descriptor->color_trc_ul)->id;
             st->codecpar->color_space     = mxf_get_codec_ul(ff_mxf_color_space_uls, &descriptor->color_space_ul)->id;
             if (descriptor->mastering) {
-                ret = av_stream_add_side_data(st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
+                ret = av_packet_add_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
                                               (uint8_t *)descriptor->mastering,
                                               sizeof(*descriptor->mastering));
                 if (ret < 0)
@@ -2988,7 +2989,7 @@  static int mxf_parse_structural_metadata(MXFContext *mxf)
                 descriptor->mastering = NULL;
             }
             if (descriptor->coll) {
-                ret = av_stream_add_side_data(st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
+                ret = av_packet_add_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
                                               (uint8_t *)descriptor->coll,
                                               descriptor->coll_size);
                 if (ret < 0)
diff --git a/libavformat/mxfenc.c b/libavformat/mxfenc.c
index d8252ed68f..8ed4b0a49f 100644
--- a/libavformat/mxfenc.c
+++ b/libavformat/mxfenc.c
@@ -556,7 +556,7 @@  static void mxf_write_primer_pack(AVFormatContext *s)
         if (s->streams[i]->codecpar->codec_id == AV_CODEC_ID_H264 && !sc->avc_intra) {
             will_have_avc_tags = 1;
         }
-        if (av_stream_get_side_data(s->streams[i], AV_PKT_DATA_MASTERING_DISPLAY_METADATA, NULL)) {
+        if (av_packet_get_side_data_from_set(&s->streams[i]->codecpar->side_data_set, AV_PKT_DATA_MASTERING_DISPLAY_METADATA)) {
             will_have_mastering_tags = 1;
         }
         if (s->streams[i]->codecpar->codec_id == AV_CODEC_ID_FFV1) {
@@ -1158,7 +1158,7 @@  static int64_t mxf_write_cdci_common(AVFormatContext *s, AVStream *st, const UID
     const MXFCodecUL *color_trc_ul;
     const MXFCodecUL *color_space_ul;
     int64_t pos = mxf_write_generic_desc(s, st, key);
-    uint8_t *side_data;
+    AVPacketSideData *side_data;
 
     color_primaries_ul = mxf_get_codec_ul_by_id(ff_mxf_color_primaries_uls, st->codecpar->color_primaries);
     color_trc_ul       = mxf_get_codec_ul_by_id(ff_mxf_color_trc_uls, st->codecpar->color_trc);
@@ -1344,9 +1344,9 @@  static int64_t mxf_write_cdci_common(AVFormatContext *s, AVStream *st, const UID
     avio_write(pb, *sc->codec_ul, 16);
 
     // Mastering Display metadata
-    side_data = av_stream_get_side_data(st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA, NULL);
+    side_data = av_packet_get_side_data_from_set(&st->codecpar->side_data_set, AV_PKT_DATA_MASTERING_DISPLAY_METADATA);
     if (side_data) {
-        const AVMasteringDisplayMetadata *metadata = (const AVMasteringDisplayMetadata*)side_data;
+        const AVMasteringDisplayMetadata *metadata = (const AVMasteringDisplayMetadata*)side_data->data;
         if (metadata->has_primaries) {
             mxf_write_local_tag(s, 12, 0x8301);
             avio_wb16(pb, rescale_mastering_chroma(metadata->display_primaries[0][0]));
diff --git a/libavformat/options.c b/libavformat/options.c
index e4a3aceed0..ef0b593d36 100644
--- a/libavformat/options.c
+++ b/libavformat/options.c
@@ -309,7 +309,9 @@  AVStream *avformat_new_stream(AVFormatContext *s, const AVCodec *c)
 
     st->sample_aspect_ratio = (AVRational) { 0, 1 };
 
+#if FF_API_AVSTREAM_SIDE_DATA
     sti->inject_global_side_data = si->inject_global_side_data;
+#endif
 
     sti->need_context_update = 1;
 
diff --git a/libavformat/replaygain.c b/libavformat/replaygain.c
index 915bcb2382..302bd75aec 100644
--- a/libavformat/replaygain.c
+++ b/libavformat/replaygain.c
@@ -69,16 +69,18 @@  static int32_t parse_value(const char *value, int32_t min)
 int ff_replaygain_export_raw(AVStream *st, int32_t tg, uint32_t tp,
                              int32_t ag, uint32_t ap)
 {
+    AVPacketSideData *sd;
     AVReplayGain *replaygain;
 
     if (tg == INT32_MIN && ag == INT32_MIN)
         return 0;
 
-    replaygain = (AVReplayGain*)av_stream_new_side_data(st, AV_PKT_DATA_REPLAYGAIN,
+    sd = av_packet_new_side_data_to_set(&st->codecpar->side_data_set, AV_PKT_DATA_REPLAYGAIN,
                                                         sizeof(*replaygain));
-    if (!replaygain)
+    if (!sd)
         return AVERROR(ENOMEM);
 
+    replaygain = (AVReplayGain*)sd->data;
     replaygain->track_gain = tg;
     replaygain->track_peak = tp;
     replaygain->album_gain = ag;
diff --git a/libavformat/seek.c b/libavformat/seek.c
index 386312cd3a..0180188595 100644
--- a/libavformat/seek.c
+++ b/libavformat/seek.c
@@ -745,8 +745,10 @@  void ff_read_frame_flush(AVFormatContext *s)
         for (int j = 0; j < MAX_REORDER_DELAY + 1; j++)
             sti->pts_buffer[j] = AV_NOPTS_VALUE;
 
+#if FF_API_AVSTREAM_SIDE_DATA
         if (si->inject_global_side_data)
             sti->inject_global_side_data = 1;
+#endif
 
         sti->skip_samples = 0;
     }
diff --git a/libavformat/version_major.h b/libavformat/version_major.h
index 293fbd3397..c348e3eb37 100644
--- a/libavformat/version_major.h
+++ b/libavformat/version_major.h
@@ -45,6 +45,7 @@ 
 #define FF_API_GET_END_PTS              (LIBAVFORMAT_VERSION_MAJOR < 61)
 #define FF_API_AVIODIRCONTEXT           (LIBAVFORMAT_VERSION_MAJOR < 61)
 #define FF_API_AVFORMAT_IO_CLOSE        (LIBAVFORMAT_VERSION_MAJOR < 61)
+#define FF_API_AVSTREAM_SIDE_DATA       (LIBAVFORMAT_VERSION_MAJOR < 61)
 
 
 #define FF_API_R_FRAME_RATE            1