diff mbox series

[FFmpeg-devel,03/10] avformat/avformat: use the side data from AVStream.codecpar

Message ID 20230906174431.45558-4-jamrial@gmail.com
State New
Headers show
Series AVCodecContext and AVCodecParameters side data | expand

Commit Message

James Almer Sept. 6, 2023, 5:44 p.m. UTC
Deprecate AVStream.side_data and its helpers in favor of the AVStream's
codecpar.side_data.

This will considerably simplify the propagation of global side data to decoders
and from encoders. Instead of having to do it inside packets, it will be
available during init().
Global and frame specific side data will therefore be distinct.

Signed-off-by: James Almer <jamrial@gmail.com>
---
 libavdevice/android_camera.c |  9 ++--
 libavformat/avformat.c       | 42 ++++---------------
 libavformat/avformat.h       | 40 +++++++++++++++++-
 libavformat/concatdec.c      |  1 -
 libavformat/dashdec.c        | 11 -----
 libavformat/demux.c          | 54 +++++++++++++++---------
 libavformat/demux_utils.c    |  4 ++
 libavformat/dovi_isom.c      |  8 ++--
 libavformat/dump.c           |  6 +--
 libavformat/hls.c            | 11 -----
 libavformat/hlsenc.c         | 11 ++---
 libavformat/internal.h       |  4 ++
 libavformat/matroskadec.c    | 45 ++++++++++----------
 libavformat/matroskaenc.c    | 48 ++++++++++++---------
 libavformat/mov.c            | 81 ++++++++++++++++++------------------
 libavformat/movenc.c         | 73 +++++++++++++++-----------------
 libavformat/mp3enc.c         |  8 ++--
 libavformat/mpegenc.c        | 18 +++++---
 libavformat/mpegts.c         |  8 ++--
 libavformat/mux.c            | 19 +++++++++
 libavformat/mxfdec.c         | 22 +++++-----
 libavformat/mxfenc.c         |  8 ++--
 libavformat/options.c        |  2 +
 libavformat/replaygain.c     |  9 ++--
 libavformat/seek.c           |  2 +
 libavformat/version_major.h  |  1 +
 26 files changed, 294 insertions(+), 251 deletions(-)

Comments

Andreas Rheinhardt Sept. 11, 2023, 7:19 p.m. UTC | #1
James Almer:
> Deprecate AVStream.side_data and its helpers in favor of the AVStream's
> codecpar.side_data.
> 
> This will considerably simplify the propagation of global side data to decoders
> and from encoders. Instead of having to do it inside packets, it will be
> available during init().
> Global and frame specific side data will therefore be distinct.
> 
> Signed-off-by: James Almer <jamrial@gmail.com>
> ---> diff --git a/libavformat/avformat.h b/libavformat/avformat.h
> index 1916aa2dc5..f78a027f64 100644
> --- a/libavformat/avformat.h
> +++ b/libavformat/avformat.h
> @@ -164,6 +164,13 @@
>   * decoding functions avcodec_send_packet() or avcodec_decode_subtitle2() if the
>   * caller wishes to decode the data.
>   *
> + * There may be no overlap between the stream's @ref AVCodecParameters.side_data
> + * "side data" and @ref AVPacket.side_data "side data" in packets. I.e. a given
> + * side data is either exported by the demuxer in AVCodecParameters, then it never
> + * appears in the packets, or the side data is exported through the packets (always
> + * in the first packet where the value becomes known or changes), then it does not
> + * appear in AVCodecParameters.
> + *

Is it actually certain that our demuxers currently abide by this? E.g.
in mpegts, stream parameters can change at any time, so why does it set
it at the stream level and not the packet level?

>   * AVPacket.pts, AVPacket.dts and AVPacket.duration timing information will be
>   * set if known. They may also be unset (i.e. AV_NOPTS_VALUE for
>   * pts/dts, 0 for duration) if the stream does not provide them. The timing
> @@ -209,6 +216,11 @@
>   *   AVCodecParameters, rather than using @ref avcodec_parameters_copy() during
>   *   remuxing: there is no guarantee that the codec context values remain valid
>   *   for both input and output format contexts.
> + * - There may be no overlap between AVCodecParameters.side_data and side data in
> + *   packets. I.e. a given side data is either set by the caller in
> + *   AVCodecParameters, then it never appears in the packets, or the side data is
> + *   sent through the packets (always in the first packet where the value becomes
> + *   known or changes), then it does not appear in AVCodecParameters.

I have to say, I don't really like this (and of course I am aware that
you are basically copying the doxy of AVPacketSideData here). As you
know, the Matroska muxer needs to add header fields in order to add
certain packet side data to blocks later. In case of seekable output,
one can update the header later, but in case of unseekable output that
is not true. I'd like there to be an easy way for the user to signal the
intention to send packet side data of a specific type later.

>   * - The caller may fill in additional information, such as @ref
>   *   AVFormatContext.metadata "global" or @ref AVStream.metadata "per-stream"
>   *   metadata, @ref AVFormatContext.chapters "chapters", @ref
> @@ -937,6 +949,7 @@ typedef struct AVStream {
>       */
>      AVPacket attached_pic;
>  
> +#if FF_API_AVSTREAM_SIDE_DATA
>      /**
>       * An array of side data that applies to the whole stream (i.e. the
>       * container does not allow it to change between packets).
> @@ -953,13 +966,20 @@ typedef struct AVStream {
>       *
>       * Freed by libavformat in avformat_free_context().
>       *
> -     * @see av_format_inject_global_side_data()
> +     * @deprecated use AVStream's @ref AVCodecParameters.side_data
> +     *             "codecpar side data".
>       */
> +    attribute_deprecated
>      AVPacketSideData *side_data;
>      /**
>       * The number of elements in the AVStream.side_data array.
> +     *
> +     * @deprecated use AVStream's @ref AVCodecParameters.side_data
> +     *             "codecpar side data".
>       */
> +    attribute_deprecated
>      int            nb_side_data;
> +#endif
>  
>      /**
>       * Flags indicating events happening on the stream, a combination of
> @@ -1715,11 +1735,18 @@ typedef struct AVFormatContext {
>      int (*io_close2)(struct AVFormatContext *s, AVIOContext *pb);
>  } AVFormatContext;
>  
> +#if FF_API_AVSTREAM_SIDE_DATA
>  /**
>   * This function will cause global side data to be injected in the next packet
>   * of each stream as well as after any subsequent seek.
> + *
> + * @deprecated global side data is always available in every AVStream's
> + *             @ref AVCodecParameters.side_data "codecpar side data" array.
> + * @see av_packet_side_data_set_get()
>   */
> +attribute_deprecated
>  void av_format_inject_global_side_data(AVFormatContext *s);
> +#endif
>  
>  /**
>   * Returns the method used to set ctx->duration.
> @@ -1844,6 +1871,7 @@ const AVClass *av_stream_get_class(void);
>   */
>  AVStream *avformat_new_stream(AVFormatContext *s, const AVCodec *c);
>  
> +#if FF_API_AVSTREAM_SIDE_DATA
>  /**
>   * Wrap an existing array as stream side data.
>   *
> @@ -1856,7 +1884,10 @@ AVStream *avformat_new_stream(AVFormatContext *s, const AVCodec *c);
>   *
>   * @return zero on success, a negative AVERROR code on failure. On failure,
>   *         the stream is unchanged and the data remains owned by the caller.
> + * @deprecated use av_packet_side_data_set_add() with the stream's
> + *             @ref AVCodecParameters.side_data "codecpar side data"
>   */
> +attribute_deprecated
>  int av_stream_add_side_data(AVStream *st, enum AVPacketSideDataType type,
>                              uint8_t *data, size_t size);
>  
> @@ -1868,7 +1899,10 @@ int av_stream_add_side_data(AVStream *st, enum AVPacketSideDataType type,
>   * @param size   side information size
>   *
>   * @return pointer to fresh allocated data or NULL otherwise
> + * @deprecated use av_packet_side_data_set_new() with the stream's
> + *             @ref AVCodecParameters.side_data "codecpar side data"
>   */
> +attribute_deprecated
>  uint8_t *av_stream_new_side_data(AVStream *stream,
>                                   enum AVPacketSideDataType type, size_t size);
>  /**
> @@ -1880,9 +1914,13 @@ uint8_t *av_stream_new_side_data(AVStream *stream,
>   *               or to zero if the desired side data is not present.
>   *
>   * @return pointer to data if present or NULL otherwise
> + * @deprecated use av_packet_side_data_set_get() with the stream's
> + *             @ref AVCodecParameters.side_data "codecpar side data"
>   */
> +attribute_deprecated
>  uint8_t *av_stream_get_side_data(const AVStream *stream,
>                                   enum AVPacketSideDataType type, size_t *size);
> +#endif
>  
>  AVProgram *av_new_program(AVFormatContext *s, int id);
>
James Almer Sept. 12, 2023, 4:27 p.m. UTC | #2
On 9/11/2023 4:19 PM, Andreas Rheinhardt wrote:
> James Almer:
>> Deprecate AVStream.side_data and its helpers in favor of the AVStream's
>> codecpar.side_data.
>>
>> This will considerably simplify the propagation of global side data to decoders
>> and from encoders. Instead of having to do it inside packets, it will be
>> available during init().
>> Global and frame specific side data will therefore be distinct.
>>
>> Signed-off-by: James Almer <jamrial@gmail.com>
>> ---> diff --git a/libavformat/avformat.h b/libavformat/avformat.h
>> index 1916aa2dc5..f78a027f64 100644
>> --- a/libavformat/avformat.h
>> +++ b/libavformat/avformat.h
>> @@ -164,6 +164,13 @@
>>    * decoding functions avcodec_send_packet() or avcodec_decode_subtitle2() if the
>>    * caller wishes to decode the data.
>>    *
>> + * There may be no overlap between the stream's @ref AVCodecParameters.side_data
>> + * "side data" and @ref AVPacket.side_data "side data" in packets. I.e. a given
>> + * side data is either exported by the demuxer in AVCodecParameters, then it never
>> + * appears in the packets, or the side data is exported through the packets (always
>> + * in the first packet where the value becomes known or changes), then it does not
>> + * appear in AVCodecParameters.
>> + *
> 
> Is it actually certain that our demuxers currently abide by this? E.g.
> in mpegts, stream parameters can change at any time, so why does it set
> it at the stream level and not the packet level?

Does mpegts change AVStream.codecpar mid demuxing? wouldn't that be an 
API violation? I assumed that was what AV_PKT_DATA_PARAM_CHANGE and 
AV_PKT_DATA_NEW_EXTRADATA packet side data were for.

> 
>>    * AVPacket.pts, AVPacket.dts and AVPacket.duration timing information will be
>>    * set if known. They may also be unset (i.e. AV_NOPTS_VALUE for
>>    * pts/dts, 0 for duration) if the stream does not provide them. The timing
>> @@ -209,6 +216,11 @@
>>    *   AVCodecParameters, rather than using @ref avcodec_parameters_copy() during
>>    *   remuxing: there is no guarantee that the codec context values remain valid
>>    *   for both input and output format contexts.
>> + * - There may be no overlap between AVCodecParameters.side_data and side data in
>> + *   packets. I.e. a given side data is either set by the caller in
>> + *   AVCodecParameters, then it never appears in the packets, or the side data is
>> + *   sent through the packets (always in the first packet where the value becomes
>> + *   known or changes), then it does not appear in AVCodecParameters.
> 
> I have to say, I don't really like this (and of course I am aware that
> you are basically copying the doxy of AVPacketSideData here). As you

I can remove this part, to be added later if needed.

> know, the Matroska muxer needs to add header fields in order to add
> certain packet side data to blocks later. In case of seekable output,
> one can update the header later, but in case of unseekable output that
> is not true. I'd like there to be an easy way for the user to signal the
> intention to send packet side data of a specific type later.

Maybe a new AVFMT_FLAG_?

> 
>>    * - The caller may fill in additional information, such as @ref
>>    *   AVFormatContext.metadata "global" or @ref AVStream.metadata "per-stream"
>>    *   metadata, @ref AVFormatContext.chapters "chapters", @ref
>> @@ -937,6 +949,7 @@ typedef struct AVStream {
>>        */
>>       AVPacket attached_pic;
>>   
>> +#if FF_API_AVSTREAM_SIDE_DATA
>>       /**
>>        * An array of side data that applies to the whole stream (i.e. the
>>        * container does not allow it to change between packets).
>> @@ -953,13 +966,20 @@ typedef struct AVStream {
>>        *
>>        * Freed by libavformat in avformat_free_context().
>>        *
>> -     * @see av_format_inject_global_side_data()
>> +     * @deprecated use AVStream's @ref AVCodecParameters.side_data
>> +     *             "codecpar side data".
>>        */
>> +    attribute_deprecated
>>       AVPacketSideData *side_data;
>>       /**
>>        * The number of elements in the AVStream.side_data array.
>> +     *
>> +     * @deprecated use AVStream's @ref AVCodecParameters.side_data
>> +     *             "codecpar side data".
>>        */
>> +    attribute_deprecated
>>       int            nb_side_data;
>> +#endif
>>   
>>       /**
>>        * Flags indicating events happening on the stream, a combination of
>> @@ -1715,11 +1735,18 @@ typedef struct AVFormatContext {
>>       int (*io_close2)(struct AVFormatContext *s, AVIOContext *pb);
>>   } AVFormatContext;
>>   
>> +#if FF_API_AVSTREAM_SIDE_DATA
>>   /**
>>    * This function will cause global side data to be injected in the next packet
>>    * of each stream as well as after any subsequent seek.
>> + *
>> + * @deprecated global side data is always available in every AVStream's
>> + *             @ref AVCodecParameters.side_data "codecpar side data" array.
>> + * @see av_packet_side_data_set_get()
>>    */
>> +attribute_deprecated
>>   void av_format_inject_global_side_data(AVFormatContext *s);
>> +#endif
>>   
>>   /**
>>    * Returns the method used to set ctx->duration.
>> @@ -1844,6 +1871,7 @@ const AVClass *av_stream_get_class(void);
>>    */
>>   AVStream *avformat_new_stream(AVFormatContext *s, const AVCodec *c);
>>   
>> +#if FF_API_AVSTREAM_SIDE_DATA
>>   /**
>>    * Wrap an existing array as stream side data.
>>    *
>> @@ -1856,7 +1884,10 @@ AVStream *avformat_new_stream(AVFormatContext *s, const AVCodec *c);
>>    *
>>    * @return zero on success, a negative AVERROR code on failure. On failure,
>>    *         the stream is unchanged and the data remains owned by the caller.
>> + * @deprecated use av_packet_side_data_set_add() with the stream's
>> + *             @ref AVCodecParameters.side_data "codecpar side data"
>>    */
>> +attribute_deprecated
>>   int av_stream_add_side_data(AVStream *st, enum AVPacketSideDataType type,
>>                               uint8_t *data, size_t size);
>>   
>> @@ -1868,7 +1899,10 @@ int av_stream_add_side_data(AVStream *st, enum AVPacketSideDataType type,
>>    * @param size   side information size
>>    *
>>    * @return pointer to fresh allocated data or NULL otherwise
>> + * @deprecated use av_packet_side_data_set_new() with the stream's
>> + *             @ref AVCodecParameters.side_data "codecpar side data"
>>    */
>> +attribute_deprecated
>>   uint8_t *av_stream_new_side_data(AVStream *stream,
>>                                    enum AVPacketSideDataType type, size_t size);
>>   /**
>> @@ -1880,9 +1914,13 @@ uint8_t *av_stream_new_side_data(AVStream *stream,
>>    *               or to zero if the desired side data is not present.
>>    *
>>    * @return pointer to data if present or NULL otherwise
>> + * @deprecated use av_packet_side_data_set_get() with the stream's
>> + *             @ref AVCodecParameters.side_data "codecpar side data"
>>    */
>> +attribute_deprecated
>>   uint8_t *av_stream_get_side_data(const AVStream *stream,
>>                                    enum AVPacketSideDataType type, size_t *size);
>> +#endif
>>   
>>   AVProgram *av_new_program(AVFormatContext *s, int id);
>>   
> 
> 
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Andreas Rheinhardt Sept. 12, 2023, 4:43 p.m. UTC | #3
James Almer:
> On 9/11/2023 4:19 PM, Andreas Rheinhardt wrote:
>> James Almer:
>>> Deprecate AVStream.side_data and its helpers in favor of the AVStream's
>>> codecpar.side_data.
>>>
>>> This will considerably simplify the propagation of global side data
>>> to decoders
>>> and from encoders. Instead of having to do it inside packets, it will be
>>> available during init().
>>> Global and frame specific side data will therefore be distinct.
>>>
>>> Signed-off-by: James Almer <jamrial@gmail.com>
>>> ---> diff --git a/libavformat/avformat.h b/libavformat/avformat.h
>>> index 1916aa2dc5..f78a027f64 100644
>>> --- a/libavformat/avformat.h
>>> +++ b/libavformat/avformat.h
>>> @@ -164,6 +164,13 @@
>>>    * decoding functions avcodec_send_packet() or
>>> avcodec_decode_subtitle2() if the
>>>    * caller wishes to decode the data.
>>>    *
>>> + * There may be no overlap between the stream's @ref
>>> AVCodecParameters.side_data
>>> + * "side data" and @ref AVPacket.side_data "side data" in packets.
>>> I.e. a given
>>> + * side data is either exported by the demuxer in AVCodecParameters,
>>> then it never
>>> + * appears in the packets, or the side data is exported through the
>>> packets (always
>>> + * in the first packet where the value becomes known or changes),
>>> then it does not
>>> + * appear in AVCodecParameters.
>>> + *
>>
>> Is it actually certain that our demuxers currently abide by this? E.g.
>> in mpegts, stream parameters can change at any time, so why does it set
>> it at the stream level and not the packet level?
> 
> Does mpegts change AVStream.codecpar mid demuxing? wouldn't that be an
> API violation? I assumed that was what AV_PKT_DATA_PARAM_CHANGE and
> AV_PKT_DATA_NEW_EXTRADATA packet side data were for.
> 

It seems that the dovi stream side data can be added (and potentially
even replaced) long after the stream has been created.
(The concat demuxer may even set extradata lateron and even change it
after it has been allocated; see match_streams() in
concat_read_packet(). The concat demuxer should actually add the global
side data of every input to the first packet from said input which means
that copying side data (whether in ff_stream_side_data_copy() or in
avcodec_parameters_copy() is potentially problematic.)

>>
>>>    * AVPacket.pts, AVPacket.dts and AVPacket.duration timing
>>> information will be
>>>    * set if known. They may also be unset (i.e. AV_NOPTS_VALUE for
>>>    * pts/dts, 0 for duration) if the stream does not provide them.
>>> The timing
>>> @@ -209,6 +216,11 @@
>>>    *   AVCodecParameters, rather than using @ref
>>> avcodec_parameters_copy() during
>>>    *   remuxing: there is no guarantee that the codec context values
>>> remain valid
>>>    *   for both input and output format contexts.
>>> + * - There may be no overlap between AVCodecParameters.side_data and
>>> side data in
>>> + *   packets. I.e. a given side data is either set by the caller in
>>> + *   AVCodecParameters, then it never appears in the packets, or the
>>> side data is
>>> + *   sent through the packets (always in the first packet where the
>>> value becomes
>>> + *   known or changes), then it does not appear in AVCodecParameters.
>>
>> I have to say, I don't really like this (and of course I am aware that
>> you are basically copying the doxy of AVPacketSideData here). As you
> 
> I can remove this part, to be added later if needed.
> 
>> know, the Matroska muxer needs to add header fields in order to add
>> certain packet side data to blocks later. In case of seekable output,
>> one can update the header later, but in case of unseekable output that
>> is not true. I'd like there to be an easy way for the user to signal the
>> intention to send packet side data of a specific type later.
> 
> Maybe a new AVFMT_FLAG_?
> 

That would potentially be a new AVFMT_FLAG_ for every side data type;
that makes no sense.

>>
>>>    * - The caller may fill in additional information, such as @ref
>>>    *   AVFormatContext.metadata "global" or @ref AVStream.metadata
>>> "per-stream"
>>>    *   metadata, @ref AVFormatContext.chapters "chapters", @ref
>>> @@ -937,6 +949,7 @@ typedef struct AVStream {
>>>        */
>>>       AVPacket attached_pic;
>>>   +#if FF_API_AVSTREAM_SIDE_DATA
>>>       /**
>>>        * An array of side data that applies to the whole stream (i.e.
>>> the
>>>        * container does not allow it to change between packets).
>>> @@ -953,13 +966,20 @@ typedef struct AVStream {
>>>        *
>>>        * Freed by libavformat in avformat_free_context().
>>>        *
>>> -     * @see av_format_inject_global_side_data()
>>> +     * @deprecated use AVStream's @ref AVCodecParameters.side_data
>>> +     *             "codecpar side data".
>>>        */
>>> +    attribute_deprecated
>>>       AVPacketSideData *side_data;
>>>       /**
>>>        * The number of elements in the AVStream.side_data array.
>>> +     *
>>> +     * @deprecated use AVStream's @ref AVCodecParameters.side_data
>>> +     *             "codecpar side data".
>>>        */
>>> +    attribute_deprecated
>>>       int            nb_side_data;
>>> +#endif
>>>         /**
>>>        * Flags indicating events happening on the stream, a
>>> combination of
>>> @@ -1715,11 +1735,18 @@ typedef struct AVFormatContext {
>>>       int (*io_close2)(struct AVFormatContext *s, AVIOContext *pb);
>>>   } AVFormatContext;
>>>   +#if FF_API_AVSTREAM_SIDE_DATA
>>>   /**
>>>    * This function will cause global side data to be injected in the
>>> next packet
>>>    * of each stream as well as after any subsequent seek.
>>> + *
>>> + * @deprecated global side data is always available in every AVStream's
>>> + *             @ref AVCodecParameters.side_data "codecpar side data"
>>> array.
>>> + * @see av_packet_side_data_set_get()
>>>    */
>>> +attribute_deprecated
>>>   void av_format_inject_global_side_data(AVFormatContext *s);
>>> +#endif
>>>     /**
>>>    * Returns the method used to set ctx->duration.
>>> @@ -1844,6 +1871,7 @@ const AVClass *av_stream_get_class(void);
>>>    */
>>>   AVStream *avformat_new_stream(AVFormatContext *s, const AVCodec *c);
>>>   +#if FF_API_AVSTREAM_SIDE_DATA
>>>   /**
>>>    * Wrap an existing array as stream side data.
>>>    *
>>> @@ -1856,7 +1884,10 @@ AVStream *avformat_new_stream(AVFormatContext
>>> *s, const AVCodec *c);
>>>    *
>>>    * @return zero on success, a negative AVERROR code on failure. On
>>> failure,
>>>    *         the stream is unchanged and the data remains owned by
>>> the caller.
>>> + * @deprecated use av_packet_side_data_set_add() with the stream's
>>> + *             @ref AVCodecParameters.side_data "codecpar side data"
>>>    */
>>> +attribute_deprecated
>>>   int av_stream_add_side_data(AVStream *st, enum AVPacketSideDataType
>>> type,
>>>                               uint8_t *data, size_t size);
>>>   @@ -1868,7 +1899,10 @@ int av_stream_add_side_data(AVStream *st,
>>> enum AVPacketSideDataType type,
>>>    * @param size   side information size
>>>    *
>>>    * @return pointer to fresh allocated data or NULL otherwise
>>> + * @deprecated use av_packet_side_data_set_new() with the stream's
>>> + *             @ref AVCodecParameters.side_data "codecpar side data"
>>>    */
>>> +attribute_deprecated
>>>   uint8_t *av_stream_new_side_data(AVStream *stream,
>>>                                    enum AVPacketSideDataType type,
>>> size_t size);
>>>   /**
>>> @@ -1880,9 +1914,13 @@ uint8_t *av_stream_new_side_data(AVStream
>>> *stream,
>>>    *               or to zero if the desired side data is not present.
>>>    *
>>>    * @return pointer to data if present or NULL otherwise
>>> + * @deprecated use av_packet_side_data_set_get() with the stream's
>>> + *             @ref AVCodecParameters.side_data "codecpar side data"
>>>    */
>>> +attribute_deprecated
>>>   uint8_t *av_stream_get_side_data(const AVStream *stream,
>>>                                    enum AVPacketSideDataType type,
>>> size_t *size);
>>> +#endif
>>>     AVProgram *av_new_program(AVFormatContext *s, int id);
>>>   
>>
James Almer Sept. 12, 2023, 4:57 p.m. UTC | #4
On 9/12/2023 1:43 PM, Andreas Rheinhardt wrote:
> James Almer:
>> On 9/11/2023 4:19 PM, Andreas Rheinhardt wrote:
>>> James Almer:
>>>> Deprecate AVStream.side_data and its helpers in favor of the AVStream's
>>>> codecpar.side_data.
>>>>
>>>> This will considerably simplify the propagation of global side data
>>>> to decoders
>>>> and from encoders. Instead of having to do it inside packets, it will be
>>>> available during init().
>>>> Global and frame specific side data will therefore be distinct.
>>>>
>>>> Signed-off-by: James Almer <jamrial@gmail.com>
>>>> ---> diff --git a/libavformat/avformat.h b/libavformat/avformat.h
>>>> index 1916aa2dc5..f78a027f64 100644
>>>> --- a/libavformat/avformat.h
>>>> +++ b/libavformat/avformat.h
>>>> @@ -164,6 +164,13 @@
>>>>     * decoding functions avcodec_send_packet() or
>>>> avcodec_decode_subtitle2() if the
>>>>     * caller wishes to decode the data.
>>>>     *
>>>> + * There may be no overlap between the stream's @ref
>>>> AVCodecParameters.side_data
>>>> + * "side data" and @ref AVPacket.side_data "side data" in packets.
>>>> I.e. a given
>>>> + * side data is either exported by the demuxer in AVCodecParameters,
>>>> then it never
>>>> + * appears in the packets, or the side data is exported through the
>>>> packets (always
>>>> + * in the first packet where the value becomes known or changes),
>>>> then it does not
>>>> + * appear in AVCodecParameters.
>>>> + *
>>>
>>> Is it actually certain that our demuxers currently abide by this? E.g.
>>> in mpegts, stream parameters can change at any time, so why does it set
>>> it at the stream level and not the packet level?
>>
>> Does mpegts change AVStream.codecpar mid demuxing? wouldn't that be an
>> API violation? I assumed that was what AV_PKT_DATA_PARAM_CHANGE and
>> AV_PKT_DATA_NEW_EXTRADATA packet side data were for.
>>
> 
> It seems that the dovi stream side data can be added (and potentially
> even replaced) long after the stream has been created.

Yeah, as well as other codecpar fields. How is the library user even 
meant to know this happened? The doxy states codecpar is filled on 
stream creation or during avformat_find_stream_info(), so they would not 
expect it to change after that.

AV_PKT_DATA_PARAM_CHANGE seems to me that it's the proper way to 
propagate these changes, yet it looks underused and fuzzily defined.
Maybe it should be expanded and improved for this.

> (The concat demuxer may even set extradata lateron and even change it
> after it has been allocated; see match_streams() in
> concat_read_packet(). The concat demuxer should actually add the global
> side data of every input to the first packet from said input which means
> that copying side data (whether in ff_stream_side_data_copy() or in
> avcodec_parameters_copy() is potentially problematic.)
> 
>>>
>>>>     * AVPacket.pts, AVPacket.dts and AVPacket.duration timing
>>>> information will be
>>>>     * set if known. They may also be unset (i.e. AV_NOPTS_VALUE for
>>>>     * pts/dts, 0 for duration) if the stream does not provide them.
>>>> The timing
>>>> @@ -209,6 +216,11 @@
>>>>     *   AVCodecParameters, rather than using @ref
>>>> avcodec_parameters_copy() during
>>>>     *   remuxing: there is no guarantee that the codec context values
>>>> remain valid
>>>>     *   for both input and output format contexts.
>>>> + * - There may be no overlap between AVCodecParameters.side_data and
>>>> side data in
>>>> + *   packets. I.e. a given side data is either set by the caller in
>>>> + *   AVCodecParameters, then it never appears in the packets, or the
>>>> side data is
>>>> + *   sent through the packets (always in the first packet where the
>>>> value becomes
>>>> + *   known or changes), then it does not appear in AVCodecParameters.
>>>
>>> I have to say, I don't really like this (and of course I am aware that
>>> you are basically copying the doxy of AVPacketSideData here). As you
>>
>> I can remove this part, to be added later if needed.
>>
>>> know, the Matroska muxer needs to add header fields in order to add
>>> certain packet side data to blocks later. In case of seekable output,
>>> one can update the header later, but in case of unseekable output that
>>> is not true. I'd like there to be an easy way for the user to signal the
>>> intention to send packet side data of a specific type later.
>>
>> Maybe a new AVFMT_FLAG_?
>>
> 
> That would potentially be a new AVFMT_FLAG_ for every side data type;
> that makes no sense.

Yeah, missed the "specific type" part, and assumed you meant only a way 
to signal a simple "Keep an eye for side data in packets" scenario.
> 
>>>
>>>>     * - The caller may fill in additional information, such as @ref
>>>>     *   AVFormatContext.metadata "global" or @ref AVStream.metadata
>>>> "per-stream"
>>>>     *   metadata, @ref AVFormatContext.chapters "chapters", @ref
>>>> @@ -937,6 +949,7 @@ typedef struct AVStream {
>>>>         */
>>>>        AVPacket attached_pic;
>>>>    +#if FF_API_AVSTREAM_SIDE_DATA
>>>>        /**
>>>>         * An array of side data that applies to the whole stream (i.e.
>>>> the
>>>>         * container does not allow it to change between packets).
>>>> @@ -953,13 +966,20 @@ typedef struct AVStream {
>>>>         *
>>>>         * Freed by libavformat in avformat_free_context().
>>>>         *
>>>> -     * @see av_format_inject_global_side_data()
>>>> +     * @deprecated use AVStream's @ref AVCodecParameters.side_data
>>>> +     *             "codecpar side data".
>>>>         */
>>>> +    attribute_deprecated
>>>>        AVPacketSideData *side_data;
>>>>        /**
>>>>         * The number of elements in the AVStream.side_data array.
>>>> +     *
>>>> +     * @deprecated use AVStream's @ref AVCodecParameters.side_data
>>>> +     *             "codecpar side data".
>>>>         */
>>>> +    attribute_deprecated
>>>>        int            nb_side_data;
>>>> +#endif
>>>>          /**
>>>>         * Flags indicating events happening on the stream, a
>>>> combination of
>>>> @@ -1715,11 +1735,18 @@ typedef struct AVFormatContext {
>>>>        int (*io_close2)(struct AVFormatContext *s, AVIOContext *pb);
>>>>    } AVFormatContext;
>>>>    +#if FF_API_AVSTREAM_SIDE_DATA
>>>>    /**
>>>>     * This function will cause global side data to be injected in the
>>>> next packet
>>>>     * of each stream as well as after any subsequent seek.
>>>> + *
>>>> + * @deprecated global side data is always available in every AVStream's
>>>> + *             @ref AVCodecParameters.side_data "codecpar side data"
>>>> array.
>>>> + * @see av_packet_side_data_set_get()
>>>>     */
>>>> +attribute_deprecated
>>>>    void av_format_inject_global_side_data(AVFormatContext *s);
>>>> +#endif
>>>>      /**
>>>>     * Returns the method used to set ctx->duration.
>>>> @@ -1844,6 +1871,7 @@ const AVClass *av_stream_get_class(void);
>>>>     */
>>>>    AVStream *avformat_new_stream(AVFormatContext *s, const AVCodec *c);
>>>>    +#if FF_API_AVSTREAM_SIDE_DATA
>>>>    /**
>>>>     * Wrap an existing array as stream side data.
>>>>     *
>>>> @@ -1856,7 +1884,10 @@ AVStream *avformat_new_stream(AVFormatContext
>>>> *s, const AVCodec *c);
>>>>     *
>>>>     * @return zero on success, a negative AVERROR code on failure. On
>>>> failure,
>>>>     *         the stream is unchanged and the data remains owned by
>>>> the caller.
>>>> + * @deprecated use av_packet_side_data_set_add() with the stream's
>>>> + *             @ref AVCodecParameters.side_data "codecpar side data"
>>>>     */
>>>> +attribute_deprecated
>>>>    int av_stream_add_side_data(AVStream *st, enum AVPacketSideDataType
>>>> type,
>>>>                                uint8_t *data, size_t size);
>>>>    @@ -1868,7 +1899,10 @@ int av_stream_add_side_data(AVStream *st,
>>>> enum AVPacketSideDataType type,
>>>>     * @param size   side information size
>>>>     *
>>>>     * @return pointer to fresh allocated data or NULL otherwise
>>>> + * @deprecated use av_packet_side_data_set_new() with the stream's
>>>> + *             @ref AVCodecParameters.side_data "codecpar side data"
>>>>     */
>>>> +attribute_deprecated
>>>>    uint8_t *av_stream_new_side_data(AVStream *stream,
>>>>                                     enum AVPacketSideDataType type,
>>>> size_t size);
>>>>    /**
>>>> @@ -1880,9 +1914,13 @@ uint8_t *av_stream_new_side_data(AVStream
>>>> *stream,
>>>>     *               or to zero if the desired side data is not present.
>>>>     *
>>>>     * @return pointer to data if present or NULL otherwise
>>>> + * @deprecated use av_packet_side_data_set_get() with the stream's
>>>> + *             @ref AVCodecParameters.side_data "codecpar side data"
>>>>     */
>>>> +attribute_deprecated
>>>>    uint8_t *av_stream_get_side_data(const AVStream *stream,
>>>>                                     enum AVPacketSideDataType type,
>>>> size_t *size);
>>>> +#endif
>>>>      AVProgram *av_new_program(AVFormatContext *s, int id);
>>>>    
>>>
> 
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
diff mbox series

Patch

diff --git a/libavdevice/android_camera.c b/libavdevice/android_camera.c
index 1934999c18..012b40aa37 100644
--- a/libavdevice/android_camera.c
+++ b/libavdevice/android_camera.c
@@ -638,7 +638,7 @@  static int wait_for_image_format(AVFormatContext *avctx)
 static int add_display_matrix(AVFormatContext *avctx, AVStream *st)
 {
     AndroidCameraCtx *ctx = avctx->priv_data;
-    uint8_t *side_data;
+    AVPacketSideData *side_data;
     int32_t display_matrix[9];
 
     av_display_rotation_set(display_matrix, ctx->sensor_orientation);
@@ -647,14 +647,15 @@  static int add_display_matrix(AVFormatContext *avctx, AVStream *st)
         av_display_matrix_flip(display_matrix, 1, 0);
     }
 
-    side_data = av_stream_new_side_data(st,
-            AV_PKT_DATA_DISPLAYMATRIX, sizeof(display_matrix));
+    side_data = av_packet_side_data_set_new(&st->codecpar->side_data,
+                                            AV_PKT_DATA_DISPLAYMATRIX,
+                                            sizeof(display_matrix), 0);
 
     if (!side_data) {
         return AVERROR(ENOMEM);
     }
 
-    memcpy(side_data, display_matrix, sizeof(display_matrix));
+    memcpy(side_data->data, display_matrix, sizeof(display_matrix));
 
     return 0;
 }
diff --git a/libavformat/avformat.c b/libavformat/avformat.c
index 356b4de931..3afc5afd7f 100644
--- a/libavformat/avformat.c
+++ b/libavformat/avformat.c
@@ -46,9 +46,13 @@  void ff_free_stream(AVStream **pst)
     if (!st)
         return;
 
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
     for (int i = 0; i < st->nb_side_data; i++)
         av_freep(&st->side_data[i].data);
     av_freep(&st->side_data);
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
 
     if (st->attached_pic.data)
         av_packet_unref(&st->attached_pic);
@@ -138,6 +142,8 @@  void avformat_free_context(AVFormatContext *s)
     av_free(s);
 }
 
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
 uint8_t *av_stream_get_side_data(const AVStream *st,
                                  enum AVPacketSideDataType type, size_t *size)
 {
@@ -205,36 +211,8 @@  uint8_t *av_stream_new_side_data(AVStream *st, enum AVPacketSideDataType type,
 
     return data;
 }
-
-int ff_stream_side_data_copy(AVStream *dst, const AVStream *src)
-{
-    /* Free existing side data*/
-    for (int i = 0; i < dst->nb_side_data; i++)
-        av_free(dst->side_data[i].data);
-    av_freep(&dst->side_data);
-    dst->nb_side_data = 0;
-
-    /* Copy side data if present */
-    if (src->nb_side_data) {
-        dst->side_data = av_calloc(src->nb_side_data,
-                                   sizeof(*dst->side_data));
-        if (!dst->side_data)
-            return AVERROR(ENOMEM);
-        dst->nb_side_data = src->nb_side_data;
-
-        for (int i = 0; i < src->nb_side_data; i++) {
-            uint8_t *data = av_memdup(src->side_data[i].data,
-                                      src->side_data[i].size);
-            if (!data)
-                return AVERROR(ENOMEM);
-            dst->side_data[i].type = src->side_data[i].type;
-            dst->side_data[i].size = src->side_data[i].size;
-            dst->side_data[i].data = data;
-        }
-    }
-
-    return 0;
-}
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
 
 /**
  * Copy all stream parameters from source to destination stream, with the
@@ -270,10 +248,6 @@  static int stream_params_copy(AVStream *dst, const AVStream *src)
     if (ret < 0)
         return ret;
 
-    ret = ff_stream_side_data_copy(dst, src);
-    if (ret < 0)
-        return ret;
-
     av_packet_unref(&dst->attached_pic);
     if (src->attached_pic.data) {
         ret = av_packet_ref(&dst->attached_pic, &src->attached_pic);
diff --git a/libavformat/avformat.h b/libavformat/avformat.h
index 1916aa2dc5..f78a027f64 100644
--- a/libavformat/avformat.h
+++ b/libavformat/avformat.h
@@ -164,6 +164,13 @@ 
  * decoding functions avcodec_send_packet() or avcodec_decode_subtitle2() if the
  * caller wishes to decode the data.
  *
+ * There may be no overlap between the stream's @ref AVCodecParameters.side_data
+ * "side data" and @ref AVPacket.side_data "side data" in packets. I.e. a given
+ * side data is either exported by the demuxer in AVCodecParameters, then it never
+ * appears in the packets, or the side data is exported through the packets (always
+ * in the first packet where the value becomes known or changes), then it does not
+ * appear in AVCodecParameters.
+ *
  * AVPacket.pts, AVPacket.dts and AVPacket.duration timing information will be
  * set if known. They may also be unset (i.e. AV_NOPTS_VALUE for
  * pts/dts, 0 for duration) if the stream does not provide them. The timing
@@ -209,6 +216,11 @@ 
  *   AVCodecParameters, rather than using @ref avcodec_parameters_copy() during
  *   remuxing: there is no guarantee that the codec context values remain valid
  *   for both input and output format contexts.
+ * - There may be no overlap between AVCodecParameters.side_data and side data in
+ *   packets. I.e. a given side data is either set by the caller in
+ *   AVCodecParameters, then it never appears in the packets, or the side data is
+ *   sent through the packets (always in the first packet where the value becomes
+ *   known or changes), then it does not appear in AVCodecParameters.
  * - The caller may fill in additional information, such as @ref
  *   AVFormatContext.metadata "global" or @ref AVStream.metadata "per-stream"
  *   metadata, @ref AVFormatContext.chapters "chapters", @ref
@@ -937,6 +949,7 @@  typedef struct AVStream {
      */
     AVPacket attached_pic;
 
+#if FF_API_AVSTREAM_SIDE_DATA
     /**
      * An array of side data that applies to the whole stream (i.e. the
      * container does not allow it to change between packets).
@@ -953,13 +966,20 @@  typedef struct AVStream {
      *
      * Freed by libavformat in avformat_free_context().
      *
-     * @see av_format_inject_global_side_data()
+     * @deprecated use AVStream's @ref AVCodecParameters.side_data
+     *             "codecpar side data".
      */
+    attribute_deprecated
     AVPacketSideData *side_data;
     /**
      * The number of elements in the AVStream.side_data array.
+     *
+     * @deprecated use AVStream's @ref AVCodecParameters.side_data
+     *             "codecpar side data".
      */
+    attribute_deprecated
     int            nb_side_data;
+#endif
 
     /**
      * Flags indicating events happening on the stream, a combination of
@@ -1715,11 +1735,18 @@  typedef struct AVFormatContext {
     int (*io_close2)(struct AVFormatContext *s, AVIOContext *pb);
 } AVFormatContext;
 
+#if FF_API_AVSTREAM_SIDE_DATA
 /**
  * This function will cause global side data to be injected in the next packet
  * of each stream as well as after any subsequent seek.
+ *
+ * @deprecated global side data is always available in every AVStream's
+ *             @ref AVCodecParameters.side_data "codecpar side data" array.
+ * @see av_packet_side_data_set_get()
  */
+attribute_deprecated
 void av_format_inject_global_side_data(AVFormatContext *s);
+#endif
 
 /**
  * Returns the method used to set ctx->duration.
@@ -1844,6 +1871,7 @@  const AVClass *av_stream_get_class(void);
  */
 AVStream *avformat_new_stream(AVFormatContext *s, const AVCodec *c);
 
+#if FF_API_AVSTREAM_SIDE_DATA
 /**
  * Wrap an existing array as stream side data.
  *
@@ -1856,7 +1884,10 @@  AVStream *avformat_new_stream(AVFormatContext *s, const AVCodec *c);
  *
  * @return zero on success, a negative AVERROR code on failure. On failure,
  *         the stream is unchanged and the data remains owned by the caller.
+ * @deprecated use av_packet_side_data_set_add() with the stream's
+ *             @ref AVCodecParameters.side_data "codecpar side data"
  */
+attribute_deprecated
 int av_stream_add_side_data(AVStream *st, enum AVPacketSideDataType type,
                             uint8_t *data, size_t size);
 
@@ -1868,7 +1899,10 @@  int av_stream_add_side_data(AVStream *st, enum AVPacketSideDataType type,
  * @param size   side information size
  *
  * @return pointer to fresh allocated data or NULL otherwise
+ * @deprecated use av_packet_side_data_set_new() with the stream's
+ *             @ref AVCodecParameters.side_data "codecpar side data"
  */
+attribute_deprecated
 uint8_t *av_stream_new_side_data(AVStream *stream,
                                  enum AVPacketSideDataType type, size_t size);
 /**
@@ -1880,9 +1914,13 @@  uint8_t *av_stream_new_side_data(AVStream *stream,
  *               or to zero if the desired side data is not present.
  *
  * @return pointer to data if present or NULL otherwise
+ * @deprecated use av_packet_side_data_set_get() with the stream's
+ *             @ref AVCodecParameters.side_data "codecpar side data"
  */
+attribute_deprecated
 uint8_t *av_stream_get_side_data(const AVStream *stream,
                                  enum AVPacketSideDataType type, size_t *size);
+#endif
 
 AVProgram *av_new_program(AVFormatContext *s, int id);
 
diff --git a/libavformat/concatdec.c b/libavformat/concatdec.c
index 5d4f67d0ac..2aeb4f1dfd 100644
--- a/libavformat/concatdec.c
+++ b/libavformat/concatdec.c
@@ -194,7 +194,6 @@  static int copy_stream_props(AVStream *st, AVStream *source_st)
     avpriv_set_pts_info(st, 64, source_st->time_base.num, source_st->time_base.den);
 
     av_dict_copy(&st->metadata, source_st->metadata, 0);
-    ff_stream_side_data_copy(st, source_st);
     return 0;
 }
 
diff --git a/libavformat/dashdec.c b/libavformat/dashdec.c
index 29d4680c68..2441087606 100644
--- a/libavformat/dashdec.c
+++ b/libavformat/dashdec.c
@@ -1952,17 +1952,6 @@  static int open_demux_for_component(AVFormatContext *s, struct representation *p
 
         // copy disposition
         st->disposition = ist->disposition;
-
-        // copy side data
-        for (int i = 0; i < ist->nb_side_data; i++) {
-            const AVPacketSideData *sd_src = &ist->side_data[i];
-            uint8_t *dst_data;
-
-            dst_data = av_stream_new_side_data(st, sd_src->type, sd_src->size);
-            if (!dst_data)
-                return AVERROR(ENOMEM);
-            memcpy(dst_data, sd_src->data, sd_src->size);
-        }
     }
 
     return 0;
diff --git a/libavformat/demux.c b/libavformat/demux.c
index fcd5daf699..76d4558ad2 100644
--- a/libavformat/demux.c
+++ b/libavformat/demux.c
@@ -1409,9 +1409,10 @@  FF_ENABLE_DEPRECATION_WARNINGS
             sti->skip_samples = 0;
         }
 
+#if FF_API_AVSTREAM_SIDE_DATA
         if (sti->inject_global_side_data) {
-            for (int i = 0; i < st->nb_side_data; i++) {
-                const AVPacketSideData *const src_sd = &st->side_data[i];
+            for (int i = 0; i < st->codecpar->side_data.nb_sd; i++) {
+                const AVPacketSideData *const src_sd = st->codecpar->side_data.sd[i];
                 uint8_t *dst_data;
 
                 if (av_packet_get_side_data(pkt, src_sd->type, NULL))
@@ -1427,6 +1428,7 @@  FF_ENABLE_DEPRECATION_WARNINGS
             }
             sti->inject_global_side_data = 0;
         }
+#endif
     }
 
     if (!si->metafree) {
@@ -2431,19 +2433,6 @@  static int extract_extradata(FFFormatContext *si, AVStream *st, const AVPacket *
     return 0;
 }
 
-static int add_coded_side_data(AVStream *st, AVCodecContext *avctx)
-{
-    for (int i = 0; i < avctx->nb_coded_side_data; i++) {
-        const AVPacketSideData *const sd_src = &avctx->coded_side_data[i];
-        uint8_t *dst_data;
-        dst_data = av_stream_new_side_data(st, sd_src->type, sd_src->size);
-        if (!dst_data)
-            return AVERROR(ENOMEM);
-        memcpy(dst_data, sd_src->data, sd_src->size);
-    }
-    return 0;
-}
-
 int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
 {
     FFFormatContext *const si = ffformatcontext(ic);
@@ -2969,9 +2958,6 @@  int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
 
         if (sti->avctx_inited) {
             ret = avcodec_parameters_from_context(st->codecpar, sti->avctx);
-            if (ret < 0)
-                goto find_stream_info_err;
-            ret = add_coded_side_data(st, sti->avctx);
             if (ret < 0)
                 goto find_stream_info_err;
 
@@ -2986,14 +2972,42 @@  int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
                         props->min_bitrate = sti->avctx->rc_min_rate;
                     if (sti->avctx->rc_max_rate > 0)
                         props->max_bitrate = sti->avctx->rc_max_rate;
-                    if (av_stream_add_side_data(st, AV_PKT_DATA_CPB_PROPERTIES,
-                                                (uint8_t *)props, cpb_size))
+                    if (!av_packet_side_data_set_add(&st->codecpar->side_data,
+                                                     AV_PKT_DATA_CPB_PROPERTIES,
+                                                     (uint8_t *)props, cpb_size, 0))
                         av_free(props);
                 }
             }
         }
 
         sti->avctx_inited = 0;
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
+        if (st->codecpar->side_data.nb_sd > 0) {
+            const AVPacketSideDataSet *set = &st->codecpar->side_data;
+
+            av_assert0(!st->side_data && !st->nb_side_data);
+            st->side_data = av_calloc(set->nb_sd, sizeof(*st->side_data));
+            if (!st->side_data) {
+                ret = AVERROR(ENOMEM);
+                goto find_stream_info_err;
+            }
+
+            for (int j = 0; j < set->nb_sd; j++) {
+                uint8_t *data = av_memdup(set->sd[j]->data,
+                                          set->sd[j]->size);
+                if (!data) {
+                    ret = AVERROR(ENOMEM);
+                    goto find_stream_info_err;
+                }
+                st->side_data[j].type = set->sd[j]->type;
+                st->side_data[j].size = set->sd[j]->size;
+                st->side_data[j].data = data;
+                st->nb_side_data++;
+            }
+        }
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
     }
 
 find_stream_info_err:
diff --git a/libavformat/demux_utils.c b/libavformat/demux_utils.c
index 56cc6e15d8..2946e82295 100644
--- a/libavformat/demux_utils.c
+++ b/libavformat/demux_utils.c
@@ -80,6 +80,8 @@  AVChapter *avpriv_new_chapter(AVFormatContext *s, int64_t id, AVRational time_ba
     return chapter;
 }
 
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
 void av_format_inject_global_side_data(AVFormatContext *s)
 {
     FFFormatContext *const si = ffformatcontext(s);
@@ -89,6 +91,8 @@  void av_format_inject_global_side_data(AVFormatContext *s)
         ffstream(st)->inject_global_side_data = 1;
     }
 }
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
 
 int avformat_queue_attached_pictures(AVFormatContext *s)
 {
diff --git a/libavformat/dovi_isom.c b/libavformat/dovi_isom.c
index c8fdf566e4..8d03d0e632 100644
--- a/libavformat/dovi_isom.c
+++ b/libavformat/dovi_isom.c
@@ -34,7 +34,6 @@  int ff_isom_parse_dvcc_dvvc(void *logctx, AVStream *st,
     uint32_t buf;
     AVDOVIDecoderConfigurationRecord *dovi;
     size_t dovi_size;
-    int ret;
 
     if (size > (1 << 30) || size < 4)
         return AVERROR_INVALIDDATA;
@@ -64,11 +63,10 @@  int ff_isom_parse_dvcc_dvvc(void *logctx, AVStream *st,
         dovi->dv_bl_signal_compatibility_id = 0;
     }
 
-    ret = av_stream_add_side_data(st, AV_PKT_DATA_DOVI_CONF,
-                                  (uint8_t *)dovi, dovi_size);
-    if (ret < 0) {
+    if (!av_packet_side_data_set_add(&st->codecpar->side_data, AV_PKT_DATA_DOVI_CONF,
+                                     (uint8_t *)dovi, dovi_size, 0)) {
         av_free(dovi);
-        return ret;
+        return AVERROR(ENOMEM);
     }
 
     av_log(logctx, AV_LOG_TRACE, "DOVI in dvcC/dvvC/dvwC box, version: %d.%d, profile: %d, level: %d, "
diff --git a/libavformat/dump.c b/libavformat/dump.c
index d31e4c2ec6..d6f32f0c68 100644
--- a/libavformat/dump.c
+++ b/libavformat/dump.c
@@ -431,11 +431,11 @@  static void dump_sidedata(void *ctx, const AVStream *st, const char *indent)
 {
     int i;
 
-    if (st->nb_side_data)
+    if (st->codecpar->side_data.nb_sd)
         av_log(ctx, AV_LOG_INFO, "%sSide data:\n", indent);
 
-    for (i = 0; i < st->nb_side_data; i++) {
-        const AVPacketSideData *sd = &st->side_data[i];
+    for (i = 0; i < st->codecpar->side_data.nb_sd; i++) {
+        const AVPacketSideData *sd = st->codecpar->side_data.sd[i];
         av_log(ctx, AV_LOG_INFO, "%s  ", indent);
 
         switch (sd->type) {
diff --git a/libavformat/hls.c b/libavformat/hls.c
index c625e30291..8f80cf64f4 100644
--- a/libavformat/hls.c
+++ b/libavformat/hls.c
@@ -1851,17 +1851,6 @@  static int set_stream_info_from_input_stream(AVStream *st, struct playlist *pls,
 
     av_dict_copy(&st->metadata, ist->metadata, 0);
 
-    // copy side data
-    for (int i = 0; i < ist->nb_side_data; i++) {
-        const AVPacketSideData *sd_src = &ist->side_data[i];
-        uint8_t *dst_data;
-
-        dst_data = av_stream_new_side_data(st, sd_src->type, sd_src->size);
-        if (!dst_data)
-            return AVERROR(ENOMEM);
-        memcpy(dst_data, sd_src->data, sd_src->size);
-    }
-
     ffstream(st)->need_context_update = 1;
 
     return 0;
diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c
index 27d97f5f72..fcc875e236 100644
--- a/libavformat/hlsenc.c
+++ b/libavformat/hlsenc.c
@@ -1351,16 +1351,17 @@  static const char* get_relative_url(const char *master_url, const char *media_ur
 
 static int64_t get_stream_bit_rate(AVStream *stream)
 {
-    AVCPBProperties *props = (AVCPBProperties*)av_stream_get_side_data(
-        stream,
-        AV_PKT_DATA_CPB_PROPERTIES,
-        NULL
+    AVPacketSideData *sd = av_packet_side_data_set_get(
+        &stream->codecpar->side_data,
+        AV_PKT_DATA_CPB_PROPERTIES
     );
 
     if (stream->codecpar->bit_rate)
         return stream->codecpar->bit_rate;
-    else if (props)
+    else if (sd) {
+        AVCPBProperties *props = (AVCPBProperties*)sd->data;
         return props->max_bitrate;
+    }
 
     return 0;
 }
diff --git a/libavformat/internal.h b/libavformat/internal.h
index 53e70ccb53..399ec16ded 100644
--- a/libavformat/internal.h
+++ b/libavformat/internal.h
@@ -147,7 +147,9 @@  typedef struct FFFormatContext {
     int missing_ts_warning;
 #endif
 
+#if FF_API_AVSTREAM_SIDE_DATA
     int inject_global_side_data;
+#endif
 
     int avoid_negative_ts_use_pts;
 
@@ -354,10 +356,12 @@  typedef struct FFStream {
     uint8_t dts_ordered;
     uint8_t dts_misordered;
 
+#if FF_API_AVSTREAM_SIDE_DATA
     /**
      * Internal data to inject global side data
      */
     int inject_global_side_data;
+#endif
 
     /**
      * display aspect ratio (0 if unknown)
diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c
index fda77b0b89..20eb75bb6d 100644
--- a/libavformat/matroskadec.c
+++ b/libavformat/matroskadec.c
@@ -2175,7 +2175,6 @@  static int mkv_stereo3d_conv(AVStream *st, MatroskaVideoStereoModeType stereo_mo
         STEREOMODE_STEREO3D_MAPPING(STEREO_MODE_CONV, NOTHING)
     };
     AVStereo3D *stereo;
-    int ret;
 
     stereo = av_stereo3d_alloc();
     if (!stereo)
@@ -2184,11 +2183,10 @@  static int mkv_stereo3d_conv(AVStream *st, MatroskaVideoStereoModeType stereo_mo
     stereo->type  = stereo_mode_conv[stereo_mode].type;
     stereo->flags = stereo_mode_conv[stereo_mode].flags;
 
-    ret = av_stream_add_side_data(st, AV_PKT_DATA_STEREO3D, (uint8_t *)stereo,
-                                  sizeof(*stereo));
-    if (ret < 0) {
+    if (!av_packet_side_data_set_add(&st->codecpar->side_data, AV_PKT_DATA_STEREO3D,
+                                     (uint8_t *)stereo, sizeof(*stereo), 0)) {
         av_freep(&stereo);
-        return ret;
+        return AVERROR(ENOMEM);
     }
 
     return 0;
@@ -2235,28 +2233,26 @@  static int mkv_parse_video_color(AVStream *st, const MatroskaTrack *track) {
     }
     if (color->max_cll && color->max_fall) {
         size_t size = 0;
-        int ret;
         AVContentLightMetadata *metadata = av_content_light_metadata_alloc(&size);
         if (!metadata)
             return AVERROR(ENOMEM);
-        ret = av_stream_add_side_data(st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
-                                      (uint8_t *)metadata, size);
-        if (ret < 0) {
+        if (!av_packet_side_data_set_add(&st->codecpar->side_data, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
+                                         (uint8_t *)metadata, size, 0)) {
             av_freep(&metadata);
-            return ret;
+            return AVERROR(ENOMEM);
         }
         metadata->MaxCLL  = color->max_cll;
         metadata->MaxFALL = color->max_fall;
     }
 
     if (has_mastering_primaries || has_mastering_luminance) {
-        AVMasteringDisplayMetadata *metadata =
-            (AVMasteringDisplayMetadata*) av_stream_new_side_data(
-                st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
-                sizeof(AVMasteringDisplayMetadata));
-        if (!metadata) {
+        AVMasteringDisplayMetadata *metadata;
+        AVPacketSideData *sd = av_packet_side_data_set_new(&st->codecpar->side_data,
+                                                           AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
+                                                           sizeof(AVMasteringDisplayMetadata), 0);
+        if (!sd)
             return AVERROR(ENOMEM);
-        }
+        metadata = (AVMasteringDisplayMetadata*)sd->data;
         memset(metadata, 0, sizeof(AVMasteringDisplayMetadata));
         if (has_mastering_primaries) {
             metadata->display_primaries[0][0] = av_d2q(mastering_meta->r_x, INT_MAX);
@@ -2282,6 +2278,7 @@  static int mkv_create_display_matrix(AVStream *st,
                                      const MatroskaTrackVideoProjection *proj,
                                      void *logctx)
 {
+    AVPacketSideData *sd;
     double pitch = proj->pitch, yaw = proj->yaw, roll = proj->roll;
     int32_t *matrix;
     int hflip;
@@ -2298,10 +2295,12 @@  static int mkv_create_display_matrix(AVStream *st,
                st->index, yaw, pitch, roll);
         return 0;
     }
-    matrix = (int32_t*)av_stream_new_side_data(st, AV_PKT_DATA_DISPLAYMATRIX,
-                                               9 * sizeof(*matrix));
-    if (!matrix)
+    sd = av_packet_side_data_set_new(&st->codecpar->side_data,
+                                     AV_PKT_DATA_DISPLAYMATRIX,
+                                     9 * sizeof(*matrix), 0);
+    if (!sd)
         return AVERROR(ENOMEM);
+    matrix = (int32_t*)sd->data;
 
     hflip = yaw != 0.0;
     /* ProjectionPoseRoll is in the counter-clockwise direction
@@ -2326,7 +2325,6 @@  static int mkv_parse_video_projection(AVStream *st, const MatroskaTrack *track,
     size_t spherical_size;
     uint32_t l = 0, t = 0, r = 0, b = 0;
     uint32_t padding = 0;
-    int ret;
 
     if (mkv_projection->private.size && priv_data[0] != 0) {
         av_log(logctx, AV_LOG_WARNING, "Unknown spherical metadata\n");
@@ -2402,11 +2400,10 @@  static int mkv_parse_video_projection(AVStream *st, const MatroskaTrack *track,
     spherical->bound_right  = r;
     spherical->bound_bottom = b;
 
-    ret = av_stream_add_side_data(st, AV_PKT_DATA_SPHERICAL, (uint8_t *)spherical,
-                                  spherical_size);
-    if (ret < 0) {
+    if (!av_packet_side_data_set_add(&st->codecpar->side_data, AV_PKT_DATA_SPHERICAL,
+                                     (uint8_t *)spherical, spherical_size, 0)) {
         av_freep(&spherical);
-        return ret;
+        return AVERROR(ENOMEM);
     }
 
     return 0;
diff --git a/libavformat/matroskaenc.c b/libavformat/matroskaenc.c
index bf2ca7106b..b188b74e82 100644
--- a/libavformat/matroskaenc.c
+++ b/libavformat/matroskaenc.c
@@ -1327,7 +1327,7 @@  fail:
 static void mkv_write_video_color(EbmlWriter *writer, const AVStream *st,
                                   const AVCodecParameters *par)
 {
-    const void *side_data;
+    const AVPacketSideData *side_data;
 
     ebml_writer_open_master(writer, MATROSKA_ID_VIDEOCOLOR);
 
@@ -1361,20 +1361,18 @@  static void mkv_write_video_color(EbmlWriter *writer, const AVStream *st,
                              (ypos >> 7) + 1);
     }
 
-    side_data = av_stream_get_side_data(st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
-                                        NULL);
+    side_data = av_packet_side_data_set_get(&st->codecpar->side_data, AV_PKT_DATA_CONTENT_LIGHT_LEVEL);
     if (side_data) {
-        const AVContentLightMetadata *metadata = side_data;
+        const AVContentLightMetadata *metadata = (AVContentLightMetadata *)side_data->data;
         ebml_writer_add_uint(writer, MATROSKA_ID_VIDEOCOLORMAXCLL,
                              metadata->MaxCLL);
         ebml_writer_add_uint(writer, MATROSKA_ID_VIDEOCOLORMAXFALL,
                              metadata->MaxFALL);
     }
 
-    side_data = av_stream_get_side_data(st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
-                                        NULL);
+    side_data = av_packet_side_data_set_get(&st->codecpar->side_data, AV_PKT_DATA_MASTERING_DISPLAY_METADATA);
     if (side_data) {
-        const AVMasteringDisplayMetadata *metadata = side_data;
+        const AVMasteringDisplayMetadata *metadata = (AVMasteringDisplayMetadata *)side_data->data;
         ebml_writer_open_master(writer, MATROSKA_ID_VIDEOCOLORMASTERINGMETA);
         if (metadata->has_primaries) {
             ebml_writer_add_float(writer, MATROSKA_ID_VIDEOCOLOR_RX,
@@ -1410,12 +1408,15 @@  static void mkv_write_video_color(EbmlWriter *writer, const AVStream *st,
 static void mkv_handle_rotation(void *logctx, const AVStream *st,
                                 double *yaw, double *roll)
 {
-    const int32_t *matrix =
-        (const int32_t*)av_stream_get_side_data(st, AV_PKT_DATA_DISPLAYMATRIX, NULL);
+    const int32_t *matrix;
+    const AVPacketSideData *side_data =
+        av_packet_side_data_set_get(&st->codecpar->side_data, AV_PKT_DATA_DISPLAYMATRIX);
 
-    if (!matrix)
+    if (!side_data)
         return;
 
+    matrix = (int32_t *)side_data->data;
+
     /* Check whether this is an affine transformation */
     if (matrix[2] || matrix[5])
         goto ignore;
@@ -1462,13 +1463,13 @@  static int mkv_handle_spherical(void *logctx, EbmlWriter *writer,
                                 const AVStream *st, uint8_t private[],
                                 double *yaw, double *pitch, double *roll)
 {
-    const AVSphericalMapping *spherical =
-        (const AVSphericalMapping *)av_stream_get_side_data(st, AV_PKT_DATA_SPHERICAL,
-                                                            NULL);
+    AVPacketSideData *sd = av_packet_side_data_set_get(&st->codecpar->side_data, AV_PKT_DATA_SPHERICAL);
+    const AVSphericalMapping *spherical;
 
-    if (!spherical)
+    if (!sd)
         return 0;
 
+    spherical = (const AVSphericalMapping *)sd->data;
     if (spherical->projection != AV_SPHERICAL_EQUIRECTANGULAR      &&
         spherical->projection != AV_SPHERICAL_EQUIRECTANGULAR_TILE &&
         spherical->projection != AV_SPHERICAL_CUBEMAP) {
@@ -1628,6 +1629,7 @@  static int mkv_write_stereo_mode(AVFormatContext *s, EbmlWriter *writer,
             format = stereo_mode;
         }
     } else {
+        AVPacketSideData *sd;
         const AVStereo3D *stereo;
         /* The following macro presumes all MATROSKA_VIDEO_STEREOMODE_TYPE_*
          * values to be in the range 0..254. */
@@ -1639,11 +1641,12 @@  static int mkv_write_stereo_mode(AVFormatContext *s, EbmlWriter *writer,
         };
         int fmt;
 
-        stereo = (const AVStereo3D*)av_stream_get_side_data(st, AV_PKT_DATA_STEREO3D,
-                                                            NULL);
-        if (!stereo)
+        sd = av_packet_side_data_set_get(&st->codecpar->side_data, AV_PKT_DATA_STEREO3D);
+        if (!sd)
             return 0;
 
+        stereo = (const AVStereo3D*)sd->data;
+
         /* A garbage AVStereo3D or something with no Matroska analogon. */
         if ((unsigned)stereo->type >= FF_ARRAY_ELEMS(conversion_table))
             return 0;
@@ -1681,6 +1684,7 @@  static void mkv_write_blockadditionmapping(AVFormatContext *s, const MatroskaMux
 {
 #if CONFIG_MATROSKA_MUXER
     const AVDOVIDecoderConfigurationRecord *dovi;
+    const AVPacketSideData *sd;
 
     if (IS_SEEKABLE(s->pb, mkv)) {
         track->blockadditionmapping_offset = avio_tell(pb);
@@ -1697,9 +1701,13 @@  static void mkv_write_blockadditionmapping(AVFormatContext *s, const MatroskaMux
         }
     }
 
-    dovi = (const AVDOVIDecoderConfigurationRecord *)
-           av_stream_get_side_data(st, AV_PKT_DATA_DOVI_CONF, NULL);
-    if (dovi && dovi->dv_profile <= 10) {
+    sd = av_packet_side_data_set_get(&st->codecpar->side_data, AV_PKT_DATA_DOVI_CONF);
+
+    if (!sd)
+        return;
+
+    dovi = (const AVDOVIDecoderConfigurationRecord *)sd->data;
+    if (dovi->dv_profile <= 10) {
         ebml_master mapping;
         uint8_t buf[ISOM_DVCC_DVVC_SIZE];
         uint32_t type;
diff --git a/libavformat/mov.c b/libavformat/mov.c
index be9975f297..a08b6c77ee 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -799,6 +799,7 @@  static int mov_read_esds(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 static int mov_read_dac3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 {
     AVStream *st;
+    AVPacketSideData *sd;
     enum AVAudioServiceType *ast;
     int ac3info, acmod, lfeon, bsmod;
     uint64_t mask;
@@ -807,11 +808,13 @@  static int mov_read_dac3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
         return 0;
     st = c->fc->streams[c->fc->nb_streams-1];
 
-    ast = (enum AVAudioServiceType*)av_stream_new_side_data(st, AV_PKT_DATA_AUDIO_SERVICE_TYPE,
-                                                            sizeof(*ast));
-    if (!ast)
+    sd = av_packet_side_data_set_new(&st->codecpar->side_data,
+                                     AV_PKT_DATA_AUDIO_SERVICE_TYPE,
+                                     sizeof(*ast), 0);
+    if (!sd)
         return AVERROR(ENOMEM);
 
+    ast = (enum AVAudioServiceType*)sd->data;
     ac3info = avio_rb24(pb);
     bsmod = (ac3info >> 14) & 0x7;
     acmod = (ac3info >> 11) & 0x7;
@@ -833,6 +836,7 @@  static int mov_read_dac3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 static int mov_read_dec3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 {
     AVStream *st;
+    AVPacketSideData *sd;
     enum AVAudioServiceType *ast;
     int eac3info, acmod, lfeon, bsmod;
     uint64_t mask;
@@ -841,11 +845,14 @@  static int mov_read_dec3(MOVContext *c, AVIOContext *pb, MOVAtom atom)
         return 0;
     st = c->fc->streams[c->fc->nb_streams-1];
 
-    ast = (enum AVAudioServiceType*)av_stream_new_side_data(st, AV_PKT_DATA_AUDIO_SERVICE_TYPE,
-                                                            sizeof(*ast));
-    if (!ast)
+    sd = av_packet_side_data_set_new(&st->codecpar->side_data,
+                                     AV_PKT_DATA_AUDIO_SERVICE_TYPE,
+                                     sizeof(*ast), 0);
+    if (!sd)
         return AVERROR(ENOMEM);
 
+    ast = (enum AVAudioServiceType*)sd->data;
+
     /* No need to parse fields for additional independent substreams and its
      * associated dependent substreams since libavcodec's E-AC-3 decoder
      * does not support them yet. */
@@ -1747,7 +1754,6 @@  static int mov_read_pcmc(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 static int mov_read_colr(MOVContext *c, AVIOContext *pb, MOVAtom atom)
 {
     AVStream *st;
-    uint8_t *icc_profile;
     char color_parameter_type[5] = { 0 };
     uint16_t color_primaries, color_trc, color_matrix;
     int ret;
@@ -1768,10 +1774,12 @@  static int mov_read_colr(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     }
 
     if (!strncmp(color_parameter_type, "prof", 4)) {
-        icc_profile = av_stream_new_side_data(st, AV_PKT_DATA_ICC_PROFILE, atom.size - 4);
-        if (!icc_profile)
+        AVPacketSideData *sd = av_packet_side_data_set_new(&st->codecpar->side_data,
+                                                           AV_PKT_DATA_ICC_PROFILE,
+                                                           atom.size - 4, 0);
+        if (!sd)
             return AVERROR(ENOMEM);
-        ret = ffio_read_size(pb, icc_profile, atom.size - 4);
+        ret = ffio_read_size(pb, sd->data, atom.size - 4);
         if (ret < 0)
             return ret;
     } else {
@@ -6838,8 +6846,9 @@  static int mov_read_pssh(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     AVEncryptionInitInfo *info, *old_init_info;
     uint8_t **key_ids;
     AVStream *st;
-    uint8_t *side_data, *extra_data, *old_side_data;
-    size_t side_data_size, old_side_data_size;
+    AVPacketSideData *old_side_data;
+    uint8_t *side_data, *extra_data;
+    size_t side_data_size;
     int ret = 0;
     unsigned int version, kid_count, extra_data_size, alloc_size = 0;
 
@@ -6917,9 +6926,9 @@  static int mov_read_pssh(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     info->data_size = extra_data_size;
 
     // If there is existing initialization data, append to the list.
-    old_side_data = av_stream_get_side_data(st, AV_PKT_DATA_ENCRYPTION_INIT_INFO, &old_side_data_size);
+    old_side_data = av_packet_side_data_set_get(&st->codecpar->side_data, AV_PKT_DATA_ENCRYPTION_INIT_INFO);
     if (old_side_data) {
-        old_init_info = av_encryption_init_info_get_side_data(old_side_data, old_side_data_size);
+        old_init_info = av_encryption_init_info_get_side_data(old_side_data->data, old_side_data->size);
         if (old_init_info) {
             // Append to the end of the list.
             for (AVEncryptionInitInfo *cur = old_init_info;; cur = cur->next) {
@@ -6941,9 +6950,8 @@  static int mov_read_pssh(MOVContext *c, AVIOContext *pb, MOVAtom atom)
         ret = AVERROR(ENOMEM);
         goto finish;
     }
-    ret = av_stream_add_side_data(st, AV_PKT_DATA_ENCRYPTION_INIT_INFO,
-                                  side_data, side_data_size);
-    if (ret < 0)
+    if (!av_packet_side_data_set_add(&st->codecpar->side_data, AV_PKT_DATA_ENCRYPTION_INIT_INFO,
+                                     side_data, side_data_size, 0))
         av_free(side_data);
 
 finish:
@@ -8720,46 +8728,37 @@  static int mov_read_header(AVFormatContext *s)
             break;
         case AVMEDIA_TYPE_VIDEO:
             if (sc->display_matrix) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_DISPLAYMATRIX, (uint8_t*)sc->display_matrix,
-                                              sizeof(int32_t) * 9);
-                if (err < 0)
-                    return err;
+                if (!av_packet_side_data_set_add(&st->codecpar->side_data, AV_PKT_DATA_DISPLAYMATRIX,
+                                                 (uint8_t*)sc->display_matrix, sizeof(int32_t) * 9, 0))
+                    return AVERROR(ENOMEM);
 
                 sc->display_matrix = NULL;
             }
             if (sc->stereo3d) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_STEREO3D,
-                                              (uint8_t *)sc->stereo3d,
-                                              sizeof(*sc->stereo3d));
-                if (err < 0)
-                    return err;
+                if (!av_packet_side_data_set_add(&st->codecpar->side_data, AV_PKT_DATA_STEREO3D,
+                                                 (uint8_t *)sc->stereo3d, sizeof(*sc->stereo3d), 0))
+                    return AVERROR(ENOMEM);
 
                 sc->stereo3d = NULL;
             }
             if (sc->spherical) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_SPHERICAL,
-                                              (uint8_t *)sc->spherical,
-                                              sc->spherical_size);
-                if (err < 0)
-                    return err;
+                if (!av_packet_side_data_set_add(&st->codecpar->side_data, AV_PKT_DATA_SPHERICAL,
+                                                 (uint8_t *)sc->spherical, sc->spherical_size, 0))
+                    return AVERROR(ENOMEM);
 
                 sc->spherical = NULL;
             }
             if (sc->mastering) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
-                                              (uint8_t *)sc->mastering,
-                                              sizeof(*sc->mastering));
-                if (err < 0)
-                    return err;
+                if (!av_packet_side_data_set_add(&st->codecpar->side_data, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
+                                                 (uint8_t *)sc->mastering, sizeof(*sc->mastering), 0))
+                    return AVERROR(ENOMEM);
 
                 sc->mastering = NULL;
             }
             if (sc->coll) {
-                err = av_stream_add_side_data(st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
-                                              (uint8_t *)sc->coll,
-                                              sc->coll_size);
-                if (err < 0)
-                    return err;
+                if (!av_packet_side_data_set_add(&st->codecpar->side_data, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
+                                                 (uint8_t *)sc->coll, sc->coll_size, 0))
+                    return AVERROR(ENOMEM);
 
                 sc->coll = NULL;
             }
diff --git a/libavformat/movenc.c b/libavformat/movenc.c
index 696ae5a6c9..66efed0363 100644
--- a/libavformat/movenc.c
+++ b/libavformat/movenc.c
@@ -672,11 +672,11 @@  struct mpeg4_bit_rate_values {
 
 static struct mpeg4_bit_rate_values calculate_mpeg4_bit_rates(MOVTrack *track)
 {
-    AVCPBProperties *props = track->st ?
-        (AVCPBProperties*)av_stream_get_side_data(track->st,
-                                                  AV_PKT_DATA_CPB_PROPERTIES,
-                                                  NULL) :
+    AVPacketSideData *sd = track->st ?
+        av_packet_side_data_set_get(&track->st->codecpar->side_data,
+                                         AV_PKT_DATA_CPB_PROPERTIES) :
         NULL;
+    AVCPBProperties *props = sd ? (AVCPBProperties *)sd->data : NULL;
     struct mpeg4_bit_rate_values bit_rates = { 0 };
 
     bit_rates.avg_bit_rate = compute_avg_bitrate(track);
@@ -2129,18 +2129,16 @@  static int mov_write_colr_tag(AVIOContext *pb, MOVTrack *track, int prefer_icc)
     // Ref (MOV): https://developer.apple.com/library/mac/technotes/tn2162/_index.html#//apple_ref/doc/uid/DTS40013070-CH1-TNTAG9
     // Ref (MP4): ISO/IEC 14496-12:2012
 
-    const uint8_t *icc_profile;
-    size_t icc_profile_size;
-
     if (prefer_icc) {
-        icc_profile = av_stream_get_side_data(track->st, AV_PKT_DATA_ICC_PROFILE, &icc_profile_size);
+        AVPacketSideData *sd = av_packet_side_data_set_get(&track->st->codecpar->side_data,
+                                                                AV_PKT_DATA_ICC_PROFILE);
 
-        if (icc_profile) {
-            avio_wb32(pb, 12 + icc_profile_size);
+        if (sd) {
+            avio_wb32(pb, 12 + sd->size);
             ffio_wfourcc(pb, "colr");
             ffio_wfourcc(pb, "prof");
-            avio_write(pb, icc_profile, icc_profile_size);
-            return 12 + icc_profile_size;
+            avio_write(pb, sd->data, sd->size);
+            return 12 + sd->size;
         }
         else {
             av_log(NULL, AV_LOG_INFO, "no ICC profile found, will write nclx/nclc colour info instead\n");
@@ -2173,14 +2171,14 @@  static int mov_write_colr_tag(AVIOContext *pb, MOVTrack *track, int prefer_icc)
 
 static int mov_write_clli_tag(AVIOContext *pb, MOVTrack *track)
 {
-    const uint8_t *side_data;
+    const AVPacketSideData *side_data;
     const AVContentLightMetadata *content_light_metadata;
 
-    side_data = av_stream_get_side_data(track->st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL, NULL);
+    side_data = av_packet_side_data_set_get(&track->st->codecpar->side_data, AV_PKT_DATA_CONTENT_LIGHT_LEVEL);
     if (!side_data) {
         return 0;
     }
-    content_light_metadata = (const AVContentLightMetadata*)side_data;
+    content_light_metadata = (const AVContentLightMetadata*)side_data->data;
 
     avio_wb32(pb, 12); // size
     ffio_wfourcc(pb, "clli");
@@ -2198,11 +2196,12 @@  static int mov_write_mdcv_tag(AVIOContext *pb, MOVTrack *track)
 {
     const int chroma_den = 50000;
     const int luma_den = 10000;
-    const uint8_t *side_data;
-    const AVMasteringDisplayMetadata *metadata;
+    const AVPacketSideData *side_data;
+    const AVMasteringDisplayMetadata *metadata = NULL;
 
-    side_data = av_stream_get_side_data(track->st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA, NULL);
-    metadata = (const AVMasteringDisplayMetadata*)side_data;
+    side_data = av_packet_side_data_set_get(&track->st->codecpar->side_data, AV_PKT_DATA_MASTERING_DISPLAY_METADATA);
+    if (side_data)
+        metadata = (const AVMasteringDisplayMetadata*)side_data->data;
     if (!metadata || !metadata->has_primaries || !metadata->has_luminance) {
         return 0;
     }
@@ -2421,7 +2420,7 @@  static int mov_write_video_tag(AVFormatContext *s, AVIOContext *pb, MOVMuxContex
                              track->par->color_trc != AVCOL_TRC_UNSPECIFIED &&
                              track->par->color_space != AVCOL_SPC_UNSPECIFIED;
         if (has_color_info || mov->flags & FF_MOV_FLAG_WRITE_COLR ||
-            av_stream_get_side_data(track->st, AV_PKT_DATA_ICC_PROFILE, NULL)) {
+            av_packet_side_data_set_get(&track->st->codecpar->side_data, AV_PKT_DATA_ICC_PROFILE)) {
             int prefer_icc = mov->flags & FF_MOV_FLAG_PREFER_ICC || !has_color_info;
             mov_write_colr_tag(pb, track, prefer_icc);
         }
@@ -2435,17 +2434,16 @@  static int mov_write_video_tag(AVFormatContext *s, AVIOContext *pb, MOVMuxContex
     }
 
     if (track->mode == MODE_MP4 && mov->fc->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL) {
-        AVStereo3D* stereo_3d = (AVStereo3D*) av_stream_get_side_data(track->st, AV_PKT_DATA_STEREO3D, NULL);
-        AVSphericalMapping* spherical_mapping = (AVSphericalMapping*)av_stream_get_side_data(track->st, AV_PKT_DATA_SPHERICAL, NULL);
-        AVDOVIDecoderConfigurationRecord *dovi = (AVDOVIDecoderConfigurationRecord *)
-                                                 av_stream_get_side_data(track->st, AV_PKT_DATA_DOVI_CONF, NULL);
+        AVPacketSideData *stereo_3d = av_packet_side_data_set_get(&track->st->codecpar->side_data, AV_PKT_DATA_STEREO3D);
+        AVPacketSideData *spherical_mapping = av_packet_side_data_set_get(&track->st->codecpar->side_data, AV_PKT_DATA_SPHERICAL);
+        AVPacketSideData *dovi = av_packet_side_data_set_get(&track->st->codecpar->side_data, AV_PKT_DATA_DOVI_CONF);
 
         if (stereo_3d)
-            mov_write_st3d_tag(s, pb, stereo_3d);
+            mov_write_st3d_tag(s, pb, (AVStereo3D*)stereo_3d->data);
         if (spherical_mapping)
-            mov_write_sv3d_tag(mov->fc, pb, spherical_mapping);
+            mov_write_sv3d_tag(mov->fc, pb, (AVSphericalMapping*)spherical_mapping->data);
         if (dovi)
-            mov_write_dvcc_dvvc_tag(s, pb, dovi);
+            mov_write_dvcc_dvvc_tag(s, pb, (AVDOVIDecoderConfigurationRecord *)dovi->data);
     }
 
     if (track->par->sample_aspect_ratio.den && track->par->sample_aspect_ratio.num) {
@@ -3392,7 +3390,6 @@  static int mov_write_tkhd_tag(AVIOContext *pb, MOVMuxContext *mov,
     int group   = 0;
 
     uint32_t *display_matrix = NULL;
-    size_t display_matrix_size;
     int       i;
 
     if (mov->mode == MODE_AVIF)
@@ -3402,15 +3399,15 @@  static int mov_write_tkhd_tag(AVIOContext *pb, MOVMuxContext *mov,
             duration *= mov->avif_loop_count;
 
     if (st) {
+        AVPacketSideData *sd;
         if (mov->per_stream_grouping)
             group = st->index;
         else
             group = st->codecpar->codec_type;
 
-        display_matrix = (uint32_t*)av_stream_get_side_data(st, AV_PKT_DATA_DISPLAYMATRIX,
-                                                            &display_matrix_size);
-        if (display_matrix && display_matrix_size < 9 * sizeof(*display_matrix))
-            display_matrix = NULL;
+        sd = av_packet_side_data_set_get(&st->codecpar->side_data, AV_PKT_DATA_DISPLAYMATRIX);
+        if (sd && sd->size == 9 * sizeof(*display_matrix))
+            display_matrix = (uint32_t *)sd->data;
     }
 
     if (track->flags & MOV_TRACK_ENABLED)
@@ -4608,12 +4605,10 @@  static int mov_write_moov_tag(AVIOContext *pb, MOVMuxContext *mov,
             track->tref_tag = MKTAG('h','i','n','t');
             track->tref_id = mov->tracks[track->src_track].track_id;
         } else if (track->par->codec_type == AVMEDIA_TYPE_AUDIO) {
-            size_t size;
-            int *fallback;
-            fallback = (int*)av_stream_get_side_data(track->st,
-                                                     AV_PKT_DATA_FALLBACK_TRACK,
-                                                     &size);
-            if (fallback != NULL && size == sizeof(int)) {
+            AVPacketSideData *sd = av_packet_side_data_set_get(&track->st->codecpar->side_data,
+                                                                    AV_PKT_DATA_FALLBACK_TRACK );
+            if (sd && sd->size == sizeof(int)) {
+                int *fallback = (int *)sd->data;
                 if (*fallback >= 0 && *fallback < mov->nb_streams) {
                     track->tref_tag = MKTAG('f','a','l','l');
                     track->tref_id = mov->tracks[*fallback].track_id;
@@ -5446,7 +5441,7 @@  static int mov_write_ftyp_tag(AVIOContext *pb, AVFormatContext *s)
         if (st->codecpar->codec_id == AV_CODEC_ID_AC3 ||
             st->codecpar->codec_id == AV_CODEC_ID_EAC3 ||
             st->codecpar->codec_id == AV_CODEC_ID_TRUEHD ||
-            av_stream_get_side_data(st, AV_PKT_DATA_DOVI_CONF, NULL))
+            av_packet_side_data_set_get(&st->codecpar->side_data, AV_PKT_DATA_DOVI_CONF))
             has_dolby = 1;
     }
 
diff --git a/libavformat/mp3enc.c b/libavformat/mp3enc.c
index 5e81f72a59..9fc1c4fea4 100644
--- a/libavformat/mp3enc.c
+++ b/libavformat/mp3enc.c
@@ -400,10 +400,10 @@  static int mp3_queue_flush(AVFormatContext *s)
 static void mp3_update_xing(AVFormatContext *s)
 {
     MP3Context  *mp3 = s->priv_data;
+    AVPacketSideData *sd;
     AVReplayGain *rg;
     uint16_t tag_crc;
     uint8_t *toc;
-    size_t rg_size;
     int i;
     int64_t old_pos = avio_tell(s->pb);
 
@@ -423,11 +423,11 @@  static void mp3_update_xing(AVFormatContext *s)
     }
 
     /* write replaygain */
-    rg = (AVReplayGain*)av_stream_get_side_data(s->streams[0], AV_PKT_DATA_REPLAYGAIN,
-                                                &rg_size);
-    if (rg && rg_size >= sizeof(*rg)) {
+    sd = av_packet_side_data_set_get(&s->streams[0]->codecpar->side_data, AV_PKT_DATA_REPLAYGAIN);
+    if (sd && sd->size >= sizeof(*rg)) {
         uint16_t val;
 
+        rg = (AVReplayGain *)sd->data;
         AV_WB32(mp3->xing_frame + mp3->xing_offset + 131,
                 av_rescale(rg->track_peak, 1 << 23, 100000));
 
diff --git a/libavformat/mpegenc.c b/libavformat/mpegenc.c
index c06e308296..3f3b94112d 100644
--- a/libavformat/mpegenc.c
+++ b/libavformat/mpegenc.c
@@ -342,8 +342,6 @@  static av_cold int mpeg_mux_init(AVFormatContext *ctx)
     lpcm_id = LPCM_ID;
 
     for (i = 0; i < ctx->nb_streams; i++) {
-        AVCPBProperties *props;
-
         st     = ctx->streams[i];
         stream = av_mallocz(sizeof(StreamInfo));
         if (!stream)
@@ -430,13 +428,17 @@  static av_cold int mpeg_mux_init(AVFormatContext *ctx)
             stream->max_buffer_size = 4 * 1024;
             s->audio_bound++;
             break;
-        case AVMEDIA_TYPE_VIDEO:
+        case AVMEDIA_TYPE_VIDEO: {
+            AVPacketSideData *sd;
+            AVCPBProperties *props = NULL;
             if (st->codecpar->codec_id == AV_CODEC_ID_H264)
                 stream->id = h264_id++;
             else
                 stream->id = mpv_id++;
 
-            props = (AVCPBProperties*)av_stream_get_side_data(st, AV_PKT_DATA_CPB_PROPERTIES, NULL);
+            sd = av_packet_side_data_set_get(&st->codecpar->side_data, AV_PKT_DATA_CPB_PROPERTIES);
+            if (sd)
+                props = (AVCPBProperties*)sd->data;
             if (props && props->buffer_size)
                 stream->max_buffer_size = 6 * 1024 + props->buffer_size / 8;
             else {
@@ -453,6 +455,7 @@  static av_cold int mpeg_mux_init(AVFormatContext *ctx)
             }
             s->video_bound++;
             break;
+        }
         case AVMEDIA_TYPE_SUBTITLE:
             stream->id              = mps_id++;
             stream->max_buffer_size = 16 * 1024;
@@ -470,12 +473,15 @@  static av_cold int mpeg_mux_init(AVFormatContext *ctx)
     audio_bitrate = 0;
     video_bitrate = 0;
     for (i = 0; i < ctx->nb_streams; i++) {
-        AVCPBProperties *props;
+        AVPacketSideData *sd;
+        AVCPBProperties *props = NULL;
         int codec_rate;
         st     = ctx->streams[i];
         stream = (StreamInfo *)st->priv_data;
 
-        props = (AVCPBProperties*)av_stream_get_side_data(st, AV_PKT_DATA_CPB_PROPERTIES, NULL);
+        sd = av_packet_side_data_set_get(&st->codecpar->side_data, AV_PKT_DATA_CPB_PROPERTIES);
+        if (sd)
+            props = (AVCPBProperties*)sd->data;
         if (props)
             codec_rate = props->max_bitrate;
         else
diff --git a/libavformat/mpegts.c b/libavformat/mpegts.c
index 0b3edda817..3ba723d060 100644
--- a/libavformat/mpegts.c
+++ b/libavformat/mpegts.c
@@ -2190,7 +2190,6 @@  int ff_parse_mpeg2_descriptor(AVFormatContext *fc, AVStream *st, int stream_type
             uint32_t buf;
             AVDOVIDecoderConfigurationRecord *dovi;
             size_t dovi_size;
-            int ret;
             int dependency_pid;
 
             if (desc_end - *pp < 4) // (8 + 8 + 7 + 6 + 1 + 1 + 1) / 8
@@ -2221,11 +2220,10 @@  int ff_parse_mpeg2_descriptor(AVFormatContext *fc, AVStream *st, int stream_type
                 dovi->dv_bl_signal_compatibility_id = 0;
             }
 
-            ret = av_stream_add_side_data(st, AV_PKT_DATA_DOVI_CONF,
-                                          (uint8_t *)dovi, dovi_size);
-            if (ret < 0) {
+            if (!av_packet_side_data_set_add(&st->codecpar->side_data, AV_PKT_DATA_DOVI_CONF,
+                                             (uint8_t *)dovi, dovi_size, 0)) {
                 av_free(dovi);
-                return ret;
+                return AVERROR(ENOMEM);
             }
 
             av_log(fc, AV_LOG_TRACE, "DOVI, version: %d.%d, profile: %d, level: %d, "
diff --git a/libavformat/mux.c b/libavformat/mux.c
index 415bd3948f..ae07c8839e 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -277,6 +277,25 @@  FF_ENABLE_DEPRECATION_WARNINGS
             break;
         }
 
+#if FF_API_AVSTREAM_SIDE_DATA
+FF_DISABLE_DEPRECATION_WARNINGS
+        /* if the caller is using the deprecated AVStream side_data API,
+         * copy its contents to AVStream.codecpar, giving it priority
+           over existing side data in the latter */
+        for (int i = 0; i < st->nb_side_data; i++) {
+            const AVPacketSideData *sd_src = &st->side_data[i];
+            AVPacketSideData *sd_dst;
+
+            sd_dst = av_packet_side_data_set_new(&st->codecpar->side_data, sd_src->type, sd_src->size, 0);
+            if (!sd_dst) {
+                ret = AVERROR(ENOMEM);
+                goto fail;
+            }
+            memcpy(sd_dst->data, sd_src->data, sd_src->size);
+        }
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
+
         desc = avcodec_descriptor_get(par->codec_id);
         if (desc && desc->props & AV_CODEC_PROP_REORDER)
             sti->reorder = 1;
diff --git a/libavformat/mxfdec.c b/libavformat/mxfdec.c
index 34230ece98..2438b7c7bf 100644
--- a/libavformat/mxfdec.c
+++ b/libavformat/mxfdec.c
@@ -2577,10 +2577,12 @@  static int parse_mca_labels(MXFContext *mxf, MXFTrack *source_track, MXFDescript
 
     if (service_type != AV_AUDIO_SERVICE_TYPE_NB && service_type != AV_AUDIO_SERVICE_TYPE_MAIN && !ambigous_service_type) {
         enum AVAudioServiceType *ast;
-        uint8_t* side_data = av_stream_new_side_data(st, AV_PKT_DATA_AUDIO_SERVICE_TYPE, sizeof(*ast));
+        AVPacketSideData *side_data = av_packet_side_data_set_new(&st->codecpar->side_data,
+                                                                  AV_PKT_DATA_AUDIO_SERVICE_TYPE,
+                                                                  sizeof(*ast), 0);
         if (!side_data)
             return AVERROR(ENOMEM);
-        ast = (enum AVAudioServiceType*)side_data;
+        ast = (enum AVAudioServiceType*)side_data->data;
         *ast = service_type;
     }
 
@@ -2980,19 +2982,19 @@  static int mxf_parse_structural_metadata(MXFContext *mxf)
             st->codecpar->color_trc       = mxf_get_codec_ul(ff_mxf_color_trc_uls, &descriptor->color_trc_ul)->id;
             st->codecpar->color_space     = mxf_get_codec_ul(ff_mxf_color_space_uls, &descriptor->color_space_ul)->id;
             if (descriptor->mastering) {
-                ret = av_stream_add_side_data(st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
-                                              (uint8_t *)descriptor->mastering,
-                                              sizeof(*descriptor->mastering));
-                if (ret < 0)
+                if (!av_packet_side_data_set_add(&st->codecpar->side_data, AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
+                                                 (uint8_t *)descriptor->mastering, sizeof(*descriptor->mastering), 0)) {
+                    ret = AVERROR(ENOMEM);
                     goto fail_and_free;
+                }
                 descriptor->mastering = NULL;
             }
             if (descriptor->coll) {
-                ret = av_stream_add_side_data(st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
-                                              (uint8_t *)descriptor->coll,
-                                              descriptor->coll_size);
-                if (ret < 0)
+                if (!av_packet_side_data_set_add(&st->codecpar->side_data, AV_PKT_DATA_CONTENT_LIGHT_LEVEL,
+                                                 (uint8_t *)descriptor->coll, descriptor->coll_size, 0)) {
+                    ret = AVERROR(ENOMEM);
                     goto fail_and_free;
+                }
                 descriptor->coll = NULL;
             }
         } else if (st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
diff --git a/libavformat/mxfenc.c b/libavformat/mxfenc.c
index d8252ed68f..33922400e6 100644
--- a/libavformat/mxfenc.c
+++ b/libavformat/mxfenc.c
@@ -556,7 +556,7 @@  static void mxf_write_primer_pack(AVFormatContext *s)
         if (s->streams[i]->codecpar->codec_id == AV_CODEC_ID_H264 && !sc->avc_intra) {
             will_have_avc_tags = 1;
         }
-        if (av_stream_get_side_data(s->streams[i], AV_PKT_DATA_MASTERING_DISPLAY_METADATA, NULL)) {
+        if (av_packet_side_data_set_get(&s->streams[i]->codecpar->side_data, AV_PKT_DATA_MASTERING_DISPLAY_METADATA)) {
             will_have_mastering_tags = 1;
         }
         if (s->streams[i]->codecpar->codec_id == AV_CODEC_ID_FFV1) {
@@ -1158,7 +1158,7 @@  static int64_t mxf_write_cdci_common(AVFormatContext *s, AVStream *st, const UID
     const MXFCodecUL *color_trc_ul;
     const MXFCodecUL *color_space_ul;
     int64_t pos = mxf_write_generic_desc(s, st, key);
-    uint8_t *side_data;
+    AVPacketSideData *side_data;
 
     color_primaries_ul = mxf_get_codec_ul_by_id(ff_mxf_color_primaries_uls, st->codecpar->color_primaries);
     color_trc_ul       = mxf_get_codec_ul_by_id(ff_mxf_color_trc_uls, st->codecpar->color_trc);
@@ -1344,9 +1344,9 @@  static int64_t mxf_write_cdci_common(AVFormatContext *s, AVStream *st, const UID
     avio_write(pb, *sc->codec_ul, 16);
 
     // Mastering Display metadata
-    side_data = av_stream_get_side_data(st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA, NULL);
+    side_data = av_packet_side_data_set_get(&st->codecpar->side_data, AV_PKT_DATA_MASTERING_DISPLAY_METADATA);
     if (side_data) {
-        const AVMasteringDisplayMetadata *metadata = (const AVMasteringDisplayMetadata*)side_data;
+        const AVMasteringDisplayMetadata *metadata = (const AVMasteringDisplayMetadata*)side_data->data;
         if (metadata->has_primaries) {
             mxf_write_local_tag(s, 12, 0x8301);
             avio_wb16(pb, rescale_mastering_chroma(metadata->display_primaries[0][0]));
diff --git a/libavformat/options.c b/libavformat/options.c
index e4a3aceed0..ef0b593d36 100644
--- a/libavformat/options.c
+++ b/libavformat/options.c
@@ -309,7 +309,9 @@  AVStream *avformat_new_stream(AVFormatContext *s, const AVCodec *c)
 
     st->sample_aspect_ratio = (AVRational) { 0, 1 };
 
+#if FF_API_AVSTREAM_SIDE_DATA
     sti->inject_global_side_data = si->inject_global_side_data;
+#endif
 
     sti->need_context_update = 1;
 
diff --git a/libavformat/replaygain.c b/libavformat/replaygain.c
index 915bcb2382..ab3e39318b 100644
--- a/libavformat/replaygain.c
+++ b/libavformat/replaygain.c
@@ -69,16 +69,19 @@  static int32_t parse_value(const char *value, int32_t min)
 int ff_replaygain_export_raw(AVStream *st, int32_t tg, uint32_t tp,
                              int32_t ag, uint32_t ap)
 {
+    AVPacketSideData *sd;
     AVReplayGain *replaygain;
 
     if (tg == INT32_MIN && ag == INT32_MIN)
         return 0;
 
-    replaygain = (AVReplayGain*)av_stream_new_side_data(st, AV_PKT_DATA_REPLAYGAIN,
-                                                        sizeof(*replaygain));
-    if (!replaygain)
+    sd = av_packet_side_data_set_new(&st->codecpar->side_data,
+                                     AV_PKT_DATA_REPLAYGAIN,
+                                     sizeof(*replaygain), 0);
+    if (!sd)
         return AVERROR(ENOMEM);
 
+    replaygain = (AVReplayGain*)sd->data;
     replaygain->track_gain = tg;
     replaygain->track_peak = tp;
     replaygain->album_gain = ag;
diff --git a/libavformat/seek.c b/libavformat/seek.c
index 386312cd3a..0180188595 100644
--- a/libavformat/seek.c
+++ b/libavformat/seek.c
@@ -745,8 +745,10 @@  void ff_read_frame_flush(AVFormatContext *s)
         for (int j = 0; j < MAX_REORDER_DELAY + 1; j++)
             sti->pts_buffer[j] = AV_NOPTS_VALUE;
 
+#if FF_API_AVSTREAM_SIDE_DATA
         if (si->inject_global_side_data)
             sti->inject_global_side_data = 1;
+#endif
 
         sti->skip_samples = 0;
     }
diff --git a/libavformat/version_major.h b/libavformat/version_major.h
index 293fbd3397..c348e3eb37 100644
--- a/libavformat/version_major.h
+++ b/libavformat/version_major.h
@@ -45,6 +45,7 @@ 
 #define FF_API_GET_END_PTS              (LIBAVFORMAT_VERSION_MAJOR < 61)
 #define FF_API_AVIODIRCONTEXT           (LIBAVFORMAT_VERSION_MAJOR < 61)
 #define FF_API_AVFORMAT_IO_CLOSE        (LIBAVFORMAT_VERSION_MAJOR < 61)
+#define FF_API_AVSTREAM_SIDE_DATA       (LIBAVFORMAT_VERSION_MAJOR < 61)
 
 
 #define FF_API_R_FRAME_RATE            1