diff mbox series

[FFmpeg-devel,v2] avformat/mux: Fix leaks of uncoded frames on errors

Message ID 20200331133219.21079-1-andreas.rheinhardt@gmail.com
State Superseded
Headers show
Series [FFmpeg-devel,v2] avformat/mux: Fix leaks of uncoded frames on errors | expand

Checks

Context Check Description
andriy/ffmpeg-patchwork success Make fate finished

Commit Message

Andreas Rheinhardt March 31, 2020, 1:32 p.m. UTC 
If writing an uncoded frame fails at the preparatory steps of
av_[interleaved_]write_frame(), the frame would be either not freed
at all in case of av_write_frame() or would leak when the fake packet
would be unreferenced like an ordinary packet.
There is also a memleak when the output format is not suited for
writing uncoded frames as the documentation states that ownership of the
frame passes to av_[interleaved_]write_uncoded_frame(). Both of these
memleaks have been fixed.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
---
V2: Now actually resetting the data of the AVPacket containing an
uncoded frame if said frame is freed.

 libavformat/mux.c | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)
diff mbox series

Patch

diff --git a/libavformat/mux.c b/libavformat/mux.c
index 814d773b9d..dc8233a0d5 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -749,7 +749,7 @@  static int write_packet(AVFormatContext *s, AVPacket *pkt)
         AVFrame *frame = (AVFrame *)pkt->data;
         av_assert0(pkt->size == UNCODED_FRAME_PACKET_SIZE);
         ret = s->oformat->write_uncoded_frame(s, pkt->stream_index, &frame, 0);
-        av_frame_free(&frame);
+        av_frame_free((AVFrame **)&pkt->data);
     } else {
         ret = s->oformat->write_packet(s, pkt);
     }
@@ -1239,7 +1239,11 @@  int av_interleaved_write_frame(AVFormatContext *s, AVPacket *pkt)
             return ret;
     }
 fail:
-    av_packet_unref(pkt);
+    // This is a deviation from the usual behaviour
+    // of av_interleaved_write_frame: We leave cleaning
+    // up uncoded frames to write_uncoded_frame_internal.
+    if (!(pkt->flags & AV_PKT_FLAG_UNCODED_FRAME))
+        av_packet_unref(pkt);
     return ret;
 }
 
@@ -1324,10 +1328,13 @@  static int write_uncoded_frame_internal(AVFormatContext *s, int stream_index,
                                         AVFrame *frame, int interleaved)
 {
     AVPacket pkt, *pktp;
+    int ret;
 
     av_assert0(s->oformat);
-    if (!s->oformat->write_uncoded_frame)
-        return AVERROR(ENOSYS);
+    if (!s->oformat->write_uncoded_frame) {
+        ret = AVERROR(ENOSYS);
+        goto free;
+    }
 
     if (!frame) {
         pktp = NULL;
@@ -1343,8 +1350,14 @@  static int write_uncoded_frame_internal(AVFormatContext *s, int stream_index,
         pkt.flags |= AV_PKT_FLAG_UNCODED_FRAME;
     }
 
-    return interleaved ? av_interleaved_write_frame(s, pktp) :
-                         av_write_frame(s, pktp);
+    ret = interleaved ? av_interleaved_write_frame(s, pktp) :
+                        av_write_frame(s, pktp);
+    if (ret < 0 && pktp) {
+        frame = (AVFrame*)pktp->data;
+    free:
+        av_frame_free(&frame);
+    }
+    return ret;
 }
 
 int av_write_uncoded_frame(AVFormatContext *s, int stream_index,