diff mbox series

[FFmpeg-devel,01/13] avcodec/mpegvideo_enc: Fix abort on allocation errors

Message ID AS8P250MB074414A7969C8EDBBF6EA09E8FC9A@AS8P250MB0744.EURP250.PROD.OUTLOOK.COM
State Accepted
Commit b96ba62bddd1e06f42bad1407455834b19686e32
Headers show
Series [FFmpeg-devel,01/13] avcodec/mpegvideo_enc: Fix abort on allocation errors | expand

Checks

Context Check Description
yinshiyou/make_loongarch64 success Make finished
yinshiyou/make_fate_loongarch64 success Make fate finished

Commit Message

Andreas Rheinhardt Oct. 6, 2023, 2:38 a.m. UTC
mpegvideo_enc uses a fixed-size array of Pictures; a slot is
considered taken if the Picture's AVFrame is set.
When an error happens after a slot has been taken, this Picture
has typically not been reset and is therefore not usable for
future requests. The code aborts when one runs out of slots
and this can happen in case of allocation failures.
Fix this by always unreferencing a Picture in case of errors.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
 libavcodec/mpegvideo_enc.c | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)
diff mbox series

Patch

diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c
index 5bf4b06a11..6cd9d89e1b 100644
--- a/libavcodec/mpegvideo_enc.c
+++ b/libavcodec/mpegvideo_enc.c
@@ -1223,8 +1223,10 @@  static int load_input_picture(MpegEncContext *s, const AVFrame *pic_arg)
             }
         }
         ret = av_frame_copy_props(pic->f, pic_arg);
-        if (ret < 0)
+        if (ret < 0) {
+            ff_mpeg_unref_picture(s->avctx, pic);
             return ret;
+        }
 
         pic->display_picture_number = display_picture_number;
         pic->f->pts = pts; // we set this here to avoid modifying pic_arg
@@ -1535,8 +1537,10 @@  static int select_input_picture(MpegEncContext *s)
                 }
             } else if (s->b_frame_strategy == 2) {
                 b_frames = estimate_best_b_count(s);
-                if (b_frames < 0)
+                if (b_frames < 0) {
+                    ff_mpeg_unref_picture(s->avctx, s->input_picture[0]);
                     return b_frames;
+                }
             }
 
             emms_c();
@@ -1591,7 +1595,7 @@  no_output_pic:
 
         if ((ret = av_frame_ref(s->new_picture,
                                 s->reordered_input_picture[0]->f)))
-            return ret;
+            goto fail;
 
         if (s->reordered_input_picture[0]->shared || s->avctx->rc_buffer_size) {
             // input is a shared pix, so we can't modify it -> allocate a new
@@ -1604,13 +1608,15 @@  no_output_pic:
             pic = &s->picture[i];
 
             pic->reference = s->reordered_input_picture[0]->reference;
-            if (alloc_picture(s, pic, 0) < 0) {
-                return -1;
-            }
+            ret = alloc_picture(s, pic, 0);
+            if (ret < 0)
+                goto fail;
 
             ret = av_frame_copy_props(pic->f, s->reordered_input_picture[0]->f);
-            if (ret < 0)
-                return ret;
+            if (ret < 0) {
+                ff_mpeg_unref_picture(s->avctx, pic);
+                goto fail;
+            }
             pic->coded_picture_number = s->reordered_input_picture[0]->coded_picture_number;
             pic->display_picture_number = s->reordered_input_picture[0]->display_picture_number;
 
@@ -1631,6 +1637,9 @@  no_output_pic:
 
     }
     return 0;
+fail:
+    ff_mpeg_unref_picture(s->avctx, s->reordered_input_picture[0]);
+    return ret;
 }
 
 static void frame_end(MpegEncContext *s)