| Message ID | 20201104204702.256762-1-andreas.rheinhardt@gmail.com |
|---|---|
| State | Accepted |
| Commit | 4b4c7d6e1a700ef6b784b2f0b093e200ed049c20 |
| Headers | show |
| Series | [FFmpeg-devel] avcodec/mss4: Fix memleaks upon allocation error | expand |
| Context | Check | Description |
|---|---|---|
| andriy/x86_make | success | Make finished |
| andriy/x86_make_fate | success | Make fate finished |
| andriy/PPC64_make | success | Make finished |
| andriy/PPC64_make_fate | success | Make fate finished |
Andreas Rheinhardt: > During init the mts2 decoder allocates several VLCs and then several > buffers in a loop; if one of the latter allocations fails, only the VLCs > are freed, not any buffers that might already have been successfully > allocated. This commit fixes this by setting the > FF_CODEC_CAP_INIT_CLEANUP flag. > > Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com> > --- > libavcodec/mss4.c | 7 ++----- > 1 file changed, 2 insertions(+), 5 deletions(-) > > diff --git a/libavcodec/mss4.c b/libavcodec/mss4.c > index 432df294d6..c421a07c5d 100644 > --- a/libavcodec/mss4.c > +++ b/libavcodec/mss4.c > @@ -648,7 +648,6 @@ static av_cold int mss4_decode_init(AVCodecContext *avctx) > > if (mss4_init_vlcs(c)) { > av_log(avctx, AV_LOG_ERROR, "Cannot initialise VLCs\n"); > - mss4_free_vlcs(c); > return AVERROR(ENOMEM); > } > for (i = 0; i < 3; i++) { > @@ -656,16 +655,13 @@ static av_cold int mss4_decode_init(AVCodecContext *avctx) > c->prev_dc[i] = av_malloc_array(c->dc_stride[i], sizeof(**c->prev_dc)); > if (!c->prev_dc[i]) { > av_log(avctx, AV_LOG_ERROR, "Cannot allocate buffer\n"); > - mss4_free_vlcs(c); > return AVERROR(ENOMEM); > } > } > > c->pic = av_frame_alloc(); > - if (!c->pic) { > - mss4_decode_end(avctx); > + if (!c->pic) > return AVERROR(ENOMEM); > - } > > avctx->pix_fmt = AV_PIX_FMT_YUV444P; > > @@ -682,4 +678,5 @@ AVCodec ff_mts2_decoder = { > .close = mss4_decode_end, > .decode = mss4_decode_frame, > .capabilities = AV_CODEC_CAP_DR1, > + .caps_internal = FF_CODEC_CAP_INIT_CLEANUP, > }; > Will apply this patch later today unless there are objections. - Andreas
diff --git a/libavcodec/mss4.c b/libavcodec/mss4.c index 432df294d6..c421a07c5d 100644 --- a/libavcodec/mss4.c +++ b/libavcodec/mss4.c @@ -648,7 +648,6 @@ static av_cold int mss4_decode_init(AVCodecContext *avctx) if (mss4_init_vlcs(c)) { av_log(avctx, AV_LOG_ERROR, "Cannot initialise VLCs\n"); - mss4_free_vlcs(c); return AVERROR(ENOMEM); } for (i = 0; i < 3; i++) { @@ -656,16 +655,13 @@ static av_cold int mss4_decode_init(AVCodecContext *avctx) c->prev_dc[i] = av_malloc_array(c->dc_stride[i], sizeof(**c->prev_dc)); if (!c->prev_dc[i]) { av_log(avctx, AV_LOG_ERROR, "Cannot allocate buffer\n"); - mss4_free_vlcs(c); return AVERROR(ENOMEM); } } c->pic = av_frame_alloc(); - if (!c->pic) { - mss4_decode_end(avctx); + if (!c->pic) return AVERROR(ENOMEM); - } avctx->pix_fmt = AV_PIX_FMT_YUV444P; @@ -682,4 +678,5 @@ AVCodec ff_mts2_decoder = { .close = mss4_decode_end, .decode = mss4_decode_frame, .capabilities = AV_CODEC_CAP_DR1, + .caps_internal = FF_CODEC_CAP_INIT_CLEANUP, };
During init the mts2 decoder allocates several VLCs and then several buffers in a loop; if one of the latter allocations fails, only the VLCs are freed, not any buffers that might already have been successfully allocated. This commit fixes this by setting the FF_CODEC_CAP_INIT_CLEANUP flag. Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com> --- libavcodec/mss4.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-)