diff mbox series

[FFmpeg-devel,v3] avcodec/v4l2_m2m_dec: Remove redundant packet and fix double free

Message ID 20200509173541.7667-1-andriy.gelman@gmail.com
State New
Headers show
Series [FFmpeg-devel,v3] avcodec/v4l2_m2m_dec: Remove redundant packet and fix double free
Related show

Checks

Context Check Description
andriy/default pending
andriy/make success Make finished
andriy/make_fate success Make fate finished

Commit Message

Andriy Gelman May 9, 2020, 5:35 p.m. UTC
From: Andriy Gelman <andriy.gelman@gmail.com>

v4l2_receive_frame() uses two packets s->buf_pkt and avpkt. If avpkt
cannot be enqueued, the packet is buffered in s->buf_pkt and enqueued in
the next call. Currently the ownership transfer between the two packets
is not properly handled. A double free occurs if
ff_v4l2_context_enqueue_packet() returns EAGAIN and v4l2_try_start
returns EINVAL.

In fact, having two AVPackets is not needed and everything can be
handled by s->buf_pkt.

This commit removes the local avpkt from v4l2_receive_frame(), meaning
that the ownership transfer doesn't need to be handled and the double
free is fixed.

Signed-off-by: Andriy Gelman <andriy.gelman@gmail.com>
---

Sorry, forgot to squash the commit from v1 so v2 didn't apply. This is correct version.

Supersedes:
https://patchwork.ffmpeg.org/project/ffmpeg/patch/20200505055454.28683-1-andriy.gelman@gmail.com/


 libavcodec/v4l2_m2m_dec.c | 37 +++++++++++++++----------------------
 1 file changed, 15 insertions(+), 22 deletions(-)

Comments

Andriy Gelman May 16, 2020, 3:53 p.m. UTC | #1
On Sat, 09. May 13:35, Andriy Gelman wrote:
> From: Andriy Gelman <andriy.gelman@gmail.com>
> 
> v4l2_receive_frame() uses two packets s->buf_pkt and avpkt. If avpkt
> cannot be enqueued, the packet is buffered in s->buf_pkt and enqueued in
> the next call. Currently the ownership transfer between the two packets
> is not properly handled. A double free occurs if
> ff_v4l2_context_enqueue_packet() returns EAGAIN and v4l2_try_start
> returns EINVAL.
> 
> In fact, having two AVPackets is not needed and everything can be
> handled by s->buf_pkt.
> 
> This commit removes the local avpkt from v4l2_receive_frame(), meaning
> that the ownership transfer doesn't need to be handled and the double
> free is fixed.
> 
> Signed-off-by: Andriy Gelman <andriy.gelman@gmail.com>
> ---
> 
> Sorry, forgot to squash the commit from v1 so v2 didn't apply. This is correct version.
> 
> Supersedes:
> https://patchwork.ffmpeg.org/project/ffmpeg/patch/20200505055454.28683-1-andriy.gelman@gmail.com/
> 
> 
>  libavcodec/v4l2_m2m_dec.c | 37 +++++++++++++++----------------------
>  1 file changed, 15 insertions(+), 22 deletions(-)
> 
> diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c
> index 3e17e0fcac..b038efed9c 100644
> --- a/libavcodec/v4l2_m2m_dec.c
> +++ b/libavcodec/v4l2_m2m_dec.c
> @@ -138,14 +138,10 @@ static int v4l2_receive_frame(AVCodecContext *avctx, AVFrame *frame)
>      V4L2m2mContext *s = ((V4L2m2mPriv*)avctx->priv_data)->context;
>      V4L2Context *const capture = &s->capture;
>      V4L2Context *const output = &s->output;
> -    AVPacket avpkt = {0};
>      int ret;
>  
> -    if (s->buf_pkt.size) {
> -        avpkt = s->buf_pkt;
> -        memset(&s->buf_pkt, 0, sizeof(AVPacket));
> -    } else {
> -        ret = ff_decode_get_packet(avctx, &avpkt);
> +    if (!s->buf_pkt.size) {
> +        ret = ff_decode_get_packet(avctx, &s->buf_pkt);
>          if (ret < 0 && ret != AVERROR_EOF)
>              return ret;
>      }
> @@ -153,32 +149,29 @@ static int v4l2_receive_frame(AVCodecContext *avctx, AVFrame *frame)
>      if (s->draining)
>          goto dequeue;
>  
> -    ret = ff_v4l2_context_enqueue_packet(output, &avpkt);
> -    if (ret < 0) {
> -        if (ret != AVERROR(EAGAIN))
> -           return ret;
> +    ret = ff_v4l2_context_enqueue_packet(output, &s->buf_pkt);
> +    if (ret < 0 && ret != AVERROR(EAGAIN))
> +        goto fail;
>  
> -        s->buf_pkt = avpkt;
> -        /* no input buffers available, continue dequeing */
> -    }
> +    /* if EAGAIN don't unref packet and try to enqueue in the next iteration */
> +    if (ret != AVERROR(EAGAIN))
> +        av_packet_unref(&s->buf_pkt);
>  
> -    if (avpkt.size) {
> +    if (!s->draining) {
>          ret = v4l2_try_start(avctx);
>          if (ret) {
> -            av_packet_unref(&avpkt);
> -
>              /* cant recover */
> -            if (ret == AVERROR(ENOMEM))
> -                return ret;
> -
> -            return 0;
> +            if (ret != AVERROR(ENOMEM))
> +                ret = 0;
> +            goto fail;
>          }
>      }
>  
>  dequeue:
> -    if (!s->buf_pkt.size)
> -        av_packet_unref(&avpkt);
>      return ff_v4l2_context_dequeue_frame(capture, frame, -1);
> +fail:
> +    av_packet_unref(&s->buf_pkt);
> +    return ret;
>  }
>  
>  static av_cold int v4l2_decode_init(AVCodecContext *avctx)
> -- 
> 2.25.1
> 

ping
Andriy Gelman May 25, 2020, 6:59 p.m. UTC | #2
On Sat, 16. May 11:53, Andriy Gelman wrote:
> On Sat, 09. May 13:35, Andriy Gelman wrote:
> > From: Andriy Gelman <andriy.gelman@gmail.com>
> > 
> > v4l2_receive_frame() uses two packets s->buf_pkt and avpkt. If avpkt
> > cannot be enqueued, the packet is buffered in s->buf_pkt and enqueued in
> > the next call. Currently the ownership transfer between the two packets
> > is not properly handled. A double free occurs if
> > ff_v4l2_context_enqueue_packet() returns EAGAIN and v4l2_try_start
> > returns EINVAL.
> > 
> > In fact, having two AVPackets is not needed and everything can be
> > handled by s->buf_pkt.
> > 
> > This commit removes the local avpkt from v4l2_receive_frame(), meaning
> > that the ownership transfer doesn't need to be handled and the double
> > free is fixed.
> > 
> > Signed-off-by: Andriy Gelman <andriy.gelman@gmail.com>
> > ---
> > 
> > Sorry, forgot to squash the commit from v1 so v2 didn't apply. This is correct version.
> > 
> > Supersedes:
> > https://patchwork.ffmpeg.org/project/ffmpeg/patch/20200505055454.28683-1-andriy.gelman@gmail.com/
> > 
> > 
> >  libavcodec/v4l2_m2m_dec.c | 37 +++++++++++++++----------------------
> >  1 file changed, 15 insertions(+), 22 deletions(-)
> > 
> > diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c
> > index 3e17e0fcac..b038efed9c 100644
> > --- a/libavcodec/v4l2_m2m_dec.c
> > +++ b/libavcodec/v4l2_m2m_dec.c
> > @@ -138,14 +138,10 @@ static int v4l2_receive_frame(AVCodecContext *avctx, AVFrame *frame)
> >      V4L2m2mContext *s = ((V4L2m2mPriv*)avctx->priv_data)->context;
> >      V4L2Context *const capture = &s->capture;
> >      V4L2Context *const output = &s->output;
> > -    AVPacket avpkt = {0};
> >      int ret;
> >  
> > -    if (s->buf_pkt.size) {
> > -        avpkt = s->buf_pkt;
> > -        memset(&s->buf_pkt, 0, sizeof(AVPacket));
> > -    } else {
> > -        ret = ff_decode_get_packet(avctx, &avpkt);
> > +    if (!s->buf_pkt.size) {
> > +        ret = ff_decode_get_packet(avctx, &s->buf_pkt);
> >          if (ret < 0 && ret != AVERROR_EOF)
> >              return ret;
> >      }
> > @@ -153,32 +149,29 @@ static int v4l2_receive_frame(AVCodecContext *avctx, AVFrame *frame)
> >      if (s->draining)
> >          goto dequeue;
> >  
> > -    ret = ff_v4l2_context_enqueue_packet(output, &avpkt);
> > -    if (ret < 0) {
> > -        if (ret != AVERROR(EAGAIN))
> > -           return ret;
> > +    ret = ff_v4l2_context_enqueue_packet(output, &s->buf_pkt);
> > +    if (ret < 0 && ret != AVERROR(EAGAIN))
> > +        goto fail;
> >  
> > -        s->buf_pkt = avpkt;
> > -        /* no input buffers available, continue dequeing */
> > -    }
> > +    /* if EAGAIN don't unref packet and try to enqueue in the next iteration */
> > +    if (ret != AVERROR(EAGAIN))
> > +        av_packet_unref(&s->buf_pkt);
> >  
> > -    if (avpkt.size) {
> > +    if (!s->draining) {
> >          ret = v4l2_try_start(avctx);
> >          if (ret) {
> > -            av_packet_unref(&avpkt);
> > -
> >              /* cant recover */
> > -            if (ret == AVERROR(ENOMEM))
> > -                return ret;
> > -
> > -            return 0;
> > +            if (ret != AVERROR(ENOMEM))
> > +                ret = 0;
> > +            goto fail;
> >          }
> >      }
> >  
> >  dequeue:
> > -    if (!s->buf_pkt.size)
> > -        av_packet_unref(&avpkt);
> >      return ff_v4l2_context_dequeue_frame(capture, frame, -1);
> > +fail:
> > +    av_packet_unref(&s->buf_pkt);
> > +    return ret;
> >  }
> >  
> >  static av_cold int v4l2_decode_init(AVCodecContext *avctx)
> > -- 
> > 2.25.1
> > 
> 
> ping
> 

ping
Andriy Gelman June 24, 2020, 1:26 p.m. UTC | #3
On Mon, 25. May 14:59, Andriy Gelman wrote:
> On Sat, 16. May 11:53, Andriy Gelman wrote:
> > On Sat, 09. May 13:35, Andriy Gelman wrote:
> > > From: Andriy Gelman <andriy.gelman@gmail.com>
> > > 
> > > v4l2_receive_frame() uses two packets s->buf_pkt and avpkt. If avpkt
> > > cannot be enqueued, the packet is buffered in s->buf_pkt and enqueued in
> > > the next call. Currently the ownership transfer between the two packets
> > > is not properly handled. A double free occurs if
> > > ff_v4l2_context_enqueue_packet() returns EAGAIN and v4l2_try_start
> > > returns EINVAL.
> > > 
> > > In fact, having two AVPackets is not needed and everything can be
> > > handled by s->buf_pkt.
> > > 
> > > This commit removes the local avpkt from v4l2_receive_frame(), meaning
> > > that the ownership transfer doesn't need to be handled and the double
> > > free is fixed.
> > > 
> > > Signed-off-by: Andriy Gelman <andriy.gelman@gmail.com>
> > > ---
> > > 
> > > Sorry, forgot to squash the commit from v1 so v2 didn't apply. This is correct version.
> > > 
> > > Supersedes:
> > > https://patchwork.ffmpeg.org/project/ffmpeg/patch/20200505055454.28683-1-andriy.gelman@gmail.com/
> > > 
> > > 
> > >  libavcodec/v4l2_m2m_dec.c | 37 +++++++++++++++----------------------
> > >  1 file changed, 15 insertions(+), 22 deletions(-)
> > > 
> > > diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c
> > > index 3e17e0fcac..b038efed9c 100644
> > > --- a/libavcodec/v4l2_m2m_dec.c
> > > +++ b/libavcodec/v4l2_m2m_dec.c
> > > @@ -138,14 +138,10 @@ static int v4l2_receive_frame(AVCodecContext *avctx, AVFrame *frame)
> > >      V4L2m2mContext *s = ((V4L2m2mPriv*)avctx->priv_data)->context;
> > >      V4L2Context *const capture = &s->capture;
> > >      V4L2Context *const output = &s->output;
> > > -    AVPacket avpkt = {0};
> > >      int ret;
> > >  
> > > -    if (s->buf_pkt.size) {
> > > -        avpkt = s->buf_pkt;
> > > -        memset(&s->buf_pkt, 0, sizeof(AVPacket));
> > > -    } else {
> > > -        ret = ff_decode_get_packet(avctx, &avpkt);
> > > +    if (!s->buf_pkt.size) {
> > > +        ret = ff_decode_get_packet(avctx, &s->buf_pkt);
> > >          if (ret < 0 && ret != AVERROR_EOF)
> > >              return ret;
> > >      }
> > > @@ -153,32 +149,29 @@ static int v4l2_receive_frame(AVCodecContext *avctx, AVFrame *frame)
> > >      if (s->draining)
> > >          goto dequeue;
> > >  
> > > -    ret = ff_v4l2_context_enqueue_packet(output, &avpkt);
> > > -    if (ret < 0) {
> > > -        if (ret != AVERROR(EAGAIN))
> > > -           return ret;
> > > +    ret = ff_v4l2_context_enqueue_packet(output, &s->buf_pkt);
> > > +    if (ret < 0 && ret != AVERROR(EAGAIN))
> > > +        goto fail;
> > >  
> > > -        s->buf_pkt = avpkt;
> > > -        /* no input buffers available, continue dequeing */
> > > -    }
> > > +    /* if EAGAIN don't unref packet and try to enqueue in the next iteration */
> > > +    if (ret != AVERROR(EAGAIN))
> > > +        av_packet_unref(&s->buf_pkt);
> > >  
> > > -    if (avpkt.size) {
> > > +    if (!s->draining) {
> > >          ret = v4l2_try_start(avctx);
> > >          if (ret) {
> > > -            av_packet_unref(&avpkt);
> > > -
> > >              /* cant recover */
> > > -            if (ret == AVERROR(ENOMEM))
> > > -                return ret;
> > > -
> > > -            return 0;
> > > +            if (ret != AVERROR(ENOMEM))
> > > +                ret = 0;
> > > +            goto fail;
> > >          }
> > >      }
> > >  
> > >  dequeue:
> > > -    if (!s->buf_pkt.size)
> > > -        av_packet_unref(&avpkt);
> > >      return ff_v4l2_context_dequeue_frame(capture, frame, -1);
> > > +fail:
> > > +    av_packet_unref(&s->buf_pkt);
> > > +    return ret;
> > >  }
> > >  
> > >  static av_cold int v4l2_decode_init(AVCodecContext *avctx)
> > > -- 
> > > 2.25.1
> > > 
> > 
> > ping
> > 
> 
> ping
> 

ping
diff mbox series

Patch

diff --git a/libavcodec/v4l2_m2m_dec.c b/libavcodec/v4l2_m2m_dec.c
index 3e17e0fcac..b038efed9c 100644
--- a/libavcodec/v4l2_m2m_dec.c
+++ b/libavcodec/v4l2_m2m_dec.c
@@ -138,14 +138,10 @@  static int v4l2_receive_frame(AVCodecContext *avctx, AVFrame *frame)
     V4L2m2mContext *s = ((V4L2m2mPriv*)avctx->priv_data)->context;
     V4L2Context *const capture = &s->capture;
     V4L2Context *const output = &s->output;
-    AVPacket avpkt = {0};
     int ret;
 
-    if (s->buf_pkt.size) {
-        avpkt = s->buf_pkt;
-        memset(&s->buf_pkt, 0, sizeof(AVPacket));
-    } else {
-        ret = ff_decode_get_packet(avctx, &avpkt);
+    if (!s->buf_pkt.size) {
+        ret = ff_decode_get_packet(avctx, &s->buf_pkt);
         if (ret < 0 && ret != AVERROR_EOF)
             return ret;
     }
@@ -153,32 +149,29 @@  static int v4l2_receive_frame(AVCodecContext *avctx, AVFrame *frame)
     if (s->draining)
         goto dequeue;
 
-    ret = ff_v4l2_context_enqueue_packet(output, &avpkt);
-    if (ret < 0) {
-        if (ret != AVERROR(EAGAIN))
-           return ret;
+    ret = ff_v4l2_context_enqueue_packet(output, &s->buf_pkt);
+    if (ret < 0 && ret != AVERROR(EAGAIN))
+        goto fail;
 
-        s->buf_pkt = avpkt;
-        /* no input buffers available, continue dequeing */
-    }
+    /* if EAGAIN don't unref packet and try to enqueue in the next iteration */
+    if (ret != AVERROR(EAGAIN))
+        av_packet_unref(&s->buf_pkt);
 
-    if (avpkt.size) {
+    if (!s->draining) {
         ret = v4l2_try_start(avctx);
         if (ret) {
-            av_packet_unref(&avpkt);
-
             /* cant recover */
-            if (ret == AVERROR(ENOMEM))
-                return ret;
-
-            return 0;
+            if (ret != AVERROR(ENOMEM))
+                ret = 0;
+            goto fail;
         }
     }
 
 dequeue:
-    if (!s->buf_pkt.size)
-        av_packet_unref(&avpkt);
     return ff_v4l2_context_dequeue_frame(capture, frame, -1);
+fail:
+    av_packet_unref(&s->buf_pkt);
+    return ret;
 }
 
 static av_cold int v4l2_decode_init(AVCodecContext *avctx)