diff mbox

[FFmpeg-devel] avcodec/extract_extradata: don't allocate more space than needed when removing NALUs in h264/hevc

Message ID 20180309174000.7596-1-jamrial@gmail.com
State Accepted
Commit 70f11ee9981fce6145795b9c970343eb9237812d
Headers show

Commit Message

James Almer March 9, 2018, 5:40 p.m. UTC
Signed-off-by: James Almer <jamrial@gmail.com>
---
 libavcodec/extract_extradata_bsf.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

Comments

James Almer March 10, 2018, 3:48 p.m. UTC | #1
On 3/9/2018 2:40 PM, James Almer wrote:
> Signed-off-by: James Almer <jamrial@gmail.com>
> ---
>  libavcodec/extract_extradata_bsf.c | 12 +++++++-----
>  1 file changed, 7 insertions(+), 5 deletions(-)
> 
> diff --git a/libavcodec/extract_extradata_bsf.c b/libavcodec/extract_extradata_bsf.c
> index fbfd12aeef..4e2d601742 100644
> --- a/libavcodec/extract_extradata_bsf.c
> +++ b/libavcodec/extract_extradata_bsf.c
> @@ -62,7 +62,7 @@ static int extract_extradata_h2645(AVBSFContext *ctx, AVPacket *pkt,
>      ExtractExtradataContext *s = ctx->priv_data;
>  
>      H2645Packet h2645_pkt = { 0 };
> -    int extradata_size = 0;
> +    int extradata_size = 0, filtered_size = 0;
>      const int *extradata_nal_types;
>      int nb_extradata_nal_types;
>      int i, has_sps = 0, has_vps = 0, ret = 0;
> @@ -90,6 +90,8 @@ static int extract_extradata_h2645(AVBSFContext *ctx, AVPacket *pkt,
>              } else {
>                  if (nal->type == H264_NAL_SPS) has_sps = 1;
>              }
> +        } else if (s->remove) {
> +            filtered_size += nal->raw_size + 3;
>          }
>      }
>  
> @@ -100,11 +102,13 @@ static int extract_extradata_h2645(AVBSFContext *ctx, AVPacket *pkt,
>          uint8_t *extradata, *filtered_data;
>  
>          if (s->remove) {
> -            filtered_buf = av_buffer_alloc(pkt->size + AV_INPUT_BUFFER_PADDING_SIZE);
> +            filtered_buf = av_buffer_alloc(filtered_size + AV_INPUT_BUFFER_PADDING_SIZE);
>              if (!filtered_buf) {
>                  ret = AVERROR(ENOMEM);
>                  goto fail;
>              }
> +            memset(filtered_buf->data + filtered_size, 0, AV_INPUT_BUFFER_PADDING_SIZE);
> +
>              filtered_data = filtered_buf->data;
>          }
>  
> @@ -137,9 +141,7 @@ static int extract_extradata_h2645(AVBSFContext *ctx, AVPacket *pkt,
>              av_buffer_unref(&pkt->buf);
>              pkt->buf  = filtered_buf;
>              pkt->data = filtered_buf->data;
> -            pkt->size = filtered_data - filtered_buf->data;
> -
> -            memset(pkt->data + pkt->size, 0, AV_INPUT_BUFFER_PADDING_SIZE);
> +            pkt->size = filtered_size;
>          }
>      }

Will push this later today.
James Almer March 10, 2018, 11:01 p.m. UTC | #2
On 3/10/2018 12:48 PM, James Almer wrote:
> On 3/9/2018 2:40 PM, James Almer wrote:
>> Signed-off-by: James Almer <jamrial@gmail.com>
>> ---
>>  libavcodec/extract_extradata_bsf.c | 12 +++++++-----
>>  1 file changed, 7 insertions(+), 5 deletions(-)
>>
> Will push this later today.
> 

Pushed.
diff mbox

Patch

diff --git a/libavcodec/extract_extradata_bsf.c b/libavcodec/extract_extradata_bsf.c
index fbfd12aeef..4e2d601742 100644
--- a/libavcodec/extract_extradata_bsf.c
+++ b/libavcodec/extract_extradata_bsf.c
@@ -62,7 +62,7 @@  static int extract_extradata_h2645(AVBSFContext *ctx, AVPacket *pkt,
     ExtractExtradataContext *s = ctx->priv_data;
 
     H2645Packet h2645_pkt = { 0 };
-    int extradata_size = 0;
+    int extradata_size = 0, filtered_size = 0;
     const int *extradata_nal_types;
     int nb_extradata_nal_types;
     int i, has_sps = 0, has_vps = 0, ret = 0;
@@ -90,6 +90,8 @@  static int extract_extradata_h2645(AVBSFContext *ctx, AVPacket *pkt,
             } else {
                 if (nal->type == H264_NAL_SPS) has_sps = 1;
             }
+        } else if (s->remove) {
+            filtered_size += nal->raw_size + 3;
         }
     }
 
@@ -100,11 +102,13 @@  static int extract_extradata_h2645(AVBSFContext *ctx, AVPacket *pkt,
         uint8_t *extradata, *filtered_data;
 
         if (s->remove) {
-            filtered_buf = av_buffer_alloc(pkt->size + AV_INPUT_BUFFER_PADDING_SIZE);
+            filtered_buf = av_buffer_alloc(filtered_size + AV_INPUT_BUFFER_PADDING_SIZE);
             if (!filtered_buf) {
                 ret = AVERROR(ENOMEM);
                 goto fail;
             }
+            memset(filtered_buf->data + filtered_size, 0, AV_INPUT_BUFFER_PADDING_SIZE);
+
             filtered_data = filtered_buf->data;
         }
 
@@ -137,9 +141,7 @@  static int extract_extradata_h2645(AVBSFContext *ctx, AVPacket *pkt,
             av_buffer_unref(&pkt->buf);
             pkt->buf  = filtered_buf;
             pkt->data = filtered_buf->data;
-            pkt->size = filtered_data - filtered_buf->data;
-
-            memset(pkt->data + pkt->size, 0, AV_INPUT_BUFFER_PADDING_SIZE);
+            pkt->size = filtered_size;
         }
     }