diff mbox series

[FFmpeg-devel] avcodec/h2645_parse: remove initial skipped_bytes_pos buffer

Message ID 20201009213510.14533-1-jamrial@gmail.com
State New
Headers show
Series [FFmpeg-devel] avcodec/h2645_parse: remove initial skipped_bytes_pos buffer
Related show

Checks

Context Check Description
andriy/make success Make finished
andriy/make_fate success Make fate finished
andriy/PPC64_make warning Make failed

Commit Message

James Almer Oct. 9, 2020, 9:35 p.m. UTC
Allocate it only when it's needed.

Fixes: OOM
Fixes: 23817/clusterfuzz-testcase-minimized-ffmpeg_BSF_H264_METADATA_fuzzer-6300869057576960

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: James Almer <jamrial@gmail.com>
---
 libavcodec/h2645_parse.c | 28 ++++++++++------------------
 1 file changed, 10 insertions(+), 18 deletions(-)

Comments

Andreas Rheinhardt Oct. 9, 2020, 10:43 p.m. UTC | #1
James Almer:
> Allocate it only when it's needed.
> 
> Fixes: OOM
> Fixes: 23817/clusterfuzz-testcase-minimized-ffmpeg_BSF_H264_METADATA_fuzzer-6300869057576960
> 
> Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
> Signed-off-by: James Almer <jamrial@gmail.com>
> ---
>  libavcodec/h2645_parse.c | 28 ++++++++++------------------
>  1 file changed, 10 insertions(+), 18 deletions(-)
> 
> diff --git a/libavcodec/h2645_parse.c b/libavcodec/h2645_parse.c
> index 0f98b49fbe..929fb34eef 100644
> --- a/libavcodec/h2645_parse.c
> +++ b/libavcodec/h2645_parse.c
> @@ -108,22 +108,20 @@ int ff_h2645_extract_rbsp(const uint8_t *src, int length,
>                  dst[di++] = 0;
>                  si       += 3;
>  
> -                if (nal->skipped_bytes_pos) {
> -                    nal->skipped_bytes++;
> -                    if (nal->skipped_bytes_pos_size < nal->skipped_bytes) {
> -                        nal->skipped_bytes_pos_size *= 2;
> -                        av_assert0(nal->skipped_bytes_pos_size >= nal->skipped_bytes);
> -                        av_reallocp_array(&nal->skipped_bytes_pos,
> +                nal->skipped_bytes++;
> +                if (nal->skipped_bytes_pos_size < nal->skipped_bytes) {
> +                    nal->skipped_bytes_pos_size = nal->skipped_bytes * 2;
> +                    av_assert0(nal->skipped_bytes_pos_size >= nal->skipped_bytes);
> +                    av_reallocp_array(&nal->skipped_bytes_pos,
>                                  nal->skipped_bytes_pos_size,
>                                  sizeof(*nal->skipped_bytes_pos));
> -                        if (!nal->skipped_bytes_pos) {
> -                            nal->skipped_bytes_pos_size = 0;
> -                            return AVERROR(ENOMEM);
> -                        }
> +                    if (!nal->skipped_bytes_pos) {
> +                        nal->skipped_bytes_pos_size = 0;
> +                        return AVERROR(ENOMEM);
>                      }
> -                    if (nal->skipped_bytes_pos)
> -                        nal->skipped_bytes_pos[nal->skipped_bytes-1] = di - 1;
>                  }
> +                if (nal->skipped_bytes_pos)
> +                    nal->skipped_bytes_pos[nal->skipped_bytes-1] = di - 1;
>                  continue;
>              } else // next start code
>                  goto nsc;
> @@ -466,12 +464,6 @@ int ff_h2645_packet_split(H2645Packet *pkt, const uint8_t *buf, int length,
>              pkt->nals = tmp;
>              memset(pkt->nals + pkt->nals_allocated, 0, sizeof(*pkt->nals));
>  
> -            nal = &pkt->nals[pkt->nb_nals];
> -            nal->skipped_bytes_pos_size = 1024; // initial buffer size
> -            nal->skipped_bytes_pos = av_malloc_array(nal->skipped_bytes_pos_size, sizeof(*nal->skipped_bytes_pos));
> -            if (!nal->skipped_bytes_pos)
> -                return AVERROR(ENOMEM);
> -
>              pkt->nals_allocated = new_size;
>          }
>          nal = &pkt->nals[pkt->nb_nals];
> 
This has the disadvantage that it always keeps the position of the 0x03
even when the caller does not want it; most callers don't want it, the
only exception is the HEVC decoder. Your patch will lead to memleaks for
the users that use ff_h2645_extract_rbsp() directly.

- Andreas
James Almer Oct. 9, 2020, 11:35 p.m. UTC | #2
On 10/9/2020 7:43 PM, Andreas Rheinhardt wrote:
> James Almer:
>> Allocate it only when it's needed.
>>
>> Fixes: OOM
>> Fixes: 23817/clusterfuzz-testcase-minimized-ffmpeg_BSF_H264_METADATA_fuzzer-6300869057576960
>>
>> Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
>> Signed-off-by: James Almer <jamrial@gmail.com>
>> ---
>>  libavcodec/h2645_parse.c | 28 ++++++++++------------------
>>  1 file changed, 10 insertions(+), 18 deletions(-)
>>
>> diff --git a/libavcodec/h2645_parse.c b/libavcodec/h2645_parse.c
>> index 0f98b49fbe..929fb34eef 100644
>> --- a/libavcodec/h2645_parse.c
>> +++ b/libavcodec/h2645_parse.c
>> @@ -108,22 +108,20 @@ int ff_h2645_extract_rbsp(const uint8_t *src, int length,
>>                  dst[di++] = 0;
>>                  si       += 3;
>>  
>> -                if (nal->skipped_bytes_pos) {
>> -                    nal->skipped_bytes++;
>> -                    if (nal->skipped_bytes_pos_size < nal->skipped_bytes) {
>> -                        nal->skipped_bytes_pos_size *= 2;
>> -                        av_assert0(nal->skipped_bytes_pos_size >= nal->skipped_bytes);
>> -                        av_reallocp_array(&nal->skipped_bytes_pos,
>> +                nal->skipped_bytes++;
>> +                if (nal->skipped_bytes_pos_size < nal->skipped_bytes) {
>> +                    nal->skipped_bytes_pos_size = nal->skipped_bytes * 2;
>> +                    av_assert0(nal->skipped_bytes_pos_size >= nal->skipped_bytes);
>> +                    av_reallocp_array(&nal->skipped_bytes_pos,
>>                                  nal->skipped_bytes_pos_size,
>>                                  sizeof(*nal->skipped_bytes_pos));
>> -                        if (!nal->skipped_bytes_pos) {
>> -                            nal->skipped_bytes_pos_size = 0;
>> -                            return AVERROR(ENOMEM);
>> -                        }
>> +                    if (!nal->skipped_bytes_pos) {
>> +                        nal->skipped_bytes_pos_size = 0;
>> +                        return AVERROR(ENOMEM);
>>                      }
>> -                    if (nal->skipped_bytes_pos)
>> -                        nal->skipped_bytes_pos[nal->skipped_bytes-1] = di - 1;
>>                  }
>> +                if (nal->skipped_bytes_pos)
>> +                    nal->skipped_bytes_pos[nal->skipped_bytes-1] = di - 1;
>>                  continue;
>>              } else // next start code
>>                  goto nsc;
>> @@ -466,12 +464,6 @@ int ff_h2645_packet_split(H2645Packet *pkt, const uint8_t *buf, int length,
>>              pkt->nals = tmp;
>>              memset(pkt->nals + pkt->nals_allocated, 0, sizeof(*pkt->nals));
>>  
>> -            nal = &pkt->nals[pkt->nb_nals];
>> -            nal->skipped_bytes_pos_size = 1024; // initial buffer size
>> -            nal->skipped_bytes_pos = av_malloc_array(nal->skipped_bytes_pos_size, sizeof(*nal->skipped_bytes_pos));
>> -            if (!nal->skipped_bytes_pos)
>> -                return AVERROR(ENOMEM);
>> -
>>              pkt->nals_allocated = new_size;
>>          }
>>          nal = &pkt->nals[pkt->nb_nals];
>>
> This has the disadvantage that it always keeps the position of the 0x03
> even when the caller does not want it; most callers don't want it, the
> only exception is the HEVC decoder. Your patch will lead to memleaks for
> the users that use ff_h2645_extract_rbsp() directly.

Those users can be adapted to free the buffer, but then again a flag can
be added to ff_h2645_extract_rbsp() for this purpose, i guess.

> 
> - Andreas
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
diff mbox series

Patch

diff --git a/libavcodec/h2645_parse.c b/libavcodec/h2645_parse.c
index 0f98b49fbe..929fb34eef 100644
--- a/libavcodec/h2645_parse.c
+++ b/libavcodec/h2645_parse.c
@@ -108,22 +108,20 @@  int ff_h2645_extract_rbsp(const uint8_t *src, int length,
                 dst[di++] = 0;
                 si       += 3;
 
-                if (nal->skipped_bytes_pos) {
-                    nal->skipped_bytes++;
-                    if (nal->skipped_bytes_pos_size < nal->skipped_bytes) {
-                        nal->skipped_bytes_pos_size *= 2;
-                        av_assert0(nal->skipped_bytes_pos_size >= nal->skipped_bytes);
-                        av_reallocp_array(&nal->skipped_bytes_pos,
+                nal->skipped_bytes++;
+                if (nal->skipped_bytes_pos_size < nal->skipped_bytes) {
+                    nal->skipped_bytes_pos_size = nal->skipped_bytes * 2;
+                    av_assert0(nal->skipped_bytes_pos_size >= nal->skipped_bytes);
+                    av_reallocp_array(&nal->skipped_bytes_pos,
                                 nal->skipped_bytes_pos_size,
                                 sizeof(*nal->skipped_bytes_pos));
-                        if (!nal->skipped_bytes_pos) {
-                            nal->skipped_bytes_pos_size = 0;
-                            return AVERROR(ENOMEM);
-                        }
+                    if (!nal->skipped_bytes_pos) {
+                        nal->skipped_bytes_pos_size = 0;
+                        return AVERROR(ENOMEM);
                     }
-                    if (nal->skipped_bytes_pos)
-                        nal->skipped_bytes_pos[nal->skipped_bytes-1] = di - 1;
                 }
+                if (nal->skipped_bytes_pos)
+                    nal->skipped_bytes_pos[nal->skipped_bytes-1] = di - 1;
                 continue;
             } else // next start code
                 goto nsc;
@@ -466,12 +464,6 @@  int ff_h2645_packet_split(H2645Packet *pkt, const uint8_t *buf, int length,
             pkt->nals = tmp;
             memset(pkt->nals + pkt->nals_allocated, 0, sizeof(*pkt->nals));
 
-            nal = &pkt->nals[pkt->nb_nals];
-            nal->skipped_bytes_pos_size = 1024; // initial buffer size
-            nal->skipped_bytes_pos = av_malloc_array(nal->skipped_bytes_pos_size, sizeof(*nal->skipped_bytes_pos));
-            if (!nal->skipped_bytes_pos)
-                return AVERROR(ENOMEM);
-
             pkt->nals_allocated = new_size;
         }
         nal = &pkt->nals[pkt->nb_nals];