diff mbox series

[FFmpeg-devel,4/4] avcodec/bonk: Check for undefined overflow in predictor_calc_error()

Message ID 20221105201629.1980-4-michael@niedermayer.cc
State New
Headers show
Series [FFmpeg-devel,1/4] avcodec/dts2pts_bsf: Check ctx for NULL before ff_cbs_flush() | expand

Checks

Context Check Description
yinshiyou/make_loongarch64 success Make finished
yinshiyou/make_fate_loongarch64 success Make fate finished
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished

Commit Message

Michael Niedermayer Nov. 5, 2022, 8:16 p.m. UTC
Fixes: signed integer overflow: 22 * -2107998208 cannot be represented in type 'int'
Fixes: 51363/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_BONK_fuzzer-5660734784143360

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavcodec/bonk.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

Comments

Paul B Mahol Nov. 6, 2022, 8:50 a.m. UTC | #1
On 11/5/22, Michael Niedermayer <michael@niedermayer.cc> wrote:
> Fixes: signed integer overflow: 22 * -2107998208 cannot be represented in
> type 'int'
> Fixes:
> 51363/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_BONK_fuzzer-5660734784143360
>
> Found-by: continuous fuzzing process
> https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
> Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
> ---
>  libavcodec/bonk.c | 11 +++++++++--
>  1 file changed, 9 insertions(+), 2 deletions(-)
>
> diff --git a/libavcodec/bonk.c b/libavcodec/bonk.c
> index 1695229dbd..40963aa7c6 100644
> --- a/libavcodec/bonk.c
> +++ b/libavcodec/bonk.c
> @@ -278,10 +278,13 @@ static int predictor_calc_error(int *k, int *state,
> int order, int error)
>          *state_ptr = &(state[order-2]);
>
>      for (i = order-2; i >= 0; i--, k_ptr--, state_ptr--) {
> -        int k_value = *k_ptr, state_value = *state_ptr;
> +        int64_t k_value = *k_ptr, state_value = *state_ptr;
>
>          x -= shift_down(k_value * state_value, LATTICE_SHIFT);
> -        state_ptr[1] = state_value + shift_down(k_value * x,
> LATTICE_SHIFT);
> +        k_value *= x;
> +        if ((int32_t)k_value != k_value)
> +            return AVERROR_INVALIDDATA;
> +        state_ptr[1] = state_value + shift_down(k_value, LATTICE_SHIFT);
>      }
>
>      // don't drift too far, to avoid overflows
> @@ -366,6 +369,8 @@ static int bonk_decode(AVCodecContext *avctx, AVFrame
> *frame,
>              int64_t t64;
>              for (int j = 0; j < s->down_sampling - 1; j++) {
>                  sample[0] = predictor_calc_error(s->k, state, s->n_taps,
> 0);
> +                if (sample[0] == AVERROR_INVALIDDATA)
> +                    return sample[0];
>                  sample++;
>              }
>
> @@ -374,6 +379,8 @@ static int bonk_decode(AVCodecContext *avctx, AVFrame
> *frame,
>                  return AVERROR_INVALIDDATA;
>
>              sample[0] = predictor_calc_error(s->k, state, s->n_taps, t64);
> +            if (sample[0] == AVERROR_INVALIDDATA)
> +                return sample[0];
>              sample++;
>          }
>
> --
> 2.17.1
>


NAK,

slowing things down by using int64_t

> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
Michael Niedermayer Nov. 6, 2022, 12:30 p.m. UTC | #2
On Sun, Nov 06, 2022 at 09:50:47AM +0100, Paul B Mahol wrote:
> On 11/5/22, Michael Niedermayer <michael@niedermayer.cc> wrote:
> > Fixes: signed integer overflow: 22 * -2107998208 cannot be represented in
> > type 'int'
> > Fixes:
> > 51363/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_BONK_fuzzer-5660734784143360
> >
> > Found-by: continuous fuzzing process
> > https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
> > Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
> > ---
> >  libavcodec/bonk.c | 11 +++++++++--
> >  1 file changed, 9 insertions(+), 2 deletions(-)
> >
> > diff --git a/libavcodec/bonk.c b/libavcodec/bonk.c
> > index 1695229dbd..40963aa7c6 100644
> > --- a/libavcodec/bonk.c
> > +++ b/libavcodec/bonk.c
> > @@ -278,10 +278,13 @@ static int predictor_calc_error(int *k, int *state,
> > int order, int error)
> >          *state_ptr = &(state[order-2]);
> >
> >      for (i = order-2; i >= 0; i--, k_ptr--, state_ptr--) {
> > -        int k_value = *k_ptr, state_value = *state_ptr;
> > +        int64_t k_value = *k_ptr, state_value = *state_ptr;
> >
> >          x -= shift_down(k_value * state_value, LATTICE_SHIFT);
> > -        state_ptr[1] = state_value + shift_down(k_value * x,
> > LATTICE_SHIFT);
> > +        k_value *= x;
> > +        if ((int32_t)k_value != k_value)
> > +            return AVERROR_INVALIDDATA;
> > +        state_ptr[1] = state_value + shift_down(k_value, LATTICE_SHIFT);
> >      }
> >
> >      // don't drift too far, to avoid overflows
> > @@ -366,6 +369,8 @@ static int bonk_decode(AVCodecContext *avctx, AVFrame
> > *frame,
> >              int64_t t64;
> >              for (int j = 0; j < s->down_sampling - 1; j++) {
> >                  sample[0] = predictor_calc_error(s->k, state, s->n_taps,
> > 0);
> > +                if (sample[0] == AVERROR_INVALIDDATA)
> > +                    return sample[0];
> >                  sample++;
> >              }
> >
> > @@ -374,6 +379,8 @@ static int bonk_decode(AVCodecContext *avctx, AVFrame
> > *frame,
> >                  return AVERROR_INVALIDDATA;
> >
> >              sample[0] = predictor_calc_error(s->k, state, s->n_taps, t64);
> > +            if (sample[0] == AVERROR_INVALIDDATA)
> > +                return sample[0];
> >              sample++;
> >          }
> >
> > --
> > 2.17.1
> >
> 
> 
> NAK,
> 
> slowing things down by using int64_t

ill post some other solution, it seems slower indeed, i was hoping
gcc would produce smarter code

thx

[...]
diff mbox series

Patch

diff --git a/libavcodec/bonk.c b/libavcodec/bonk.c
index 1695229dbd..40963aa7c6 100644
--- a/libavcodec/bonk.c
+++ b/libavcodec/bonk.c
@@ -278,10 +278,13 @@  static int predictor_calc_error(int *k, int *state, int order, int error)
         *state_ptr = &(state[order-2]);
 
     for (i = order-2; i >= 0; i--, k_ptr--, state_ptr--) {
-        int k_value = *k_ptr, state_value = *state_ptr;
+        int64_t k_value = *k_ptr, state_value = *state_ptr;
 
         x -= shift_down(k_value * state_value, LATTICE_SHIFT);
-        state_ptr[1] = state_value + shift_down(k_value * x, LATTICE_SHIFT);
+        k_value *= x;
+        if ((int32_t)k_value != k_value)
+            return AVERROR_INVALIDDATA;
+        state_ptr[1] = state_value + shift_down(k_value, LATTICE_SHIFT);
     }
 
     // don't drift too far, to avoid overflows
@@ -366,6 +369,8 @@  static int bonk_decode(AVCodecContext *avctx, AVFrame *frame,
             int64_t t64;
             for (int j = 0; j < s->down_sampling - 1; j++) {
                 sample[0] = predictor_calc_error(s->k, state, s->n_taps, 0);
+                if (sample[0] == AVERROR_INVALIDDATA)
+                    return sample[0];
                 sample++;
             }
 
@@ -374,6 +379,8 @@  static int bonk_decode(AVCodecContext *avctx, AVFrame *frame,
                 return AVERROR_INVALIDDATA;
 
             sample[0] = predictor_calc_error(s->k, state, s->n_taps, t64);
+            if (sample[0] == AVERROR_INVALIDDATA)
+                return sample[0];
             sample++;
         }