diff mbox series

[FFmpeg-devel,3/4] avcodec/apedec: Fix integer overflows in

Message ID 20210720202942.8574-3-michael@niedermayer.cc
State New
Headers show
Series [FFmpeg-devel,1/4] tools/target_dec_fuzzer: Adjust the threshold for VP3 | expand

Checks

Context Check Description
andriy/x86_make success Make finished
andriy/x86_make_fate fail Make fate failed
andriy/PPC64_make success Make finished
andriy/PPC64_make_fate warning Make fate failed

Commit Message

Michael Niedermayer July 20, 2021, 8:29 p.m. UTC 
Fixes: signed integer overflow: -2146255836 - 197885745 cannot be represented in type 'int'
Fixes: 35911/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_APE_fuzzer-5799270526615552

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavcodec/apedec.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
diff mbox series

Patch

diff --git a/libavcodec/apedec.c b/libavcodec/apedec.c
index 022a6f9398..2bbc18fee0 100644
--- a/libavcodec/apedec.c
+++ b/libavcodec/apedec.c
@@ -909,22 +909,22 @@  static av_always_inline int filter_3800(APEPredictor *p,
         return predictionA;
     }
     d2 =  p->buf[delayA];
-    d1 = (p->buf[delayA] - p->buf[delayA - 1]) * 2U;
-    d0 =  p->buf[delayA] + ((p->buf[delayA - 2] - p->buf[delayA - 1]) * 8U);
+    d1 = (p->buf[delayA] - (unsigned)p->buf[delayA - 1]) * 2;
+    d0 =  p->buf[delayA] + ((p->buf[delayA - 2] - (unsigned)p->buf[delayA - 1]) * 8);
     d3 =  p->buf[delayB] * 2U - p->buf[delayB - 1];
     d4 =  p->buf[delayB];
 
-    predictionA = d0 * p->coeffsA[filter][0] +
-                  d1 * p->coeffsA[filter][1] +
-                  d2 * p->coeffsA[filter][2];
+    predictionA = d0 * (unsigned)p->coeffsA[filter][0] +
+                  d1 * (unsigned)p->coeffsA[filter][1] +
+                  d2 * (unsigned)p->coeffsA[filter][2];
 
     sign = APESIGN(decoded);
     p->coeffsA[filter][0] += (((d0 >> 30) & 2) - 1) * sign;
     p->coeffsA[filter][1] += (((d1 >> 28) & 8) - 4) * sign;
     p->coeffsA[filter][2] += (((d2 >> 28) & 8) - 4) * sign;
 
-    predictionB = d3 * p->coeffsB[filter][0] -
-                  d4 * p->coeffsB[filter][1];
+    predictionB = d3 * (unsigned)p->coeffsB[filter][0] -
+                  d4 * (unsigned)p->coeffsB[filter][1];
     p->lastA[filter] = decoded + (predictionA >> 11);
     sign = APESIGN(p->lastA[filter]);
     p->coeffsB[filter][0] += (((d3 >> 29) & 4) - 2) * sign;