diff mbox

[FFmpeg-devel,2/5] avcodec/apedec: Fix several integer overflows in predictor_update_filter() and do_apply_filter()

Message ID 20190903001426.8957-2-michael@niedermayer.cc
State Accepted
Commit 1e95a3e8a7250060befd9a5fba69151bb2a6690c
Headers show

Commit Message

Michael Niedermayer Sept. 3, 2019, 12:14 a.m. UTC 
Fixes: negation of -2147483648 cannot be represented in type 'int'; cast to an unsigned type to negate this value to itself
Fixes: signed integer overflow: -14527961 - 2147483425 cannot be represented in type 'int'
Fixes: 16380/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_APE_fuzzer-5645957131141120

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavcodec/apedec.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

Comments

Michael Niedermayer Sept. 28, 2019, 5:19 p.m. UTC | #1 
On Tue, Sep 03, 2019 at 02:14:23AM +0200, Michael Niedermayer wrote:
> Fixes: negation of -2147483648 cannot be represented in type 'int'; cast to an unsigned type to negate this value to itself
> Fixes: signed integer overflow: -14527961 - 2147483425 cannot be represented in type 'int'
> Fixes: 16380/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_APE_fuzzer-5645957131141120
> 
> Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
> Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
> ---
>  libavcodec/apedec.c | 6 +++---
>  1 file changed, 3 insertions(+), 3 deletions(-)

will apply

[...]
diff mbox

Patch

diff --git a/libavcodec/apedec.c b/libavcodec/apedec.c
index ed22f0f019..e218f7c043 100644
--- a/libavcodec/apedec.c
+++ b/libavcodec/apedec.c
@@ -1121,7 +1121,7 @@  static av_always_inline int predictor_update_filter(APEPredictor *p,
 
     p->buf[delayA]     = p->lastA[filter];
     p->buf[adaptA]     = APESIGN(p->buf[delayA]);
-    p->buf[delayA - 1] = p->buf[delayA] - p->buf[delayA - 1];
+    p->buf[delayA - 1] = p->buf[delayA] - (unsigned)p->buf[delayA - 1];
     p->buf[adaptA - 1] = APESIGN(p->buf[delayA - 1]);
 
     predictionA = p->buf[delayA    ] * p->coeffsA[filter][0] +
@@ -1132,7 +1132,7 @@  static av_always_inline int predictor_update_filter(APEPredictor *p,
     /*  Apply a scaled first-order filter compression */
     p->buf[delayB]     = p->filterA[filter ^ 1] - ((int)(p->filterB[filter] * 31U) >> 5);
     p->buf[adaptB]     = APESIGN(p->buf[delayB]);
-    p->buf[delayB - 1] = p->buf[delayB] - p->buf[delayB - 1];
+    p->buf[delayB - 1] = p->buf[delayB] - (unsigned)p->buf[delayB - 1];
     p->buf[adaptB - 1] = APESIGN(p->buf[delayB - 1]);
     p->filterB[filter] = p->filterA[filter ^ 1];
 
@@ -1282,7 +1282,7 @@  static void do_apply_filter(APEContext *ctx, int version, APEFilter *f,
             /* Version 3.98 and later files */
 
             /* Update the adaption coefficients */
-            absres = FFABS(res);
+            absres = res < 0 ? -(unsigned)res : res;
             if (absres)
                 *f->adaptcoeffs = APESIGN(res) *
                                   (8 << ((absres > f->avg * 3) + (absres > f->avg * 4 / 3)));