diff mbox series

[FFmpeg-devel,3/3] avcodec/mv30: Fix multiple integer overflows

Message ID 20201004194143.10582-3-michael@niedermayer.cc
State Accepted
Commit 77cdc684792e6ce0b95a5308d7b61a6906fb5d15
Headers show
Series [FFmpeg-devel,1/3] avcodec/h2645_parse: Limit initial skipped_bytes_pos_size to nal size / 16 | expand

Checks

Context Check Description
andriy/default pending
andriy/make success Make finished
andriy/make_fate success Make fate finished

Commit Message

Michael Niedermayer Oct. 4, 2020, 7:41 p.m. UTC 
Fixes: signed integer overflow: -895002 * 2400 cannot be represented in type 'int'
Fixes: 26052/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MV30_fuzzer-5431812577558528

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavcodec/mv30.c | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

Comments

Michael Niedermayer Oct. 17, 2020, 12:38 p.m. UTC | #1 
On Sun, Oct 04, 2020 at 09:41:43PM +0200, Michael Niedermayer wrote:
> Fixes: signed integer overflow: -895002 * 2400 cannot be represented in type 'int'
> Fixes: 26052/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MV30_fuzzer-5431812577558528
> 
> Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
> Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
> ---
>  libavcodec/mv30.c | 24 ++++++++++++------------
>  1 file changed, 12 insertions(+), 12 deletions(-)

will apply

[...]
diff mbox series

Patch

diff --git a/libavcodec/mv30.c b/libavcodec/mv30.c
index ff60be881d..9f28199478 100644
--- a/libavcodec/mv30.c
+++ b/libavcodec/mv30.c
@@ -198,12 +198,12 @@  static void idct_add(uint8_t *dst, int stride,
 
 static inline void idct2_1d(int *blk, int step)
 {
-    const int t0 = blk[0 * step];
-    const int t1 = blk[1 * step];
-    const int t2 = (int)(t1 * 473U) >> 8;
-    const int t3 = t2 - t1;
-    const int t4 =  ((int)(t1 * 362U) >> 8) - t3;
-    const int t5 = (((int)(t1 * 277U) >> 8) - t2) + t4;
+    const unsigned int  t0 = blk[0 * step];
+    const unsigned int t1 = blk[1 * step];
+    const unsigned int t2 = (int)(t1 * 473U) >> 8;
+    const unsigned int t3 = t2 - t1;
+    const unsigned int t4 =  ((int)(t1 * 362U) >> 8) - t3;
+    const unsigned int t5 = (((int)(t1 * 277U) >> 8) - t2) + t4;
 
     blk[0 * step] = t1 + t0;
     blk[1 * step] = t0 + t3;
@@ -305,14 +305,14 @@  static int decode_intra_block(AVCodecContext *avctx, int mode,
     case 1:
         fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
         pfill[0] += fill;
-        block[0] = ((pfill[0] * qtab[0]) >> 5) + 128;
+        block[0] = ((int)((unsigned)pfill[0] * qtab[0]) >> 5) + 128;
         s->bdsp.fill_block_tab[1](dst, block[0], linesize, 8);
         break;
     case 2:
         memset(block, 0, sizeof(*block) * 64);
         fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
         pfill[0] += fill;
-        block[0] = pfill[0] * qtab[0];
+        block[0] = (unsigned)pfill[0] * qtab[0];
         block[1] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[1];
         block[8] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[8];
         block[9] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[9];
@@ -321,7 +321,7 @@  static int decode_intra_block(AVCodecContext *avctx, int mode,
     case 3:
         fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
         pfill[0] += fill;
-        block[0] = pfill[0] * qtab[0];
+        block[0] = (unsigned)pfill[0] * qtab[0];
         for (int i = 1; i < 64; i++)
             block[zigzag[i]] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[zigzag[i]];
         idct_put(dst, linesize, block);
@@ -346,14 +346,14 @@  static int decode_inter_block(AVCodecContext *avctx, int mode,
     case 1:
         fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
         pfill[0] += fill;
-        block[0] = (pfill[0] * qtab[0]) >> 5;
+        block[0] = (int)((unsigned)pfill[0] * qtab[0]) >> 5;
         update_inter_block(dst, linesize, src, in_linesize, block[0]);
         break;
     case 2:
         memset(block, 0, sizeof(*block) * 64);
         fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
         pfill[0] += fill;
-        block[0] = pfill[0] * qtab[0];
+        block[0] = (unsigned)pfill[0] * qtab[0];
         block[1] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[1];
         block[8] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[8];
         block[9] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[9];
@@ -362,7 +362,7 @@  static int decode_inter_block(AVCodecContext *avctx, int mode,
     case 3:
         fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
         pfill[0] += fill;
-        block[0] = pfill[0] * qtab[0];
+        block[0] = (unsigned)pfill[0] * qtab[0];
         for (int i = 1; i < 64; i++)
             block[zigzag[i]] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[zigzag[i]];
         idct_add(dst, linesize, src, in_linesize, block);