diff mbox series

[FFmpeg-devel,4/6] avcodec/jrevdct: Fix UB left shifts of negative numbers

Message ID GV1P250MB0737ADD29F22F514991A8C068F549@GV1P250MB0737.EURP250.PROD.OUTLOOK.COM
State Accepted
Commit fe3c2c8bbe017ac2fd2cc9cab2ccb29bad52a30b
Headers show
Series [FFmpeg-devel,1/6] avcodec/g723_1enc: Remove unnecessary av_clipl_int32() | expand

Commit Message

Andreas Rheinhardt Sept. 28, 2022, 6:58 p.m. UTC 
Affected the rv20-1239 FATE test.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
 libavcodec/jrevdct.c | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

Comments

Michael Niedermayer Sept. 29, 2022, 6:04 p.m. UTC | #1 
On Wed, Sep 28, 2022 at 08:58:16PM +0200, Andreas Rheinhardt wrote:
> Affected the rv20-1239 FATE test.
> 
> Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
> ---
>  libavcodec/jrevdct.c | 38 +++++++++++++++++++-------------------
>  1 file changed, 19 insertions(+), 19 deletions(-)

iam a bit surprised this was not spotted long ago but LGTM

thx

[...]
Andreas Rheinhardt Sept. 29, 2022, 6:08 p.m. UTC | #2 
Michael Niedermayer:
> On Wed, Sep 28, 2022 at 08:58:16PM +0200, Andreas Rheinhardt wrote:
>> Affected the rv20-1239 FATE test.
>>
>> Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
>> ---
>>  libavcodec/jrevdct.c | 38 +++++++++++++++++++-------------------
>>  1 file changed, 19 insertions(+), 19 deletions(-)
> 
> iam a bit surprised this was not spotted long ago but LGTM
> 
> thx
> 

Most of these changes are in ff_j_rev_dct4 which is only used if lowres
== 1; I guess the fuzzer doesn't use lowres which explains why this has
not been found before.

- Andreas
diff mbox series

Patch

diff --git a/libavcodec/jrevdct.c b/libavcodec/jrevdct.c
index 36160cb663..7f1863515f 100644
--- a/libavcodec/jrevdct.c
+++ b/libavcodec/jrevdct.c
@@ -255,7 +255,7 @@  void ff_j_rev_dct(DCTBLOCK data)
       if (d0) {
           /* Compute a 32 bit value to assign. */
           int16_t dcval = (int16_t) (d0 * (1 << PASS1_BITS));
-          register int v = (dcval & 0xffff) | ((dcval * (1 << 16)) & 0xffff0000);
+          register unsigned v = (dcval & 0xffff) | ((uint32_t)dcval << 16);
 
           AV_WN32A(&idataptr[ 0], v);
           AV_WN32A(&idataptr[ 4], v);
@@ -988,8 +988,8 @@  void ff_j_rev_dct4(DCTBLOCK data)
       /* AC terms all zero */
       if (d0) {
           /* Compute a 32 bit value to assign. */
-          int16_t dcval = (int16_t) (d0 << PASS1_BITS);
-          register int v = (dcval & 0xffff) | ((dcval << 16) & 0xffff0000);
+          int16_t dcval = (int16_t) (d0 * (1 << PASS1_BITS));
+          register unsigned v = (dcval & 0xffff) | ((uint32_t)dcval << 16);
 
           AV_WN32A(&idataptr[0], v);
           AV_WN32A(&idataptr[4], v);
@@ -1008,8 +1008,8 @@  void ff_j_rev_dct4(DCTBLOCK data)
                     tmp2 = z1 + MULTIPLY(-d6, FIX_1_847759065);
                     tmp3 = z1 + MULTIPLY(d2, FIX_0_765366865);
 
-                    tmp0 = (d0 + d4) << CONST_BITS;
-                    tmp1 = (d0 - d4) << CONST_BITS;
+                    tmp0 = (d0 + d4) * (1 << CONST_BITS);
+                    tmp1 = (d0 - d4) * (1 << CONST_BITS);
 
                     tmp10 = tmp0 + tmp3;
                     tmp13 = tmp0 - tmp3;
@@ -1020,8 +1020,8 @@  void ff_j_rev_dct4(DCTBLOCK data)
                     tmp2 = MULTIPLY(-d6, FIX_1_306562965);
                     tmp3 = MULTIPLY(d6, FIX_0_541196100);
 
-                    tmp0 = (d0 + d4) << CONST_BITS;
-                    tmp1 = (d0 - d4) << CONST_BITS;
+                    tmp0 = (d0 + d4) * (1 << CONST_BITS);
+                    tmp1 = (d0 - d4) * (1 << CONST_BITS);
 
                     tmp10 = tmp0 + tmp3;
                     tmp13 = tmp0 - tmp3;
@@ -1034,8 +1034,8 @@  void ff_j_rev_dct4(DCTBLOCK data)
                     tmp2 = MULTIPLY(d2, FIX_0_541196100);
                     tmp3 = MULTIPLY(d2, FIX_1_306562965);
 
-                    tmp0 = (d0 + d4) << CONST_BITS;
-                    tmp1 = (d0 - d4) << CONST_BITS;
+                    tmp0 = (d0 + d4) * (1 << CONST_BITS);
+                    tmp1 = (d0 - d4) * (1 << CONST_BITS);
 
                     tmp10 = tmp0 + tmp3;
                     tmp13 = tmp0 - tmp3;
@@ -1043,8 +1043,8 @@  void ff_j_rev_dct4(DCTBLOCK data)
                     tmp12 = tmp1 - tmp2;
             } else {
                     /* d0 != 0, d2 == 0, d4 != 0, d6 == 0 */
-                    tmp10 = tmp13 = (d0 + d4) << CONST_BITS;
-                    tmp11 = tmp12 = (d0 - d4) << CONST_BITS;
+                    tmp10 = tmp13 = (d0 + d4) * (1 << CONST_BITS);
+                    tmp11 = tmp12 = (d0 - d4) * (1 << CONST_BITS);
             }
       }
 
@@ -1086,8 +1086,8 @@  void ff_j_rev_dct4(DCTBLOCK data)
                     tmp2 = z1 + MULTIPLY(-d6, FIX_1_847759065);
                     tmp3 = z1 + MULTIPLY(d2, FIX_0_765366865);
 
-                    tmp0 = (d0 + d4) << CONST_BITS;
-                    tmp1 = (d0 - d4) << CONST_BITS;
+                    tmp0 = (d0 + d4) * (1 << CONST_BITS);
+                    tmp1 = (d0 - d4) * (1 << CONST_BITS);
 
                     tmp10 = tmp0 + tmp3;
                     tmp13 = tmp0 - tmp3;
@@ -1098,8 +1098,8 @@  void ff_j_rev_dct4(DCTBLOCK data)
                     tmp2 = MULTIPLY(-d6, FIX_1_306562965);
                     tmp3 = MULTIPLY(d6, FIX_0_541196100);
 
-                    tmp0 = (d0 + d4) << CONST_BITS;
-                    tmp1 = (d0 - d4) << CONST_BITS;
+                    tmp0 = (d0 + d4) * (1 << CONST_BITS);
+                    tmp1 = (d0 - d4) * (1 << CONST_BITS);
 
                     tmp10 = tmp0 + tmp3;
                     tmp13 = tmp0 - tmp3;
@@ -1112,8 +1112,8 @@  void ff_j_rev_dct4(DCTBLOCK data)
                     tmp2 = MULTIPLY(d2, FIX_0_541196100);
                     tmp3 = MULTIPLY(d2, FIX_1_306562965);
 
-                    tmp0 = (d0 + d4) << CONST_BITS;
-                    tmp1 = (d0 - d4) << CONST_BITS;
+                    tmp0 = (d0 + d4) * (1 << CONST_BITS);
+                    tmp1 = (d0 - d4) * (1 << CONST_BITS);
 
                     tmp10 = tmp0 + tmp3;
                     tmp13 = tmp0 - tmp3;
@@ -1121,8 +1121,8 @@  void ff_j_rev_dct4(DCTBLOCK data)
                     tmp12 = tmp1 - tmp2;
             } else {
                     /* d0 != 0, d2 == 0, d4 != 0, d6 == 0 */
-                    tmp10 = tmp13 = (d0 + d4) << CONST_BITS;
-                    tmp11 = tmp12 = (d0 - d4) << CONST_BITS;
+                    tmp10 = tmp13 = (d0 + d4) * (1 << CONST_BITS);
+                    tmp11 = tmp12 = (d0 - d4) * (1 << CONST_BITS);
             }
     }