diff mbox series

[FFmpeg-devel,2/2] swscale/input: Use more unsigned intermediates

Message ID 20221115221657.23290-2-michael@niedermayer.cc
State Accepted
Commit ba209e3d5142fd31bb6c3e05c5b183118a278afc
Headers show
Series [FFmpeg-devel,1/2] swscale/input: Use unsigned intermediates in rgb64ToUV_c_template | expand

Checks

Context Check Description
yinshiyou/make_loongarch64 success Make finished
yinshiyou/make_fate_loongarch64 success Make fate finished
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished

Commit Message

Michael Niedermayer Nov. 15, 2022, 10:16 p.m. UTC 
Same principle as previous commit, with sufficiently huge rgb2yuv table
values this produces wrong results and undefined behavior.
The unsigned produces the same incorrect results. That is probably
ok as these cases with huge values seem not to occur in any real
use case.

There are more cases but someone is refactoring them, so i didnt yet change
them.

Fixes: signed integer overflow
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libswscale/input.c | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

Comments

Michael Niedermayer Nov. 15, 2022, 10:19 p.m. UTC | #1 
On Tue, Nov 15, 2022 at 11:16:57PM +0100, Michael Niedermayer wrote:
> Same principle as previous commit, with sufficiently huge rgb2yuv table
> values this produces wrong results and undefined behavior.
> The unsigned produces the same incorrect results. That is probably
> ok as these cases with huge values seem not to occur in any real
> use case.
> 
> There are more cases but someone is refactoring them, so i didnt yet change
> them.
> 
> Fixes: signed integer overflow
> Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
> ---
>  libswscale/input.c | 24 ++++++++++++------------
>  1 file changed, 12 insertions(+), 12 deletions(-)

I can merge these 2 if people prefer

[...]
Michael Niedermayer Nov. 19, 2022, 9:41 p.m. UTC | #2 
On Tue, Nov 15, 2022 at 11:16:57PM +0100, Michael Niedermayer wrote:
> Same principle as previous commit, with sufficiently huge rgb2yuv table
> values this produces wrong results and undefined behavior.
> The unsigned produces the same incorrect results. That is probably
> ok as these cases with huge values seem not to occur in any real
> use case.
> 
> There are more cases but someone is refactoring them, so i didnt yet change
> them.
> 
> Fixes: signed integer overflow
> Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
> ---
>  libswscale/input.c | 24 ++++++++++++------------
>  1 file changed, 12 insertions(+), 12 deletions(-)

will apply patch set

[...]
diff mbox series

Patch

diff --git a/libswscale/input.c b/libswscale/input.c
index d7dbedd82f..d5676062a2 100644
--- a/libswscale/input.c
+++ b/libswscale/input.c
@@ -84,9 +84,9 @@  rgb64ToUV_half_c_template(uint16_t *dstU, uint16_t *dstV,
     int32_t rv = rgb2yuv[RV_IDX], gv = rgb2yuv[GV_IDX], bv = rgb2yuv[BV_IDX];
     av_assert1(src1==src2);
     for (i = 0; i < width; i++) {
-        int r_b = (input_pixel(&src1[8 * i + 0]) + input_pixel(&src1[8 * i + 4]) + 1) >> 1;
-        int   g = (input_pixel(&src1[8 * i + 1]) + input_pixel(&src1[8 * i + 5]) + 1) >> 1;
-        int b_r = (input_pixel(&src1[8 * i + 2]) + input_pixel(&src1[8 * i + 6]) + 1) >> 1;
+        unsigned r_b = (input_pixel(&src1[8 * i + 0]) + input_pixel(&src1[8 * i + 4]) + 1) >> 1;
+        unsigned   g = (input_pixel(&src1[8 * i + 1]) + input_pixel(&src1[8 * i + 5]) + 1) >> 1;
+        unsigned b_r = (input_pixel(&src1[8 * i + 2]) + input_pixel(&src1[8 * i + 6]) + 1) >> 1;
 
         dstU[i]= (ru*r + gu*g + bu*b + (0x10001<<(RGB2YUV_SHIFT-1))) >> RGB2YUV_SHIFT;
         dstV[i]= (rv*r + gv*g + bv*b + (0x10001<<(RGB2YUV_SHIFT-1))) >> RGB2YUV_SHIFT;
@@ -158,9 +158,9 @@  static av_always_inline void rgb48ToUV_c_template(uint16_t *dstU,
     int32_t rv = rgb2yuv[RV_IDX], gv = rgb2yuv[GV_IDX], bv = rgb2yuv[BV_IDX];
     av_assert1(src1 == src2);
     for (i = 0; i < width; i++) {
-        int r_b = input_pixel(&src1[i * 3 + 0]);
-        int g   = input_pixel(&src1[i * 3 + 1]);
-        int b_r = input_pixel(&src1[i * 3 + 2]);
+        unsigned r_b = input_pixel(&src1[i * 3 + 0]);
+        unsigned g   = input_pixel(&src1[i * 3 + 1]);
+        unsigned b_r = input_pixel(&src1[i * 3 + 2]);
 
         dstU[i] = (ru*r + gu*g + bu*b + (0x10001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT;
         dstV[i] = (rv*r + gv*g + bv*b + (0x10001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT;
@@ -180,12 +180,12 @@  static av_always_inline void rgb48ToUV_half_c_template(uint16_t *dstU,
     int32_t rv = rgb2yuv[RV_IDX], gv = rgb2yuv[GV_IDX], bv = rgb2yuv[BV_IDX];
     av_assert1(src1 == src2);
     for (i = 0; i < width; i++) {
-        int r_b = (input_pixel(&src1[6 * i + 0]) +
-                   input_pixel(&src1[6 * i + 3]) + 1) >> 1;
-        int g   = (input_pixel(&src1[6 * i + 1]) +
-                   input_pixel(&src1[6 * i + 4]) + 1) >> 1;
-        int b_r = (input_pixel(&src1[6 * i + 2]) +
-                   input_pixel(&src1[6 * i + 5]) + 1) >> 1;
+        unsigned r_b = (input_pixel(&src1[6 * i + 0]) +
+                        input_pixel(&src1[6 * i + 3]) + 1) >> 1;
+        unsigned g   = (input_pixel(&src1[6 * i + 1]) +
+                        input_pixel(&src1[6 * i + 4]) + 1) >> 1;
+        unsigned b_r = (input_pixel(&src1[6 * i + 2]) +
+                        input_pixel(&src1[6 * i + 5]) + 1) >> 1;
 
         dstU[i] = (ru*r + gu*g + bu*b + (0x10001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT;
         dstV[i] = (rv*r + gv*g + bv*b + (0x10001 << (RGB2YUV_SHIFT - 1))) >> RGB2YUV_SHIFT;