diff mbox series

[FFmpeg-devel,7/7] avcodec/fraps: Fix segfault with negative linesizes

Message ID GV1P250MB07372CA9E506502EDFAAAB1B8F569@GV1P250MB0737.EURP250.PROD.OUTLOOK.COM
State Accepted
Commit 451b310d4f0ba5a38c3f67ec2ce39bcb62fcf59b
Headers show
Series [FFmpeg-devel,1/7] avcodec/sunrast: Use ptrdiff_t for stride | expand

Commit Message

Andreas Rheinhardt Sept. 30, 2022, 5:05 p.m. UTC 
Using unsigned and negative linesizes doesn't really work.
Use ptrdiff_t instead. This fixes the fraps-v0 and fraps-v1
FATE tests with negative linesizes.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
 libavcodec/fraps.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)
diff mbox series

Patch

diff --git a/libavcodec/fraps.c b/libavcodec/fraps.c
index 9c8cbf7323..4c4c46b602 100644
--- a/libavcodec/fraps.c
+++ b/libavcodec/fraps.c
@@ -141,7 +141,6 @@  static int decode_frame(AVCodecContext *avctx, AVFrame *f,
     int buf_size           = avpkt->size;
     uint32_t header;
     unsigned int version,header_size;
-    unsigned int x, y;
     const uint32_t *buf32;
     uint32_t *luma1,*luma2,*cb,*cr;
     uint32_t offs[4];
@@ -238,12 +237,12 @@  static int decode_frame(AVCodecContext *avctx, AVFrame *f,
         }
 
         buf32 = (const uint32_t*)buf;
-        for (y = 0; y < avctx->height / 2; y++) {
+        for (ptrdiff_t y = 0; y < avctx->height / 2; y++) {
             luma1 = (uint32_t*)&f->data[0][  y * 2      * f->linesize[0] ];
             luma2 = (uint32_t*)&f->data[0][ (y * 2 + 1) * f->linesize[0] ];
             cr    = (uint32_t*)&f->data[1][  y          * f->linesize[1] ];
             cb    = (uint32_t*)&f->data[2][  y          * f->linesize[2] ];
-            for (x = 0; x < avctx->width; x += 8) {
+            for (ptrdiff_t x = 0; x < avctx->width; x += 8) {
                 *luma1++ = *buf32++;
                 *luma1++ = *buf32++;
                 *luma2++ = *buf32++;
@@ -258,18 +257,18 @@  static int decode_frame(AVCodecContext *avctx, AVFrame *f,
         if (is_pal) {
             uint32_t *pal = (uint32_t *)f->data[1];
 
-            for (y = 0; y < 256; y++) {
+            for (unsigned y = 0; y < 256; y++) {
                 pal[y] = AV_RL32(buf) | 0xFF000000;
                 buf += 4;
             }
 
-            for (y = 0; y <avctx->height; y++)
+            for (ptrdiff_t y = 0; y < avctx->height; y++)
                 memcpy(&f->data[0][y * f->linesize[0]],
                        &buf[y * avctx->width],
                        avctx->width);
         } else {
         /* Fraps v1 is an upside-down BGR24 */
-            for (y = 0; y<avctx->height; y++)
+            for (ptrdiff_t y = 0; y < avctx->height; y++)
                 memcpy(&f->data[0][(avctx->height - y - 1) * f->linesize[0]],
                        &buf[y * avctx->width * 3],
                        3 * avctx->width);