diff mbox series

[FFmpeg-devel,2/7] avcodec/sgidec: Support negative linesizes

Message ID GV1P250MB0737584845B197233BFAF0208F569@GV1P250MB0737.EURP250.PROD.OUTLOOK.COM
State Accepted
Commit 54cf0482bd4d58ccb5c913cf1e68468d759d74ca
Headers show
Series [FFmpeg-devel,1/7] avcodec/sunrast: Use ptrdiff_t for stride | expand

Commit Message

Andreas Rheinhardt Sept. 30, 2022, 5:05 p.m. UTC 
The earlier code used "p->data[0] + p->linesize[0] * s->height" with
the latter being unsigned, which gives the wrong value for negative
linesizes. There is also a not so obvious problem with this:
In case of negative linesizes, the last line is the start of
the allocated buffer, so using the line after the last line
would involve undefined pointer arithmetic. So don't do it.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
 libavcodec/sgidec.c | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)
diff mbox series

Patch

diff --git a/libavcodec/sgidec.c b/libavcodec/sgidec.c
index dd3dc46b48..a449859bf8 100644
--- a/libavcodec/sgidec.c
+++ b/libavcodec/sgidec.c
@@ -28,7 +28,7 @@ 
 typedef struct SgiState {
     AVCodecContext *avctx;
     unsigned int width;
-    unsigned int height;
+    int height;
     unsigned int depth;
     unsigned int bytes_per_channel;
     int linesize;
@@ -127,12 +127,11 @@  static int expand_rle_row16(SgiState *s, uint16_t *out_buf,
  * @param s the current image state
  * @return 0 if no error, else return error code.
  */
-static int read_rle_sgi(uint8_t *out_buf, SgiState *s)
+static int read_rle_sgi(uint8_t *last_line, SgiState *s)
 {
     uint8_t *dest_row;
     unsigned int len = s->height * s->depth * 4;
     GetByteContext g_table = s->g;
-    unsigned int y, z;
     unsigned int start_offset;
     int linesize, ret;
 
@@ -141,11 +140,10 @@  static int read_rle_sgi(uint8_t *out_buf, SgiState *s)
         return AVERROR_INVALIDDATA;
     }
 
-    for (z = 0; z < s->depth; z++) {
-        dest_row = out_buf;
-        for (y = 0; y < s->height; y++) {
+    for (unsigned z = 0; z < s->depth; z++) {
+        dest_row = last_line;
+        for (int remaining_lines = s->height;;) {
             linesize = s->width * s->depth;
-            dest_row -= s->linesize;
             start_offset = bytestream2_get_be32(&g_table);
             bytestream2_seek(&s->g, start_offset, SEEK_SET);
             if (s->bytes_per_channel == 1)
@@ -154,6 +152,9 @@  static int read_rle_sgi(uint8_t *out_buf, SgiState *s)
                 ret = expand_rle_row16(s, (uint16_t *)dest_row + z, linesize, s->depth);
             if (ret != s->width)
                 return AVERROR_INVALIDDATA;
+            if (--remaining_lines == 0)
+                break;
+            dest_row -= s->linesize;
         }
     }
     return 0;
@@ -204,7 +205,7 @@  static int decode_frame(AVCodecContext *avctx, AVFrame *p,
     SgiState *s = avctx->priv_data;
     unsigned int dimension, rle;
     int ret = 0;
-    uint8_t *out_buf, *out_end;
+    uint8_t *out_buf, *last_line;
 
     bytestream2_init(&s->g, avpkt->data, avpkt->size);
     if (bytestream2_get_bytes_left(&s->g) < SGI_HEADER_SIZE) {
@@ -258,14 +259,14 @@  static int decode_frame(AVCodecContext *avctx, AVFrame *p,
     p->key_frame = 1;
     out_buf = p->data[0];
 
-    out_end = out_buf + p->linesize[0] * s->height;
+    last_line = out_buf + p->linesize[0] * (s->height - 1);
 
     s->linesize = p->linesize[0];
 
     /* Skip header. */
     bytestream2_seek(&s->g, SGI_HEADER_SIZE, SEEK_SET);
     if (rle) {
-        ret = read_rle_sgi(out_end, s);
+        ret = read_rle_sgi(last_line, s);
     } else {
         ret = read_uncompressed_sgi(out_buf, s);
     }