diff mbox series

[FFmpeg-devel,2/5] avcodec/rl2: Don't presume stride to be > 0

Message ID GV1P250MB07371360E432A8180AA1ED228F549@GV1P250MB0737.EURP250.PROD.OUTLOOK.COM
State Accepted
Commit b7848d1b2abfbd22b29fa88522e550d4f68cfc49
Headers show
Series [FFmpeg-devel,1/5] avcodec/rl2: Remove wrong check | expand

Checks

Context Check Description
andriy/make_x86 success Make finished
andriy/make_fate_x86 fail Make fate failed

Commit Message

Andreas Rheinhardt Sept. 28, 2022, 4:35 p.m. UTC 
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
Lots of fate tests fail if this assumption is not fulfilled.

 libavcodec/rl2.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
diff mbox series

Patch

diff --git a/libavcodec/rl2.c b/libavcodec/rl2.c
index 2464ad59ac..467c4913a4 100644
--- a/libavcodec/rl2.c
+++ b/libavcodec/rl2.c
@@ -65,7 +65,7 @@  static void rl2_rle_decode(Rl2Context *s, const uint8_t *in, int size,
     int i;
     const uint8_t *back_frame = s->back_frame;
     const uint8_t *in_end     = in + size;
-    const uint8_t *out_end    = out + stride * s->avctx->height;
+    const uint8_t *out_end    = out + stride * s->avctx->height - stride_adj;
     uint8_t *line_end;
 
     /** copy start of the background frame */
@@ -100,18 +100,20 @@  static void rl2_rle_decode(Rl2Context *s, const uint8_t *in, int size,
             *out++ = (val == 0x80) ? *back_frame : val;
             back_frame++;
             if (out == line_end) {
+                if (out == out_end)
+                    return;
                  out      += stride_adj;
                  line_end += stride;
-                 if (len >= out_end - out)
-                    return;
             }
         }
     }
 
     /** copy the rest from the background frame */
     if (s->back_frame) {
-        while (out < out_end) {
+        while (1) {
             memcpy(out, back_frame, line_end - out);
+            if (line_end == out_end)
+                break;
             back_frame += line_end - out;
             out         = line_end + stride_adj;
             line_end   += stride;