diff mbox series

[FFmpeg-devel,1/2] avcodec/jpeg2000dec: fix HT block decoder issue #10905

Message ID F0908E26-AFC5-4F2F-8940-7E04177A7648@es.takushoku-u.ac.jp
State New
Headers show
Series [FFmpeg-devel,1/2] avcodec/jpeg2000dec: fix HT block decoder issue #10905 | expand

Checks

Context Check Description
yinshiyou/configure_loongarch64 warning Failed to apply patch
andriy/configure_x86 warning Failed to apply patch

Commit Message

WATANABE Osamu May 17, 2024, 7:48 p.m. UTC
Signed-off-by: Pierre-Anthony Lemieux <pal@palemieux.com>
---
libavcodec/jpeg2000htdec.c | 122 +++++++++++++++++++------------------
1 file changed, 62 insertions(+), 60 deletions(-)
diff mbox series

Patch

diff --git a/libavcodec/jpeg2000htdec.c b/libavcodec/jpeg2000htdec.c
index fa704b665e..62f70c32a8 100644
--- a/libavcodec/jpeg2000htdec.c
+++ b/libavcodec/jpeg2000htdec.c
@@ -196,7 +196,7 @@  static void jpeg2000_bitbuf_refill_forward(StateVars *buffer, const uint8_t *arr
    while (buffer->bits_left < 32) {
        buffer->tmp = 0xFF;
        buffer->bits = (buffer->last == 0xFF) ? 7 : 8;
-        if (buffer->pos <= length) {
+        if (buffer->pos < length) {
            buffer->tmp = array[buffer->pos];
            buffer->pos += 1;
            buffer->last = buffer->tmp;
@@ -508,17 +508,17 @@  static int jpeg2000_decode_sig_emb(const Jpeg2000DecoderContext *s, MelDecoderSt
}

av_always_inline
-static int jpeg2000_get_state(int x1, int x2, int width, int shift_by,
+static int jpeg2000_get_state(int x1, int x2, int stride, int shift_by,
                              const uint8_t *block_states)
{
-    return (block_states[(x1 + 1) * (width + 2) + (x2 + 1)] >> shift_by) & 1;
+    return (block_states[(x1 + 1) * stride + (x2 + 1)] >> shift_by) & 1;
}

av_always_inline
-static void jpeg2000_modify_state(int x1, int x2, int width,
-                                 int value, uint8_t *block_states)
+static void jpeg2000_modify_state(int x1, int x2, int stride,
+                                  int value, uint8_t *block_states)
{
-    block_states[(x1 + 1) * (width + 2) + (x2 + 1)] |= value;
+    block_states[(x1 + 1) * stride + (x2 + 1)] |= value;
}

av_always_inline
@@ -528,8 +528,8 @@  static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s,
                                              StateVars *mel_stream, StateVars *vlc_stream,
                                              StateVars *mag_sgn_stream, const uint8_t *Dcup,
                                              uint32_t Lcup, uint32_t Pcup, uint8_t pLSB,
-                                              int width, int height, int32_t *sample_buf,
-                                              uint8_t *block_states)
+                                              int width, int height, const int stride,
+                                              int32_t *sample_buf, uint8_t *block_states)
{
    uint16_t q                      = 0;     // Represents current quad position
    uint16_t q1, q2;
@@ -958,32 +958,32 @@  static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s,
            j1 = 2 * y;
            j2 = 2 * x;

-            sample_buf[j2 + (j1 * width)] = (int32_t)*mu;
-            jpeg2000_modify_state(j1, j2, width, *sigma, block_states);
+            sample_buf[j2 + (j1 * stride)] = (int32_t)*mu;
+            jpeg2000_modify_state(j1, j2, stride, *sigma, block_states);
            sigma += 1;
            mu += 1;

            x1 = y != quad_height - 1 || is_border_y == 0;
-            sample_buf[j2 + ((j1 + 1) * width)] = ((int32_t)*mu) * x1;
-            jpeg2000_modify_state(j1 + 1, j2, width, (*sigma) * x1, block_states);
+            sample_buf[j2 + ((j1 + 1) * stride)] = ((int32_t)*mu) * x1;
+            jpeg2000_modify_state(j1 + 1, j2, stride, (*sigma) * x1, block_states);
            sigma += 1;
            mu += 1;

            x2 = x != quad_width - 1 || is_border_x == 0;
-            sample_buf[(j2 + 1) + (j1 * width)] = ((int32_t)*mu) * x2;
-            jpeg2000_modify_state(j1, j2 + 1, width, (*sigma) * x2, block_states);
+            sample_buf[(j2 + 1) + (j1 * stride)] = ((int32_t)*mu) * x2;
+            jpeg2000_modify_state(j1, j2 + 1, stride, (*sigma) * x2, block_states);
            sigma += 1;
            mu += 1;

            x3 = x1 | x2;
-            sample_buf[(j2 + 1) + (j1 + 1) * width] = ((int32_t)*mu) * x3;
-            jpeg2000_modify_state(j1 + 1, j2 + 1, width, (*sigma) * x3, block_states);
+            sample_buf[(j2 + 1) + (j1 + 1) * stride] = ((int32_t)*mu) * x3;
+            jpeg2000_modify_state(j1 + 1, j2 + 1, stride, (*sigma) * x3, block_states);
            sigma += 1;
            mu += 1;
        }
    }
    ret = 1;
-free:
+    free:
    av_freep(&sigma_n);
    av_freep(&E);
    av_freep(&mu_n);
@@ -992,39 +992,39 @@  free:

static void jpeg2000_calc_mbr(uint8_t *mbr, const uint16_t i, const uint16_t j,
                              const uint32_t mbr_info, uint8_t causal_cond,
-                              uint8_t *block_states, int width)
+                              uint8_t *block_states, int stride)
{
    int local_mbr = 0;

-    local_mbr |= jpeg2000_get_state(i - 1, j - 1, width, HT_SHIFT_SIGMA, block_states);
-    local_mbr |= jpeg2000_get_state(i - 1, j + 0, width, HT_SHIFT_SIGMA, block_states);
-    local_mbr |= jpeg2000_get_state(i - 1, j + 1, width, HT_SHIFT_SIGMA, block_states);
+    local_mbr |= jpeg2000_get_state(i - 1, j - 1, stride, HT_SHIFT_SIGMA, block_states);
+    local_mbr |= jpeg2000_get_state(i - 1, j + 0, stride, HT_SHIFT_SIGMA, block_states);
+    local_mbr |= jpeg2000_get_state(i - 1, j + 1, stride, HT_SHIFT_SIGMA, block_states);

-    local_mbr |= jpeg2000_get_state(i + 0, j - 1, width, HT_SHIFT_SIGMA, block_states);
-    local_mbr |= jpeg2000_get_state(i + 0, j + 1, width, HT_SHIFT_SIGMA, block_states);
+    local_mbr |= jpeg2000_get_state(i + 0, j - 1, stride, HT_SHIFT_SIGMA, block_states);
+    local_mbr |= jpeg2000_get_state(i + 0, j + 1, stride, HT_SHIFT_SIGMA, block_states);

-    local_mbr |= jpeg2000_get_state(i + 1, j - 1, width, HT_SHIFT_SIGMA, block_states) * causal_cond;
-    local_mbr |= jpeg2000_get_state(i + 1, j + 0, width, HT_SHIFT_SIGMA, block_states) * causal_cond;
-    local_mbr |= jpeg2000_get_state(i + 1, j + 1, width, HT_SHIFT_SIGMA, block_states) * causal_cond;
+    local_mbr |= jpeg2000_get_state(i + 1, j - 1, stride, HT_SHIFT_SIGMA, block_states) * causal_cond;
+    local_mbr |= jpeg2000_get_state(i + 1, j + 0, stride, HT_SHIFT_SIGMA, block_states) * causal_cond;
+    local_mbr |= jpeg2000_get_state(i + 1, j + 1, stride, HT_SHIFT_SIGMA, block_states) * causal_cond;

-    local_mbr |= jpeg2000_get_state(i - 1, j - 1, width, HT_SHIFT_REF,   block_states) *
-                 jpeg2000_get_state(i - 1, j - 1, width, HT_SHIFT_SCAN,  block_states);
-    local_mbr |= jpeg2000_get_state(i - 1, j + 0, width, HT_SHIFT_REF,   block_states) *
-                 jpeg2000_get_state(i - 1, j - 1, width, HT_SHIFT_SCAN,  block_states);
-    local_mbr |= jpeg2000_get_state(i - 1, j + 1, width, HT_SHIFT_REF,   block_states) *
-                 jpeg2000_get_state(i - 1, j + 1, width, HT_SHIFT_SCAN,  block_states);
+    local_mbr |= jpeg2000_get_state(i - 1, j - 1, stride, HT_SHIFT_REF, block_states) *
+                 jpeg2000_get_state(i - 1, j - 1, stride, HT_SHIFT_SCAN, block_states);
+    local_mbr |= jpeg2000_get_state(i - 1, j + 0, stride, HT_SHIFT_REF, block_states) *
+                 jpeg2000_get_state(i - 1, j - 1, stride, HT_SHIFT_SCAN, block_states);
+    local_mbr |= jpeg2000_get_state(i - 1, j + 1, stride, HT_SHIFT_REF, block_states) *
+                 jpeg2000_get_state(i - 1, j + 1, stride, HT_SHIFT_SCAN, block_states);

-    local_mbr |= jpeg2000_get_state(i + 0, j - 1, width, HT_SHIFT_REF,   block_states) *
-                 jpeg2000_get_state(i + 0, j - 1, width, HT_SHIFT_SCAN,  block_states);
-    local_mbr |= jpeg2000_get_state(i + 0, j + 1, width, HT_SHIFT_REF,   block_states) *
-                 jpeg2000_get_state(i + 0, j + 1, width, HT_SHIFT_SCAN,  block_states);
+    local_mbr |= jpeg2000_get_state(i + 0, j - 1, stride, HT_SHIFT_REF, block_states) *
+                 jpeg2000_get_state(i + 0, j - 1, stride, HT_SHIFT_SCAN, block_states);
+    local_mbr |= jpeg2000_get_state(i + 0, j + 1, stride, HT_SHIFT_REF, block_states) *
+                 jpeg2000_get_state(i + 0, j + 1, stride, HT_SHIFT_SCAN, block_states);

-    local_mbr |= jpeg2000_get_state(i + 1, j - 1, width, HT_SHIFT_REF,  block_states) *
-                 jpeg2000_get_state(i + 1, j - 1, width, HT_SHIFT_SCAN, block_states) * causal_cond;
-    local_mbr |= jpeg2000_get_state(i + 1, j + 0, width, HT_SHIFT_REF,  block_states) *
-                 jpeg2000_get_state(i + 1, j + 0, width, HT_SHIFT_SCAN, block_states) * causal_cond;
-    local_mbr |= jpeg2000_get_state(i + 1, j + 1, width, HT_SHIFT_REF,  block_states) *
-                 jpeg2000_get_state(i + 1, j + 1, width, HT_SHIFT_SCAN, block_states) * causal_cond;
+    local_mbr |= jpeg2000_get_state(i + 1, j - 1, stride, HT_SHIFT_REF, block_states) *
+                 jpeg2000_get_state(i + 1, j - 1, stride, HT_SHIFT_SCAN, block_states) * causal_cond;
+    local_mbr |= jpeg2000_get_state(i + 1, j + 0, stride, HT_SHIFT_REF, block_states) *
+                 jpeg2000_get_state(i + 1, j + 0, stride, HT_SHIFT_SCAN, block_states) * causal_cond;
+    local_mbr |= jpeg2000_get_state(i + 1, j + 1, stride, HT_SHIFT_REF, block_states) *
+                 jpeg2000_get_state(i + 1, j + 1, stride, HT_SHIFT_SCAN, block_states) * causal_cond;

    *mbr |= local_mbr;
}
@@ -1040,18 +1040,18 @@  static void jpeg2000_process_stripes_block(StateVars *sig_prop, int i_s, int j_s
            int modify_state, cond;
            uint8_t bit;
            uint8_t causal_cond = i != (i_s + height - 1);
-            int32_t *sp = &sample_buf[j + (i * (stride - 2))];
+            int32_t *sp = &sample_buf[j + (i * (stride))];
            uint8_t mbr = 0;

            if (jpeg2000_get_state(i, j, stride - 2, HT_SHIFT_SIGMA, block_states) == 0)
-                jpeg2000_calc_mbr(&mbr, i, j, mbr_info & 0x1EF, causal_cond, block_states, stride - 2);
+                jpeg2000_calc_mbr(&mbr, i, j, mbr_info & 0x1EF, causal_cond, block_states, stride);
            mbr_info >>= 3;
            cond = mbr != 0;
            bit = jpeg2000_peek_bit(sig_prop, magref_segment, magref_length);
            *sp |= (bit * cond) << pLSB;
            sig_prop->bits -= cond;
            modify_state = (((1 << HT_SHIFT_REF_IND) | (1 << HT_SHIFT_REF)) * cond) | 1 << HT_SHIFT_SCAN;
-            jpeg2000_modify_state(i, j, stride - 2, modify_state, block_states);
+            jpeg2000_modify_state(i, j, stride, modify_state, block_states);
        }
    }
}
@@ -1060,8 +1060,8 @@  static void jpeg2000_process_stripes_block(StateVars *sig_prop, int i_s, int j_s
 * See procedure decodeSigPropMag at Rec. ITU-T T.814, 7.4.
*/
av_noinline
-static void jpeg2000_decode_sigprop_segment(Jpeg2000Cblk *cblk, uint16_t width,
-                                            uint16_t height, uint8_t *magref_segment,
+static void jpeg2000_decode_sigprop_segment(Jpeg2000Cblk *cblk, uint16_t width, uint16_t height,
+                                            const int stride, uint8_t *magref_segment,
                                            uint32_t magref_length, uint8_t pLSB,
                                            int32_t *sample_buf, uint8_t *block_states)
{
@@ -1071,7 +1071,6 @@  static void jpeg2000_decode_sigprop_segment(Jpeg2000Cblk *cblk, uint16_t width,
    const uint16_t num_h_stripe = width / 4;
    int b_width                 = 4;
    int b_height                = 4;
-    int stride                  = width + 2;

    int last_width;
    uint16_t i = 0, j = 0;
@@ -1114,7 +1113,7 @@  static void jpeg2000_decode_sigprop_segment(Jpeg2000Cblk *cblk, uint16_t width,
 * See procedure decodeSigPropMag at Rec. ITU-T T.814, 7.5.
*/
static int
-jpeg2000_decode_magref_segment( uint16_t width, uint16_t block_height,
+jpeg2000_decode_magref_segment( uint16_t width, uint16_t block_height, const int stride,
                                uint8_t *magref_segment,uint32_t magref_length,
                                uint8_t pLSB, int32_t *sample_buf, uint8_t *block_states)
{
@@ -1134,7 +1133,7 @@  jpeg2000_decode_magref_segment( uint16_t width, uint16_t block_height,
                 *  We move column wise, going from one quad to another. See
                 *  Rec. ITU-T T.814, Figure 7.
                 */
-                sp = &sample_buf[j + i * width];
+                sp = &sample_buf[j + i * stride];
                if (jpeg2000_get_state(i, j, width, HT_SHIFT_SIGMA, block_states) != 0) {
                    jpeg2000_modify_state(i, j, width, 1 << HT_SHIFT_REF_IND, block_states);
                    *sp |= jpeg2000_import_magref_bit(&mag_ref, magref_segment, magref_length) << pLSB;
@@ -1146,7 +1145,7 @@  jpeg2000_decode_magref_segment( uint16_t width, uint16_t block_height,
    height = block_height % 4;
    for (int j = 0; j < width; j++) {
        for (int i = i_start; i < i_start + height; i++) {
-            sp = &sample_buf[j + i * width];
+            sp = &sample_buf[j + i * stride];
            if (jpeg2000_get_state(i, j, width, HT_SHIFT_SIGMA, block_states) != 0) {
                jpeg2000_modify_state(i, j, width, 1 << HT_SHIFT_REF_IND, block_states);
                *sp |= jpeg2000_import_magref_bit(&mag_ref, magref_segment, magref_length) << pLSB;
@@ -1194,6 +1193,9 @@  ff_jpeg2000_decode_htj2k(const Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *c

    int32_t M_b = magp;

+    const int quad_buf_width = width + 4;
+    const int quad_buf_height = height + 4;
+
    /* codeblock size as constrained by Rec. ITU-T T.800, Table A.18 */
    av_assert0(width <= 1024U && height <= 1024U);
    av_assert0(width * height <= 4096);
@@ -1261,23 +1263,23 @@  ff_jpeg2000_decode_htj2k(const Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *c

    jpeg2000_init_mel_decoder(&mel_state);

-    sample_buf = av_calloc((width + 4) * (height + 4), sizeof(int32_t));
-    block_states = av_calloc((width + 4) * (height + 4), sizeof(uint8_t));
+    sample_buf = av_calloc(quad_buf_width * quad_buf_height, sizeof(int32_t));
+    block_states = av_calloc(quad_buf_width * quad_buf_height, sizeof(uint8_t));

    if (!sample_buf || !block_states) {
        ret = AVERROR(ENOMEM);
        goto free;
    }
    if ((ret = jpeg2000_decode_ht_cleanup_segment(s, cblk, t1, &mel_state, &mel, &vlc,
-                                          &mag_sgn, Dcup, Lcup, Pcup, pLSB, width,
-                                          height, sample_buf, block_states)) < 0) {
+                                                  &mag_sgn, Dcup, Lcup, Pcup, pLSB, width,
+                                                  height, quad_buf_width, sample_buf, block_states)) < 0) {
        av_log(s->avctx, AV_LOG_ERROR, "Bad HT cleanup segment\n");
        goto free;
    }

    if (cblk->npasses > 1)
-        jpeg2000_decode_sigprop_segment(cblk, width, height, Dref, Lref,
-                                pLSB - 1, sample_buf, block_states);
+        jpeg2000_decode_sigprop_segment(cblk, width, height, quad_buf_width, Dref, Lref,
+                                        pLSB - 1, sample_buf, block_states);

    if (cblk->npasses > 2) {

@@ -1286,7 +1288,7 @@  ff_jpeg2000_decode_htj2k(const Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *c
            ret = AVERROR_INVALIDDATA;
            goto free;
        }
-        if ((ret = jpeg2000_decode_magref_segment(width, height, Dref, Lref,
+        if ((ret = jpeg2000_decode_magref_segment(width, height, quad_buf_width, Dref, Lref,
                                                  pLSB - 1, sample_buf, block_states)) < 0)
            goto free;
    }
@@ -1297,14 +1299,14 @@  ff_jpeg2000_decode_htj2k(const Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *c
    for (int y = 0; y < height; y++) {
        for (int x = 0; x < width; x++) {
            n = x + (y * t1->stride);
-            val = sample_buf[x + (y * width)];
+            val = sample_buf[x + (y * quad_buf_width)];
            /* Convert sign-magnitude to two's complement. */
            val = val >> 31 ? 0x80000000 - val : val;
            val >>= (pLSB - 1);
            t1->data[n] = val;
        }
    }
-free:
+    free:
    av_freep(&sample_buf);
    av_freep(&block_states);
    return ret;