diff mbox series

[FFmpeg-devel] libavcodec/jpeg2000dec: Fix RPCL progression order

Message ID 20200708192019.8051-1-gautamramk@gmail.com
State Superseded
Headers show
Series [FFmpeg-devel] libavcodec/jpeg2000dec: Fix RPCL progression order | expand

Checks

Context Check Description
andriy/default pending
andriy/make success Make finished
andriy/make_fate success Make fate finished

Commit Message

Gautam Ramakrishnan July 8, 2020, 7:20 p.m. UTC 
From: Gautam Ramakrishnan <gautamramk@gmail.com>

This patch fixes a check in the RPCL progression order,
making it similar to the openjpeg implementation.
---
 libavcodec/jpeg2000dec.c | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

Comments

Carl Eugen Hoyos July 8, 2020, 8:06 p.m. UTC | #1 
Am Mi., 8. Juli 2020 um 21:30 Uhr schrieb Gautam Ramakrishnan
<gautamramk@gmail.com>:

> This patch was verified on p1_07.j2k

Then please add this (important) information to the commit
message, we don't want to get "more similar to openjpeg",
you can therefore remove this sentence.

Carl Eugen
diff mbox series

Patch

diff --git a/libavcodec/jpeg2000dec.c b/libavcodec/jpeg2000dec.c
index 18a933077e..3ea939f7d9 100644
--- a/libavcodec/jpeg2000dec.c
+++ b/libavcodec/jpeg2000dec.c
@@ -1393,22 +1393,28 @@  static int jpeg2000_decode_packets_po_iteration(Jpeg2000DecoderContext *s, Jpeg2
                         uint8_t reducedresno = codsty->nreslevels - 1 -reslevelno; //  ==> N_L - r
                         Jpeg2000ResLevel *rlevel = comp->reslevel + reslevelno;
                         unsigned prcx, prcy;
+                        int trx0, try0;
 
-                        int xc = x / s->cdx[compno];
-                        int yc = y / s->cdy[compno];
+                        if (!s->cdx[compno] || !s->cdy[compno])
+                            return AVERROR_INVALIDDATA;
+
+                        trx0 = ff_jpeg2000_ceildiv(tile->coord[0][0], s->cdx[compno] << reslevelno);
+                        try0 = ff_jpeg2000_ceildiv(tile->coord[1][0], s->cdy[compno] << reslevelno);
 
                         if (reslevelno >= codsty->nreslevels)
                             continue;
 
-                        if (yc % (1LL << (rlevel->log2_prec_height + reducedresno)) && y != tile->coord[1][0]) //FIXME this is a subset of the check
+                        if (!(y % ((uint64_t)s->cdy[compno] << (rlevel->log2_prec_height + reducedresno)) == 0 ||
+                             (y == tile->coord[1][0] && (try0 << reducedresno) % (1U << (reducedresno + rlevel->log2_prec_height)))))
                             continue;
 
-                        if (xc % (1LL << (rlevel->log2_prec_width + reducedresno)) && x != tile->coord[0][0]) //FIXME this is a subset of the check
+                        if (!(x % ((uint64_t)s->cdx[compno] << (rlevel->log2_prec_width + reducedresno)) == 0 ||
+                             (x == tile->coord[0][0] && (trx0 << reducedresno) % (1U << (reducedresno + rlevel->log2_prec_width)))))
                             continue;
 
                         // check if a precinct exists
-                        prcx   = ff_jpeg2000_ceildivpow2(xc, reducedresno) >> rlevel->log2_prec_width;
-                        prcy   = ff_jpeg2000_ceildivpow2(yc, reducedresno) >> rlevel->log2_prec_height;
+                        prcx   = ff_jpeg2000_ceildiv(x, s->cdx[compno] << reducedresno) >> rlevel->log2_prec_width;
+                        prcy   = ff_jpeg2000_ceildiv(y, s->cdy[compno] << reducedresno) >> rlevel->log2_prec_height;
                         prcx  -= ff_jpeg2000_ceildivpow2(comp->coord_o[0][0], reducedresno) >> rlevel->log2_prec_width;
                         prcy  -= ff_jpeg2000_ceildivpow2(comp->coord_o[1][0], reducedresno) >> rlevel->log2_prec_height;
 
@@ -1906,7 +1912,6 @@  static inline void tile_codeblocks(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile
                             continue;
                         x = cblk->coord[0][0] - band->coord[0][0];
                         y = cblk->coord[1][0] - band->coord[1][0];
-
                         if (comp->roi_shift)
                             roi_scale_cblk(cblk, comp, &t1);
                         if (codsty->transform == FF_DWT97)