diff mbox series

[FFmpeg-devel] libavcodec/jpeg2000dec: Fix RPCL progression order

Message ID 20200709060018.11888-1-gautamramk@gmail.com
State New
Headers show
Series [FFmpeg-devel] libavcodec/jpeg2000dec: Fix RPCL progression order
Related show

Checks

Context Check Description
andriy/default pending
andriy/make success Make finished
andriy/make_fate success Make fate finished

Commit Message

Gautam Ramakrishnan July 9, 2020, 6 a.m. UTC
From: Gautam Ramakrishnan <gautamramk@gmail.com>

This patch fixes a check in the RPCL progression order,
tested on p1_07.j2k.
---
 libavcodec/jpeg2000dec.c | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

Comments

Gautam Ramakrishnan July 10, 2020, 5:51 a.m. UTC | #1
On Thu, Jul 9, 2020 at 11:30 AM <gautamramk@gmail.com> wrote:
>
> From: Gautam Ramakrishnan <gautamramk@gmail.com>
>
> This patch fixes a check in the RPCL progression order,
> tested on p1_07.j2k.
> ---
>  libavcodec/jpeg2000dec.c | 19 ++++++++++++-------
>  1 file changed, 12 insertions(+), 7 deletions(-)
>
> diff --git a/libavcodec/jpeg2000dec.c b/libavcodec/jpeg2000dec.c
> index 18a933077e..3ea939f7d9 100644
> --- a/libavcodec/jpeg2000dec.c
> +++ b/libavcodec/jpeg2000dec.c
> @@ -1393,22 +1393,28 @@ static int jpeg2000_decode_packets_po_iteration(Jpeg2000DecoderContext *s, Jpeg2
>                          uint8_t reducedresno = codsty->nreslevels - 1 -reslevelno; //  ==> N_L - r
>                          Jpeg2000ResLevel *rlevel = comp->reslevel + reslevelno;
>                          unsigned prcx, prcy;
> +                        int trx0, try0;
>
> -                        int xc = x / s->cdx[compno];
> -                        int yc = y / s->cdy[compno];
> +                        if (!s->cdx[compno] || !s->cdy[compno])
> +                            return AVERROR_INVALIDDATA;
> +
> +                        trx0 = ff_jpeg2000_ceildiv(tile->coord[0][0], s->cdx[compno] << reslevelno);
> +                        try0 = ff_jpeg2000_ceildiv(tile->coord[1][0], s->cdy[compno] << reslevelno);
>
>                          if (reslevelno >= codsty->nreslevels)
>                              continue;
>
> -                        if (yc % (1LL << (rlevel->log2_prec_height + reducedresno)) && y != tile->coord[1][0]) //FIXME this is a subset of the check
> +                        if (!(y % ((uint64_t)s->cdy[compno] << (rlevel->log2_prec_height + reducedresno)) == 0 ||
> +                             (y == tile->coord[1][0] && (try0 << reducedresno) % (1U << (reducedresno + rlevel->log2_prec_height)))))
>                              continue;
>
> -                        if (xc % (1LL << (rlevel->log2_prec_width + reducedresno)) && x != tile->coord[0][0]) //FIXME this is a subset of the check
> +                        if (!(x % ((uint64_t)s->cdx[compno] << (rlevel->log2_prec_width + reducedresno)) == 0 ||
> +                             (x == tile->coord[0][0] && (trx0 << reducedresno) % (1U << (reducedresno + rlevel->log2_prec_width)))))
>                              continue;
>
>                          // check if a precinct exists
> -                        prcx   = ff_jpeg2000_ceildivpow2(xc, reducedresno) >> rlevel->log2_prec_width;
> -                        prcy   = ff_jpeg2000_ceildivpow2(yc, reducedresno) >> rlevel->log2_prec_height;
> +                        prcx   = ff_jpeg2000_ceildiv(x, s->cdx[compno] << reducedresno) >> rlevel->log2_prec_width;
> +                        prcy   = ff_jpeg2000_ceildiv(y, s->cdy[compno] << reducedresno) >> rlevel->log2_prec_height;
>                          prcx  -= ff_jpeg2000_ceildivpow2(comp->coord_o[0][0], reducedresno) >> rlevel->log2_prec_width;
>                          prcy  -= ff_jpeg2000_ceildivpow2(comp->coord_o[1][0], reducedresno) >> rlevel->log2_prec_height;
>
> @@ -1906,7 +1912,6 @@ static inline void tile_codeblocks(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile
>                              continue;
>                          x = cblk->coord[0][0] - band->coord[0][0];
>                          y = cblk->coord[1][0] - band->coord[1][0];
> -
>                          if (comp->roi_shift)
>                              roi_scale_cblk(cblk, comp, &t1);
>                          if (codsty->transform == FF_DWT97)
> --
> 2.17.1
>
Please ignore this patch, there is still a small error.
Sorry for the mistake. I shall send another patch.
diff mbox series

Patch

diff --git a/libavcodec/jpeg2000dec.c b/libavcodec/jpeg2000dec.c
index 18a933077e..3ea939f7d9 100644
--- a/libavcodec/jpeg2000dec.c
+++ b/libavcodec/jpeg2000dec.c
@@ -1393,22 +1393,28 @@  static int jpeg2000_decode_packets_po_iteration(Jpeg2000DecoderContext *s, Jpeg2
                         uint8_t reducedresno = codsty->nreslevels - 1 -reslevelno; //  ==> N_L - r
                         Jpeg2000ResLevel *rlevel = comp->reslevel + reslevelno;
                         unsigned prcx, prcy;
+                        int trx0, try0;
 
-                        int xc = x / s->cdx[compno];
-                        int yc = y / s->cdy[compno];
+                        if (!s->cdx[compno] || !s->cdy[compno])
+                            return AVERROR_INVALIDDATA;
+
+                        trx0 = ff_jpeg2000_ceildiv(tile->coord[0][0], s->cdx[compno] << reslevelno);
+                        try0 = ff_jpeg2000_ceildiv(tile->coord[1][0], s->cdy[compno] << reslevelno);
 
                         if (reslevelno >= codsty->nreslevels)
                             continue;
 
-                        if (yc % (1LL << (rlevel->log2_prec_height + reducedresno)) && y != tile->coord[1][0]) //FIXME this is a subset of the check
+                        if (!(y % ((uint64_t)s->cdy[compno] << (rlevel->log2_prec_height + reducedresno)) == 0 ||
+                             (y == tile->coord[1][0] && (try0 << reducedresno) % (1U << (reducedresno + rlevel->log2_prec_height)))))
                             continue;
 
-                        if (xc % (1LL << (rlevel->log2_prec_width + reducedresno)) && x != tile->coord[0][0]) //FIXME this is a subset of the check
+                        if (!(x % ((uint64_t)s->cdx[compno] << (rlevel->log2_prec_width + reducedresno)) == 0 ||
+                             (x == tile->coord[0][0] && (trx0 << reducedresno) % (1U << (reducedresno + rlevel->log2_prec_width)))))
                             continue;
 
                         // check if a precinct exists
-                        prcx   = ff_jpeg2000_ceildivpow2(xc, reducedresno) >> rlevel->log2_prec_width;
-                        prcy   = ff_jpeg2000_ceildivpow2(yc, reducedresno) >> rlevel->log2_prec_height;
+                        prcx   = ff_jpeg2000_ceildiv(x, s->cdx[compno] << reducedresno) >> rlevel->log2_prec_width;
+                        prcy   = ff_jpeg2000_ceildiv(y, s->cdy[compno] << reducedresno) >> rlevel->log2_prec_height;
                         prcx  -= ff_jpeg2000_ceildivpow2(comp->coord_o[0][0], reducedresno) >> rlevel->log2_prec_width;
                         prcy  -= ff_jpeg2000_ceildivpow2(comp->coord_o[1][0], reducedresno) >> rlevel->log2_prec_height;
 
@@ -1906,7 +1912,6 @@  static inline void tile_codeblocks(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile
                             continue;
                         x = cblk->coord[0][0] - band->coord[0][0];
                         y = cblk->coord[1][0] - band->coord[1][0];
-
                         if (comp->roi_shift)
                             roi_scale_cblk(cblk, comp, &t1);
                         if (codsty->transform == FF_DWT97)