| Message ID | 20240620193504.2310780-1-michael@niedermayer.cc |
|---|---|
| State | New |
| Headers | show |
| Series | [FFmpeg-devel,1/4] avcodec/j2kenc: Merge dwt_norm into lambda | expand |
| Context | Check | Description |
|---|---|---|
| yinshiyou/make_loongarch64 | success | Make finished |
| yinshiyou/make_fate_loongarch64 | success | Make fate finished |
| andriy/make_x86 | success | Make finished |
| andriy/make_fate_x86 | success | Make fate finished |
Michael Niedermayer: > This moves computations out of a loop > > Fixes: signed integer overflow: 31665934879948800 * 9998 cannot be represented in type 'long' > Fixes: 69024/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_JPEG2000_fuzzer-5949662967169024 > > Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg > Signed-off-by: Michael Niedermayer <michael@niedermayer.cc> > --- > libavcodec/j2kenc.c | 9 +++++---- > 1 file changed, 5 insertions(+), 4 deletions(-) > > diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c > index 8cf82f7216c..91e66d81048 100644 > --- a/libavcodec/j2kenc.c > +++ b/libavcodec/j2kenc.c > @@ -1349,7 +1349,7 @@ static void makelayers(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) > } > } > > -static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) > +static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda) > { > int passno, res = 0; > for (passno = 0; passno < cblk->npasses; passno++){ > @@ -1361,7 +1361,7 @@ static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) > dd = cblk->passes[passno].disto > - (res ? cblk->passes[res-1].disto : 0); > > - if (((dd * dwt_norm) >> WMSEDEC_SHIFT) * dwt_norm >= dr * lambda) > + if (dd >= dr * lambda) > res = passno+1; > } > return res; > @@ -1384,11 +1384,12 @@ static void truncpasses(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) > Jpeg2000Band *band = reslevel->band + bandno; > Jpeg2000Prec *prec = band->prec + precno; > > + int64_t dwt_norm = dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15; > + int64_t lambda_prime = av_rescale(s->lambda, 1 << WMSEDEC_SHIFT, dwt_norm * dwt_norm); > for (cblkno = 0; cblkno < prec->nb_codeblocks_height * prec->nb_codeblocks_width; cblkno++){ > Jpeg2000Cblk *cblk = prec->cblk + cblkno; > > - cblk->ninclpasses = getcut(cblk, s->lambda, > - (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15); > + cblk->ninclpasses = getcut(cblk, lambda_prime); > cblk->layers[0].data_start = cblk->data; > cblk->layers[0].cum_passes = cblk->ninclpasses; > cblk->layers[0].npasses = cblk->ninclpasses; Does this also fix the UB in the vsynth*-jpeg2000-yuva444p16 tests? - Andreas
On Fri, Jun 21, 2024 at 11:38:46AM +0200, Andreas Rheinhardt wrote: > Michael Niedermayer: > > This moves computations out of a loop > > > > Fixes: signed integer overflow: 31665934879948800 * 9998 cannot be represented in type 'long' > > Fixes: 69024/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_JPEG2000_fuzzer-5949662967169024 > > > > Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg > > Signed-off-by: Michael Niedermayer <michael@niedermayer.cc> > > --- > > libavcodec/j2kenc.c | 9 +++++---- > > 1 file changed, 5 insertions(+), 4 deletions(-) > > > > diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c > > index 8cf82f7216c..91e66d81048 100644 > > --- a/libavcodec/j2kenc.c > > +++ b/libavcodec/j2kenc.c > > @@ -1349,7 +1349,7 @@ static void makelayers(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) > > } > > } > > > > -static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) > > +static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda) > > { > > int passno, res = 0; > > for (passno = 0; passno < cblk->npasses; passno++){ > > @@ -1361,7 +1361,7 @@ static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) > > dd = cblk->passes[passno].disto > > - (res ? cblk->passes[res-1].disto : 0); > > > > - if (((dd * dwt_norm) >> WMSEDEC_SHIFT) * dwt_norm >= dr * lambda) > > + if (dd >= dr * lambda) > > res = passno+1; > > } > > return res; > > @@ -1384,11 +1384,12 @@ static void truncpasses(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) > > Jpeg2000Band *band = reslevel->band + bandno; > > Jpeg2000Prec *prec = band->prec + precno; > > > > + int64_t dwt_norm = dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15; > > + int64_t lambda_prime = av_rescale(s->lambda, 1 << WMSEDEC_SHIFT, dwt_norm * dwt_norm); > > for (cblkno = 0; cblkno < prec->nb_codeblocks_height * prec->nb_codeblocks_width; cblkno++){ > > Jpeg2000Cblk *cblk = prec->cblk + cblkno; > > > > - cblk->ninclpasses = getcut(cblk, s->lambda, > > - (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15); > > + cblk->ninclpasses = getcut(cblk, lambda_prime); > > cblk->layers[0].data_start = cblk->data; > > cblk->layers[0].cum_passes = cblk->ninclpasses; > > cblk->layers[0].npasses = cblk->ninclpasses; > > Does this also fix the UB in the vsynth*-jpeg2000-yuva444p16 tests? we will find out when this is applied, it looks like it might thx [...]
Hi, On Fri, Jun 21, 2024, 05:39 Andreas Rheinhardt < andreas.rheinhardt@outlook.com> wrote: > Michael Niedermayer: > > This moves computations out of a loop > > > > Fixes: signed integer overflow: 31665934879948800 * 9998 cannot be > represented in type 'long' > > Fixes: > 69024/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_JPEG2000_fuzzer-5949662967169024 > > > > Found-by: continuous fuzzing process > https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg > > Signed-off-by: Michael Niedermayer <michael@niedermayer.cc> > > --- > > libavcodec/j2kenc.c | 9 +++++---- > > 1 file changed, 5 insertions(+), 4 deletions(-) > > > > diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c > > index 8cf82f7216c..91e66d81048 100644 > > --- a/libavcodec/j2kenc.c > > +++ b/libavcodec/j2kenc.c > > @@ -1349,7 +1349,7 @@ static void makelayers(Jpeg2000EncoderContext *s, > Jpeg2000Tile *tile) > > } > > } > > > > -static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) > > +static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda) > > { > > int passno, res = 0; > > for (passno = 0; passno < cblk->npasses; passno++){ > > @@ -1361,7 +1361,7 @@ static int getcut(Jpeg2000Cblk *cblk, uint64_t > lambda, int dwt_norm) > > dd = cblk->passes[passno].disto > > - (res ? cblk->passes[res-1].disto : 0); > > > > - if (((dd * dwt_norm) >> WMSEDEC_SHIFT) * dwt_norm >= dr * > lambda) > > + if (dd >= dr * lambda) > > res = passno+1; > > } > > return res; > > @@ -1384,11 +1384,12 @@ static void truncpasses(Jpeg2000EncoderContext > *s, Jpeg2000Tile *tile) > > Jpeg2000Band *band = reslevel->band + bandno; > > Jpeg2000Prec *prec = band->prec + precno; > > > > + int64_t dwt_norm = dwt_norms[codsty->transform == > FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15; > > + int64_t lambda_prime = av_rescale(s->lambda, 1 << > WMSEDEC_SHIFT, dwt_norm * dwt_norm); > > for (cblkno = 0; cblkno < > prec->nb_codeblocks_height * prec->nb_codeblocks_width; cblkno++){ > > Jpeg2000Cblk *cblk = prec->cblk + cblkno; > > > > - cblk->ninclpasses = getcut(cblk, s->lambda, > > - (int64_t)dwt_norms[codsty->transform == > FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15); > > + cblk->ninclpasses = getcut(cblk, lambda_prime); > > cblk->layers[0].data_start = cblk->data; > > cblk->layers[0].cum_passes = cblk->ninclpasses; > > cblk->layers[0].npasses = cblk->ninclpasses; > > Does this also fix the UB in the vsynth*-jpeg2000-yuva444p16 tests? > > - Andreas > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". > I'm on staycation for my local jazz festival all week. Was anyone able to test if this fixed the stuff Andreas mentioned? -- Sean McGovern >
On Fri, Jun 21, 2024 at 11:09:45PM +0200, Michael Niedermayer wrote: > On Fri, Jun 21, 2024 at 11:38:46AM +0200, Andreas Rheinhardt wrote: > > Michael Niedermayer: > > > This moves computations out of a loop > > > > > > Fixes: signed integer overflow: 31665934879948800 * 9998 cannot be represented in type 'long' > > > Fixes: 69024/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_JPEG2000_fuzzer-5949662967169024 > > > > > > Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg > > > Signed-off-by: Michael Niedermayer <michael@niedermayer.cc> > > > --- > > > libavcodec/j2kenc.c | 9 +++++---- > > > 1 file changed, 5 insertions(+), 4 deletions(-) > > > > > > diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c > > > index 8cf82f7216c..91e66d81048 100644 > > > --- a/libavcodec/j2kenc.c > > > +++ b/libavcodec/j2kenc.c > > > @@ -1349,7 +1349,7 @@ static void makelayers(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) > > > } > > > } > > > > > > -static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) > > > +static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda) > > > { > > > int passno, res = 0; > > > for (passno = 0; passno < cblk->npasses; passno++){ > > > @@ -1361,7 +1361,7 @@ static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) > > > dd = cblk->passes[passno].disto > > > - (res ? cblk->passes[res-1].disto : 0); > > > > > > - if (((dd * dwt_norm) >> WMSEDEC_SHIFT) * dwt_norm >= dr * lambda) > > > + if (dd >= dr * lambda) > > > res = passno+1; > > > } > > > return res; > > > @@ -1384,11 +1384,12 @@ static void truncpasses(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) > > > Jpeg2000Band *band = reslevel->band + bandno; > > > Jpeg2000Prec *prec = band->prec + precno; > > > > > > + int64_t dwt_norm = dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15; > > > + int64_t lambda_prime = av_rescale(s->lambda, 1 << WMSEDEC_SHIFT, dwt_norm * dwt_norm); > > > for (cblkno = 0; cblkno < prec->nb_codeblocks_height * prec->nb_codeblocks_width; cblkno++){ > > > Jpeg2000Cblk *cblk = prec->cblk + cblkno; > > > > > > - cblk->ninclpasses = getcut(cblk, s->lambda, > > > - (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15); > > > + cblk->ninclpasses = getcut(cblk, lambda_prime); > > > cblk->layers[0].data_start = cblk->data; > > > cblk->layers[0].cum_passes = cblk->ninclpasses; > > > cblk->layers[0].npasses = cblk->ninclpasses; > > > > Does this also fix the UB in the vsynth*-jpeg2000-yuva444p16 tests? > > we will find out when this is applied, it looks like it might added a note that this may fix UB in these tests to teh commit message and will apply thx [...]
On Thu, Jun 20, 2024 at 09:35:01PM +0200, Michael Niedermayer wrote: > This moves computations out of a loop > > Fixes: signed integer overflow: 31665934879948800 * 9998 cannot be represented in type 'long' > Fixes: 69024/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_JPEG2000_fuzzer-5949662967169024 > > Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg > Signed-off-by: Michael Niedermayer <michael@niedermayer.cc> > --- > libavcodec/j2kenc.c | 9 +++++---- > 1 file changed, 5 insertions(+), 4 deletions(-) will apply patchset [...]
diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c index 8cf82f7216c..91e66d81048 100644 --- a/libavcodec/j2kenc.c +++ b/libavcodec/j2kenc.c @@ -1349,7 +1349,7 @@ static void makelayers(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) } } -static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) +static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda) { int passno, res = 0; for (passno = 0; passno < cblk->npasses; passno++){ @@ -1361,7 +1361,7 @@ static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) dd = cblk->passes[passno].disto - (res ? cblk->passes[res-1].disto : 0); - if (((dd * dwt_norm) >> WMSEDEC_SHIFT) * dwt_norm >= dr * lambda) + if (dd >= dr * lambda) res = passno+1; } return res; @@ -1384,11 +1384,12 @@ static void truncpasses(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) Jpeg2000Band *band = reslevel->band + bandno; Jpeg2000Prec *prec = band->prec + precno; + int64_t dwt_norm = dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15; + int64_t lambda_prime = av_rescale(s->lambda, 1 << WMSEDEC_SHIFT, dwt_norm * dwt_norm); for (cblkno = 0; cblkno < prec->nb_codeblocks_height * prec->nb_codeblocks_width; cblkno++){ Jpeg2000Cblk *cblk = prec->cblk + cblkno; - cblk->ninclpasses = getcut(cblk, s->lambda, - (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15); + cblk->ninclpasses = getcut(cblk, lambda_prime); cblk->layers[0].data_start = cblk->data; cblk->layers[0].cum_passes = cblk->ninclpasses; cblk->layers[0].npasses = cblk->ninclpasses;
This moves computations out of a loop Fixes: signed integer overflow: 31665934879948800 * 9998 cannot be represented in type 'long' Fixes: 69024/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_JPEG2000_fuzzer-5949662967169024 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer <michael@niedermayer.cc> --- libavcodec/j2kenc.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)